diff options
Diffstat (limited to 'lib')
471 files changed, 30031 insertions, 9114 deletions
diff --git a/lib/.gitignore b/lib/.gitignore index 283393faa9..7cef9d7cf3 100644 --- a/lib/.gitignore +++ b/lib/.gitignore @@ -1,32 +1,3 @@ -# common test - -/common_test/doc/src/ct.xml -/common_test/doc/src/ct_cover.xml -/common_test/doc/src/ct_ftp.xml -/common_test/doc/src/ct_master.xml -/common_test/doc/src/ct_rpc.xml -/common_test/doc/src/ct_snmp.xml -/common_test/doc/src/ct_ssh.xml -/common_test/doc/src/ct_netconfc.xml -/common_test/doc/src/ct_telnet.xml -/common_test/doc/src/unix_telnet.xml - -# edoc - -/edoc/doc/src/chapter.xml -/edoc/doc/src/edoc.xml -/edoc/doc/src/edoc_doclet.xml -/edoc/doc/src/edoc_extract.xml -/edoc/doc/src/edoc_layout.xml -/edoc/doc/src/edoc_lib.xml -/edoc/doc/src/edoc_run.xml - -# eunit - -/eunit/doc/src/chapter.xml -/eunit/doc/src/eunit.xml -/eunit/doc/src/eunit_surefire.xml - # erl_interface /erl_interface/bin @@ -34,15 +5,6 @@ /erl_interface/obj.st /erl_interface/obj -# gs - -/gs/doc/src/gs_chapter2.xml -/gs/doc/src/gs_chapter4.xml -/gs/doc/src/gs_chapter5.xml -/gs/doc/src/gs_chapter6.xml -/gs/doc/src/gs_chapter7.xml -/gs/doc/src/gs_chapter8.xml - # megaco /megaco/src/binary/megaco_ber_bin_drv_media_gateway_control_prev3a.erl @@ -129,19 +91,6 @@ /megaco/src/text/megaco_text_parser_v1.erl /megaco/src/text/megaco_text_parser_v2.erl /megaco/src/text/megaco_text_parser_v3.erl -/megaco/doc/html/mstone1.jpg - -# mnesia - -/mnesia/doc/src/Mnesia_App_A.xml -/mnesia/doc/src/Mnesia_App_B.xml -/mnesia/doc/src/Mnesia_App_C.xml -/mnesia/doc/src/Mnesia_App_D.xml -/mnesia/doc/src/Mnesia_chap2.xml -/mnesia/doc/src/Mnesia_chap3.xml -/mnesia/doc/src/Mnesia_chap4.xml -/mnesia/doc/src/Mnesia_chap5.xml -/mnesia/doc/src/Mnesia_chap7.xml # orber & cos* applications @@ -525,39 +474,3 @@ /orber/src/oe_OrberIFR.hrl /orber/src/oe_erlang.erl /orber/src/oe_erlang.hrl - -# snmp - -snmp/doc/intex.html - -# syntax_tools - -/syntax_tools/doc/src/chapter.xml -/syntax_tools/doc/src/epp_dodger.xml -/syntax_tools/doc/src/erl_comment_scan.xml -/syntax_tools/doc/src/erl_prettypr.xml -/syntax_tools/doc/src/erl_recomment.xml -/syntax_tools/doc/src/erl_syntax.xml -/syntax_tools/doc/src/erl_syntax_lib.xml -/syntax_tools/doc/src/erl_tidy.xml -/syntax_tools/doc/src/merl.xml -/syntax_tools/doc/src/merl_transform.xml -/syntax_tools/doc/src/igor.xml -/syntax_tools/doc/src/prettypr.xml - -# wx - -/wx/doc/src/chapter.xml -/wx/doc/src/gl.xml -/wx/doc/src/glu.xml -/wx/doc/src/ref_man.xml - -# xmerl - -/xmerl/doc/src/xmerl.xml -/xmerl/doc/src/xmerl_eventp.xml -/xmerl/doc/src/xmerl_scan.xml -/xmerl/doc/src/xmerl_ug.xml -/xmerl/doc/src/xmerl_xpath.xml -/xmerl/doc/src/xmerl_xs.xml -/xmerl/doc/src/xmerl_xsd.xml diff --git a/lib/Makefile b/lib/Makefile index ae466ed518..d67e605875 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -35,7 +35,7 @@ ALL_ERLANG_APPLICATIONS = xmerl edoc erl_docgen snmp otp_mibs erl_interface \ public_key ssl observer odbc diameter \ cosTransactions cosEvent cosTime cosNotification \ cosProperty cosFileTransfer cosEventDomain et megaco \ - eunit ssh eldap dialyzer hipe + eunit ssh eldap dialyzer hipe ftp tftp ifdef BUILD_ALL ERLANG_APPLICATIONS += $(ALL_ERLANG_APPLICATIONS) diff --git a/lib/asn1/doc/src/Makefile b/lib/asn1/doc/src/Makefile index 9a388e4e8a..2b5d9467d9 100644 --- a/lib/asn1/doc/src/Makefile +++ b/lib/asn1/doc/src/Makefile @@ -51,13 +51,14 @@ XML_CHAPTER_FILES = \ asn1_introduction.xml \ asn1_getting_started.xml \ asn1_overview.xml \ - asn1_spec.xml \ notes.xml BOOK_FILES = book.xml XML_FILES = $(BOOK_FILES) $(XML_APPLICATION_FILES) $(XML_REF3_FILES) \ - $(GEN_XML) $(XML_PART_FILES) $(XML_CHAPTER_FILES) + $(XML_PART_FILES) $(XML_CHAPTER_FILES) + +XML_GEN_FILES = $(GEN_XML:%=$(XMLDIR)/%) GIF_FILES = \ exclusive_Win_But.gif \ @@ -75,7 +76,8 @@ EXTRA_FILES = \ $(DEFAULT_HTML_FILES) \ $(ASN1_FILES) \ $(XML_REF3_FILES:%.xml=$(HTMLDIR)/%.html) \ - $(XML_CHAPTER_FILES:%.xml=$(HTMLDIR)/%.html) + $(XML_CHAPTER_FILES:%.xml=$(HTMLDIR)/%.html) \ + $(GEN_XML:%.xml=$(HTMLDIR)/%.html) \ MAN3_FILES = $(XML_REF3_FILES:%.xml=$(MAN3DIR)/%.3) diff --git a/lib/asn1/src/asn1ct.erl b/lib/asn1/src/asn1ct.erl index 81a2735a0d..e9e9f6eb42 100644 --- a/lib/asn1/src/asn1ct.erl +++ b/lib/asn1/src/asn1ct.erl @@ -292,8 +292,7 @@ run_passes_1([{pass,Name,Pass}|Passes], #st{run=Run}=St0) done -> ok catch - Class:Error -> - Stk = erlang:get_stacktrace(), + Class:Error:Stk -> io:format("Internal error: ~p:~p\n~p\n", [Class,Error,Stk]), {error,{internal_error,{Class,Error}}} @@ -2390,13 +2389,13 @@ in_process(Fun) -> receive {Pid, Result} -> Result; {Pid, Class, Reason, Stack} -> - ST = try throw(x) catch throw:x -> erlang:get_stacktrace() end, + ST = try throw(x) catch throw:x:Stk -> Stk end, erlang:raise(Class, Reason, Stack ++ ST) end. process(Parent, Fun) -> try Parent ! {self(), Fun()} - catch Class:Reason -> - Parent ! {self(), Class, Reason, erlang:get_stacktrace()} + catch Class:Reason:Stack -> + Parent ! {self(), Class, Reason, Stack} end. diff --git a/lib/asn1/src/asn1ct_gen.erl b/lib/asn1/src/asn1ct_gen.erl index efbc6d6380..ee039dfbab 100644 --- a/lib/asn1/src/asn1ct_gen.erl +++ b/lib/asn1/src/asn1ct_gen.erl @@ -815,8 +815,7 @@ result_line_1(Items) -> try_catch() -> [" catch",nl, - " Class:Exception when Class =:= error; Class =:= exit ->",nl, - " Stk = erlang:get_stacktrace(),",nl, + " Class:Exception:Stk when Class =:= error; Class =:= exit ->",nl, " case Exception of",nl, " {error,{asn1,Reason}} ->",nl, " {error,{asn1,{Reason,Stk}}};",nl, diff --git a/lib/asn1/test/asn1_SUITE.erl b/lib/asn1/test/asn1_SUITE.erl index bfeffa969f..8cf6745103 100644 --- a/lib/asn1/test/asn1_SUITE.erl +++ b/lib/asn1/test/asn1_SUITE.erl @@ -227,10 +227,9 @@ test(Config, TestF, Rules) -> try TestF(C, R, O) catch - Class:Reason -> + Class:Reason:Stk -> NewReason = {Reason, [{rule, R}, {options, O}]}, - erlang:raise(Class, NewReason, - erlang:get_stacktrace()) + erlang:raise(Class, NewReason, Stk) end end, Result = [run_case(Config, Fun, rule(Rule), opts(Rule)) || Rule <- Rules], diff --git a/lib/asn1/test/testUniqueObjectSets.erl b/lib/asn1/test/testUniqueObjectSets.erl index cabdb44a0c..c75a673c4b 100644 --- a/lib/asn1/test/testUniqueObjectSets.erl +++ b/lib/asn1/test/testUniqueObjectSets.erl @@ -30,8 +30,7 @@ seq_roundtrip(I, D0) -> asn1_test_lib:map_roundtrip(M, 'Seq', Enc), {ok,{'Seq',I,D}} = M:decode('Seq', Enc), D - catch C:E -> - Stk = erlang:get_stacktrace(), + catch C:E:Stk -> io:format("FAILED: ~p ~p\n", [I,D0]), erlang:raise(C, E, Stk) end. diff --git a/lib/common_test/doc/src/ct_ftp.xml b/lib/common_test/doc/src/ct_ftp.xml index e8c6f72db7..a6f01dd58e 100644 --- a/lib/common_test/doc/src/ct_ftp.xml +++ b/lib/common_test/doc/src/ct_ftp.xml @@ -33,13 +33,11 @@ <file>ct_ftp.xml</file> </header> <module>ct_ftp</module> - <modulesummary>FTP client module (based on the FTP support of the Inets - application).</modulesummary> + <modulesummary>FTP client module (based on the FTP application).</modulesummary> <description> - <p>FTP client module (based on the FTP support of the <c>Inets</c> - application).</p> + <p>FTP client module (based on the <c>ftp</c> application).</p> </description> diff --git a/lib/common_test/src/Makefile b/lib/common_test/src/Makefile index 9d751996ad..2a2a9cb5bc 100644 --- a/lib/common_test/src/Makefile +++ b/lib/common_test/src/Makefile @@ -165,3 +165,5 @@ release_tests_spec: opt release_docs_spec: docs +# Include dependencies -- list below added by Kostis Sagonas +$(EBIN)/cth_log_redirect.beam: ../../kernel/include/logger.hrl diff --git a/lib/common_test/src/common_test.app.src b/lib/common_test/src/common_test.app.src index 9becde110b..f686003637 100644 --- a/lib/common_test/src/common_test.app.src +++ b/lib/common_test/src/common_test.app.src @@ -85,6 +85,7 @@ "crypto-3.6", "debugger-4.1", "erts-7.0", + "ftp-1.0.0", "inets-6.0", "kernel-4.0", "observer-2.1", diff --git a/lib/common_test/src/ct_config.erl b/lib/common_test/src/ct_config.erl index 6c87b11f8d..e33b47b0e8 100644 --- a/lib/common_test/src/ct_config.erl +++ b/lib/common_test/src/ct_config.erl @@ -660,7 +660,7 @@ decrypt_config_file(EncryptFileName, TargetFileName, {key,Key}) -> get_crypt_key_from_file(File) -> case file:read_file(File) of {ok,Bin} -> - case catch string:lexemes(binary_to_list(Bin), [$\n,$\r]) of + case catch string:lexemes(binary_to_list(Bin), [$\n, [$\r,$\n]]) of [Key] -> Key; _ -> @@ -694,7 +694,7 @@ get_crypt_key_from_file() -> noent -> Result; _ -> - case catch string:lexemes(binary_to_list(Result), [$\n,$\r]) of + case catch string:lexemes(binary_to_list(Result), [$\n, [$\r,$\n]]) of [Key] -> io:format("~nCrypt key file: ~ts~n", [FullName]), Key; diff --git a/lib/common_test/src/ct_config_plain.erl b/lib/common_test/src/ct_config_plain.erl index e77381d7cf..d525019f7b 100644 --- a/lib/common_test/src/ct_config_plain.erl +++ b/lib/common_test/src/ct_config_plain.erl @@ -106,7 +106,7 @@ read_config_terms1({done,{eof,EL},_}, L, _, _) -> read_config_terms1({done,{error,Info,EL},_}, L, _, _) -> {error,{Info,{L,EL}}}; read_config_terms1({more,_}, L, Terms, Rest) -> - case string:lexemes(Rest, [$\n,$\r,$\t]) of + case string:lexemes(Rest, [$\n,[$\r,$\n],$\t]) of [] -> lists:reverse(Terms); _ -> diff --git a/lib/common_test/src/ct_ftp.erl b/lib/common_test/src/ct_ftp.erl index ee4a6a6c45..8b02ae3a0f 100644 --- a/lib/common_test/src/ct_ftp.erl +++ b/lib/common_test/src/ct_ftp.erl @@ -18,7 +18,7 @@ %% %CopyrightEnd% %% -%%% @doc FTP client module (based on the FTP support of the INETS application). +%%% @doc FTP client module (based on the FTP application). %%% %%% @type connection() = handle() | ct:target_name() %%% @type handle() = ct_gen_conn:handle(). Handle for a specific @@ -292,8 +292,8 @@ init(KeyOrName,{IP,Port},{Username,Password}) -> end. ftp_connect(IP,Port,Username,Password) -> - _ = inets:start(), - case inets:start(ftpc,[{host,IP},{port,Port}]) of + _ = ftp:start(), + case ftp:start_service([{host,IP},{port,Port}]) of {ok,FtpPid} -> case ftp:user(FtpPid,Username,Password) of ok -> @@ -341,7 +341,7 @@ reconnect(_Addr,_State) -> terminate(FtpPid,State) -> log(heading(terminate,State#state.target_name), "Closing FTP connection.\nHandle: ~p\n",[FtpPid]), - inets:stop(ftpc,FtpPid). + ftp:stop_service(FtpPid). %%%================================================================= diff --git a/lib/common_test/src/cth_log_redirect.erl b/lib/common_test/src/cth_log_redirect.erl index b05f0bd28b..417ea615a3 100644 --- a/lib/common_test/src/cth_log_redirect.erl +++ b/lib/common_test/src/cth_log_redirect.erl @@ -33,17 +33,19 @@ pre_init_per_testcase/4, post_init_per_testcase/5, pre_end_per_testcase/4, post_end_per_testcase/5]). -%% Event handler Callbacks --export([init/1, - handle_event/2, handle_call/2, handle_info/2, - terminate/1, terminate/2, code_change/3]). +%% Logger handler and gen_server callbacks +-export([log/2, + init/1, + handle_cast/2, handle_call/3, + terminate/1, terminate/2]). %% Other -export([handle_remote_events/1]). -include("ct.hrl"). +-include("../../kernel/src/logger_internal.hrl"). --behaviour(gen_event). +-behaviour(gen_server). -record(eh_state, {log_func, curr_suite, @@ -57,7 +59,7 @@ id(_Opts) -> init(?MODULE, _Opts) -> ct_util:mark_process(), - error_logger:add_report_handler(?MODULE), + ok = start_log_handler(), tc_log_async. pre_init_per_suite(Suite, Config, State) -> @@ -100,7 +102,7 @@ pre_end_per_testcase(_Suite, _TC, Config, State) -> post_end_per_testcase(_Suite, _TC, _Config, Result, State) -> %% Make sure that the event queue is flushed %% before ending this test case. - gen_event:call(error_logger, ?MODULE, flush, 300000), + gen_server:call(?MODULE, flush, 300000), {Result, State}. pre_end_per_group(_Suite, Group, Config, {tc_log, Group}) -> @@ -114,127 +116,155 @@ post_end_per_group(_Suite, _Group, Config, Return, State) -> set_curr_func({group,undefined}, Config), {Return, State}. -%% Copied and modified from sasl_report_tty_h.erl -init(_Type) -> +start_log_handler() -> + case whereis(?MODULE) of + undefined -> + ChildSpec = + #{id=>?MODULE, + start=>{gen_server,start_link,[{local,?MODULE},?MODULE,[],[]]}, + restart=>transient, + shutdown=>2000, + type=>worker, + modules=>[?MODULE]}, + {ok,_} = supervisor:start_child(logger_sup,ChildSpec); + _Pid -> + ok + end, + ok = logger:add_handler(?MODULE,?MODULE, + #{level=>info, + formatter=>{?DEFAULT_FORMATTER, + ?DEFAULT_FORMAT_CONFIG}}). + +init([]) -> {ok, #eh_state{log_func = tc_log_async}}. -handle_event({_Type,GL,_Msg}, #eh_state{handle_remote_events = false} = State) - when node(GL) /= node() -> - {ok, State}; -handle_event(Event, #eh_state{log_func = LogFunc} = State) -> +log(#{msg:={report,Msg},meta:=#{domain:=[beam,erlang,otp,sasl]}}=Log,Config) -> case whereis(sasl_sup) of undefined -> - sasl_not_started; + ok; % sasl application is not started _Else -> - {ok, ErrLogType} = application:get_env(sasl, errlog_type), - SReport = sasl_report:format_report(group_leader(), ErrLogType, - tag_event(Event, local)), - if is_list(SReport) -> - SaslHeader = format_header(State), - case LogFunc of - tc_log -> - ct_logs:tc_log(sasl, ?STD_IMPORTANCE, - SaslHeader, SReport, [], []); - tc_log_async -> - ct_logs:tc_log_async(sasl, ?STD_IMPORTANCE, - SaslHeader, SReport, []) - end; - true -> %% Report is an atom if no logging is to be done - ignore - end - end, - %% note that error_logger (unlike sasl) expects UTC time - EReport = error_logger_tty_h:write_event( - tag_event(Event, utc), io_lib), - if is_list(EReport) -> - ErrHeader = format_header(State), - case LogFunc of - tc_log -> - ct_logs:tc_log(error_logger, ?STD_IMPORTANCE, - ErrHeader, EReport, [], []); - tc_log_async -> - ct_logs:tc_log_async(error_logger, ?STD_IMPORTANCE, - ErrHeader, EReport, []) - end; - true -> %% Report is an atom if no logging is to be done - ignore - end, - {ok, State}. - -handle_info({'EXIT',User,killed}, State) -> - case whereis(user) of - %% init:stop/1/2 has been called, let's finish! - undefined -> - remove_handler; - User -> - remove_handler; - _ -> - {ok,State} + Level = + case application:get_env(sasl, errlog_type) of + {ok,error} -> + error; + {ok,_} -> + info; + undefined -> + info + end, + case Level of + error -> + case Msg of + #{label:={_,progress}} -> + ok; + _ -> + do_log(add_log_category(Log,sasl),Config) + end; + _ -> + do_log(add_log_category(Log,sasl),Config) + end end; +log(#{meta:=#{domain:=[beam,erlang,otp]}}=Log,Config) -> + do_log(add_log_category(Log,error_logger),Config); +log(#{meta:=#{domain:=_}},_) -> + ok; +log(Log,Config) -> + do_log(add_log_category(Log,error_logger),Config). + +add_log_category(#{meta:=Meta}=Log,Category) -> + Log#{meta=>Meta#{?MODULE=>#{category=>Category}}}. -handle_info(_, State) -> - {ok,State}. +do_log(Log,Config) -> + gen_server:call(?MODULE,{log,Log,Config}). -handle_call(flush,State) -> - {ok, ok, State}; +handle_cast(_, State) -> + {noreply,State}. -handle_call({set_curr_func,{group,Group,Conf},Config}, - State) when is_list(Config) -> +handle_call({log,#{meta:=#{gl:=GL}},_}, _From, + #eh_state{handle_remote_events=false}=State) + when node(GL) /= node() -> + {reply, ok, State}; + +handle_call({log, + #{msg:=Msg0, + meta:=#{?MODULE:=#{category:=Category}}=Meta}=Log, + #{formatter:={Formatter,FConfig}}}, + _From, + #eh_state{log_func=LogFunc}=State) -> + Header = format_header(State), + Msg = + case Msg0 of + {report,R} -> + Fun=maps:get(report_cb,Meta,fun logger:format_report/1), + Fun(R); + _ -> + Msg0 + end, + String = Formatter:format(Log#{msg=>Msg},FConfig), + case LogFunc of + tc_log -> + ct_logs:tc_log(Category, ?STD_IMPORTANCE, + Header, String, [], []); + tc_log_async -> + ct_logs:tc_log_async(sasl, ?STD_IMPORTANCE, + Header, String, []) + end, + {reply,ok,State}; + +handle_call(flush,_From,State) -> + {reply, ok, State}; + +handle_call({set_curr_func,{group,Group,Conf},Config},_From,State) + when is_list(Config) -> Parallel = case proplists:get_value(tc_group_properties, Config) of undefined -> false; Props -> lists:member(parallel, Props) end, - {ok, ok, State#eh_state{curr_group = Group, - curr_func = Conf, - parallel_tcs = Parallel}}; -handle_call({set_curr_func,{group,Group,Conf},_SkipOrFail}, State) -> - {ok, ok, State#eh_state{curr_group = Group, - curr_func = Conf, - parallel_tcs = false}}; -handle_call({set_curr_func,{group,undefined},_Config}, State) -> - {ok, ok, State#eh_state{curr_group = undefined, - curr_func = undefined, - parallel_tcs = false}}; -handle_call({set_curr_func,{Suite,Conf},_Config}, State) -> - {ok, ok, State#eh_state{curr_suite = Suite, - curr_func = Conf, - parallel_tcs = false}}; -handle_call({set_curr_func,undefined,_Config}, State) -> - {ok, ok, State#eh_state{curr_suite = undefined, - curr_func = undefined, - parallel_tcs = false}}; -handle_call({set_curr_func,TC,_Config}, State) -> - {ok, ok, State#eh_state{curr_func = TC}}; - -handle_call({set_logfunc,NewLogFunc}, State) -> - {ok, NewLogFunc, State#eh_state{log_func = NewLogFunc}}; - -handle_call({handle_remote_events,Bool}, State) -> - {ok, ok, State#eh_state{handle_remote_events = Bool}}; - -handle_call(_Query, _State) -> - {error, bad_query}. + {reply, ok, State#eh_state{curr_group = Group, + curr_func = Conf, + parallel_tcs = Parallel}}; +handle_call({set_curr_func,{group,Group,Conf},_SkipOrFail}, _From, State) -> + {reply, ok, State#eh_state{curr_group = Group, + curr_func = Conf, + parallel_tcs = false}}; +handle_call({set_curr_func,{group,undefined},_Config}, _From, State) -> + {reply, ok, State#eh_state{curr_group = undefined, + curr_func = undefined, + parallel_tcs = false}}; +handle_call({set_curr_func,{Suite,Conf},_Config}, _From, State) -> + {reply, ok, State#eh_state{curr_suite = Suite, + curr_func = Conf, + parallel_tcs = false}}; +handle_call({set_curr_func,undefined,_Config}, _From, State) -> + {reply, ok, State#eh_state{curr_suite = undefined, + curr_func = undefined, + parallel_tcs = false}}; +handle_call({set_curr_func,TC,_Config}, _From, State) -> + {reply, ok, State#eh_state{curr_func = TC}}; + +handle_call({set_logfunc,NewLogFunc}, _From, State) -> + {reply, NewLogFunc, State#eh_state{log_func = NewLogFunc}}; + +handle_call({handle_remote_events,Bool}, _From, State) -> + {reply, ok, State#eh_state{handle_remote_events = Bool}}. terminate(_) -> - error_logger:delete_report_handler(?MODULE), - []. + _ = logger:remove_handler(?MODULE), + _ = supervisor:terminate_child(logger_sup,?MODULE), + _ = supervisor:delete_child(logger_sup,?MODULE), + ok. terminate(_Arg, _State) -> ok. -tag_event(Event, utc) -> - {calendar:universal_time(), Event}; -tag_event(Event, _) -> - {calendar:local_time(), Event}. - set_curr_func(CurrFunc, Config) -> - gen_event:call(error_logger, ?MODULE, {set_curr_func, CurrFunc, Config}). + gen_server:call(?MODULE, {set_curr_func, CurrFunc, Config}). set_log_func(Func) -> - gen_event:call(error_logger, ?MODULE, {set_logfunc, Func}). + gen_server:call(?MODULE, {set_logfunc, Func}). handle_remote_events(Bool) -> - gen_event:call(error_logger, ?MODULE, {handle_remote_events, Bool}). + gen_server:call(?MODULE, {handle_remote_events, Bool}). %%%----------------------------------------------------------------- @@ -272,6 +302,3 @@ format_header(#eh_state{curr_suite = Suite, curr_func = TC}) -> io_lib:format("System report during ~w:~tw/1 in ~tw", [Suite,TC,Group]). - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl index 4bbf290081..1ae6c8c7c7 100644 --- a/lib/common_test/src/test_server_ctrl.erl +++ b/lib/common_test/src/test_server_ctrl.erl @@ -2301,7 +2301,7 @@ run_test_cases(TestSpec, Config, TimetrapData) -> %% test_server_io:print_buffered/1 to print the data. To help with this, %% two variables in the process dictionary are used: %% 'test_server_common_io_handler' and 'test_server_queued_io'. The values -%% are set to as follwing: +%% are set to as following: %% %% Value Meaning %% ----- ------- diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl index 3c1e887f65..6228524a88 100644 --- a/lib/common_test/test/ct_hooks_SUITE.erl +++ b/lib/common_test/test/ct_hooks_SUITE.erl @@ -258,15 +258,20 @@ cth_log(Config) when is_list(Config) -> fun(UnexpIoLog) -> {ok,Bin} = file:read_file(UnexpIoLog), Ts = string:lexemes(binary_to_list(Bin),[$\n]), - Matches = lists:foldl(fun([$=,$E,$R,$R,$O,$R|_], N) -> - N+1; - ([$L,$o,$g,$g,$e,$r|_], N) -> - N+1; + Matches = lists:foldl(fun([$=,$E,$R,$R,$O,$R|_], {E,I,L}) -> + {E+1,I,L}; + ([$=,$I,$N,$F,$O|_], {E,I,L}) -> + {E,I+1,L}; + ([$L,$o,$g,$g,$e,$r|_], {E,I,L}) -> + {E,I,L+1}; (_, N) -> N - end, 0, Ts), - ct:pal("~p matches in ~tp", [Matches,UnexpIoLog]), - if Matches > 10 -> ok; - true -> exit({no_unexpected_io_found,UnexpIoLog}) + end, {0,0,0}, Ts), + ct:pal("~p ({Error,Info,Log}) matches in ~tp", + [Matches,UnexpIoLog]), + MatchList = tuple_to_list(Matches), + case [N || N <- MatchList, N<3] of + [] -> ok; + _ -> exit({missing_unexpected_io,UnexpIoLog}) end end, UnexpIoLogs), ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/cth_log_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/cth_log_SUITE.erl index bd1ac54781..a0cd77b88b 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/cth_log_SUITE.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/cth_log_SUITE.erl @@ -124,6 +124,6 @@ gen() -> gen_loop(N) -> ct:log("Logger iteration: ~p", [N]), error_logger:error_report(N), - error_logger:info_report(progress, N), + error_logger:info_report(N), ct:sleep(150), gen_loop(N+1). diff --git a/lib/common_test/test/ct_pre_post_test_io_SUITE.erl b/lib/common_test/test/ct_pre_post_test_io_SUITE.erl index 0b85392009..538fd822c1 100644 --- a/lib/common_test/test/ct_pre_post_test_io_SUITE.erl +++ b/lib/common_test/test/ct_pre_post_test_io_SUITE.erl @@ -241,7 +241,7 @@ try_loop(_Fun, 0) -> gave_up; try_loop(Fun, N) -> try Fun() of - {error,_} -> + {Error,_} when Error==error; Error==badrpc -> timer:sleep(10), try_loop(Fun, N-1); Result -> @@ -257,7 +257,7 @@ try_loop(M, F, _A, 0) -> gave_up; try_loop(M, F, A, N) -> try apply(M, F, A) of - {error,_} -> + {Error,_Reason} when Error==error; Error==badrpc -> timer:sleep(10), try_loop(M, F, A, N-1); Result -> diff --git a/lib/compiler/doc/src/compile.xml b/lib/compiler/doc/src/compile.xml index 06afc55c07..e4491288a6 100644 --- a/lib/compiler/doc/src/compile.xml +++ b/lib/compiler/doc/src/compile.xml @@ -649,14 +649,6 @@ module.beam: module.erl \ <p>Turns off warnings for unused record types. Default is to emit warnings for unused locally defined record types.</p> </item> - - <tag><c>nowarn_get_stacktrace</c></tag> - <item> - <p>Turns off warnings for using <c>get_stacktrace/0</c> in a context - where it will probably not work in a future release. For example, - by default there will be a warning if <c>get_stacktrace/0</c> is - used following a <c>catch</c> expression.</p> - </item> </taglist> <p>Another class of warnings is generated by the compiler diff --git a/lib/compiler/src/Makefile b/lib/compiler/src/Makefile index 9e96147787..c81b81e82b 100644 --- a/lib/compiler/src/Makefile +++ b/lib/compiler/src/Makefile @@ -186,7 +186,6 @@ release_docs_spec: $(EBIN)/beam_disasm.beam: $(EGEN)/beam_opcodes.hrl beam_disasm.hrl $(EBIN)/beam_listing.beam: core_parse.hrl v3_kernel.hrl -$(EBIN)/beam_validator.beam: beam_disasm.hrl $(EBIN)/cerl.beam: core_parse.hrl $(EBIN)/compile.beam: core_parse.hrl ../../stdlib/include/erl_compile.hrl $(EBIN)/core_lib.beam: core_parse.hrl diff --git a/lib/compiler/src/beam_clean.erl b/lib/compiler/src/beam_clean.erl index 7ddf9fa2e2..955c128699 100644 --- a/lib/compiler/src/beam_clean.erl +++ b/lib/compiler/src/beam_clean.erl @@ -254,7 +254,7 @@ bs_restores([_|Is], Dict) -> bs_restores([], Dict) -> Dict. %% Pass 2. -bs_replace([{test,bs_start_match2,F,Live,[Src,Ctx],CtxR}|T], Dict, Acc) when is_atom(Ctx) -> +bs_replace([{test,bs_start_match2,F,Live,[Src,{context,Ctx}],CtxR}|T], Dict, Acc) -> Slots = case gb_trees:lookup(Ctx, Dict) of {value,Slots0} -> Slots0; none -> 0 diff --git a/lib/compiler/src/beam_utils.erl b/lib/compiler/src/beam_utils.erl index 814cfb8265..1ddad30328 100644 --- a/lib/compiler/src/beam_utils.erl +++ b/lib/compiler/src/beam_utils.erl @@ -655,9 +655,8 @@ check_liveness_at(R, Lbl, #live{lbl=Ll,res=ResMemorized}=St0) -> {Res,St#live{res=gb_trees:insert(Lbl, Res, St#live.res)}} end. -not_used({exit_not_used,St}) -> {not_used,St}; -not_used({killed,St}) -> {not_used,St}; -not_used({_,_}=Res) -> Res. +not_used({used,_}=Res) -> Res; +not_used({_,St}) -> {not_used,St}. check_liveness_ret(R, R, St) -> {used,St}; check_liveness_ret(_, _, St) -> {killed,St}. @@ -801,6 +800,10 @@ replace_labels_1([{wait,{f,Lbl}}|Is], Acc, D, Fb) -> replace_labels_1(Is, [{wait,{f,label(Lbl, D, Fb)}}|Acc], D, Fb); replace_labels_1([{wait_timeout,{f,Lbl},To}|Is], Acc, D, Fb) -> replace_labels_1(Is, [{wait_timeout,{f,label(Lbl, D, Fb)},To}|Acc], D, Fb); +replace_labels_1([{recv_mark=Op,{f,Lbl}}|Is], Acc, D, Fb) -> + replace_labels_1(Is, [{Op,{f,label(Lbl, D, Fb)}}|Acc], D, Fb); +replace_labels_1([{recv_set=Op,{f,Lbl}}|Is], Acc, D, Fb) -> + replace_labels_1(Is, [{Op,{f,label(Lbl, D, Fb)}}|Acc], D, Fb); replace_labels_1([{bif,Name,{f,Lbl},As,R}|Is], Acc, D, Fb) when Lbl =/= 0 -> replace_labels_1(Is, [{bif,Name,{f,label(Lbl, D, Fb)},As,R}|Acc], D, Fb); replace_labels_1([{gc_bif,Name,{f,Lbl},Live,As,R}|Is], Acc, D, Fb) when Lbl =/= 0 -> diff --git a/lib/compiler/src/beam_validator.erl b/lib/compiler/src/beam_validator.erl index c30ab34ac7..962f17d83c 100644 --- a/lib/compiler/src/beam_validator.erl +++ b/lib/compiler/src/beam_validator.erl @@ -27,9 +27,7 @@ %% Interface for compiler. -export([module/2, format_error/1]). --include("beam_disasm.hrl"). - --import(lists, [reverse/1,foldl/3,foreach/2,dropwhile/2]). +-import(lists, [any/2,dropwhile/2,foldl/3,foreach/2,reverse/1]). %% To be called by the compiler. @@ -365,7 +363,9 @@ valfun_1({recv_set,{f,Fail}}, Vst) when is_integer(Fail) -> Vst; %% Misc. valfun_1(remove_message, Vst) -> - Vst; + %% The message term is no longer fragile. It can be used + %% without restrictions. + remove_fragility(Vst); valfun_1({'%',_}, Vst) -> Vst; valfun_1({line,_}, Vst) -> @@ -533,7 +533,7 @@ valfun_4({bif,element,{f,Fail},[Pos,Tuple],Dst}, Vst0) -> Vst1 = branch_state(Fail, Vst0), TupleType = upgrade_tuple_type({tuple,[get_tuple_size(PosType)]}, TupleType0), Vst = set_type(TupleType, Tuple, Vst1), - set_type_reg(term, Dst, Vst); + set_type_reg(term, Tuple, Dst, Vst); valfun_4({bif,raise,{f,0},Src,_Dst}, Vst) -> validate_src(Src, Vst), kill_state(Vst); @@ -542,7 +542,8 @@ valfun_4(raw_raise=I, Vst) -> valfun_4({bif,Op,{f,Fail},Src,Dst}, Vst0) -> validate_src(Src, Vst0), Vst = branch_state(Fail, Vst0), - Type = bif_type(Op, Src, Vst), + Type0 = bif_type(Op, Src, Vst), + Type = propagate_fragility(Type0, Src, Vst), set_type_reg(Type, Dst, Vst); valfun_4({gc_bif,Op,{f,Fail},Live,Src,Dst}, #vst{current=St0}=Vst0) -> verify_live(Live, Vst0), @@ -552,7 +553,8 @@ valfun_4({gc_bif,Op,{f,Fail},Live,Src,Dst}, #vst{current=St0}=Vst0) -> Vst2 = branch_state(Fail, Vst1), Vst = prune_x_regs(Live, Vst2), validate_src(Src, Vst), - Type = bif_type(Op, Src, Vst), + Type0 = bif_type(Op, Src, Vst), + Type = propagate_fragility(Type0, Src, Vst), set_type_reg(Type, Dst, Vst); valfun_4(return, #vst{current=#st{numy=none}}=Vst) -> assert_term({x,0}, Vst), @@ -563,13 +565,20 @@ valfun_4({jump,{f,Lbl}}, Vst) -> kill_state(branch_state(Lbl, Vst)); valfun_4({loop_rec,{f,Fail},Dst}, Vst0) -> Vst = branch_state(Fail, Vst0), - set_type_reg(term, Dst, Vst); + %% This term may not be part of the root set until + %% remove_message/0 is executed. If control transfers + %% to the loop_rec_end/1 instruction, no part of + %% this term must be stored in a Y register. + set_type_reg({fragile,term}, Dst, Vst); valfun_4({wait,_}, Vst) -> + verify_y_init(Vst), kill_state(Vst); valfun_4({wait_timeout,_,Src}, Vst) -> assert_term(Src, Vst), - Vst; + verify_y_init(Vst), + prune_x_regs(0, Vst); valfun_4({loop_rec_end,_}, Vst) -> + verify_y_init(Vst), kill_state(Vst); valfun_4(timeout, #vst{current=St}=Vst) -> Vst#vst{current=St#st{x=init_regs(0, term)}}; @@ -589,17 +598,17 @@ valfun_4({select_tuple_arity,Tuple,{f,Fail},{list,Choices}}, Vst) -> kill_state(branch_arities(Choices, Tuple, branch_state(Fail, Vst))); valfun_4({get_list,Src,D1,D2}, Vst0) -> assert_type(cons, Src, Vst0), - Vst = set_type_reg(term, D1, Vst0), - set_type_reg(term, D2, Vst); + Vst = set_type_reg(term, Src, D1, Vst0), + set_type_reg(term, Src, D2, Vst); valfun_4({get_hd,Src,Dst}, Vst) -> assert_type(cons, Src, Vst), - set_type_reg(term, Dst, Vst); + set_type_reg(term, Src, Dst, Vst); valfun_4({get_tl,Src,Dst}, Vst) -> assert_type(cons, Src, Vst), - set_type_reg(term, Dst, Vst); + set_type_reg(term, Src, Dst, Vst); valfun_4({get_tuple_element,Src,I,Dst}, Vst) -> assert_type({tuple_element,I+1}, Src, Vst), - set_type_reg(term, Dst, Vst); + set_type_reg(term, Src, Dst, Vst); %% New bit syntax matching instructions. valfun_4({test,bs_start_match2,{f,Fail},Live,[Ctx,NeedSlots],Ctx}, Vst0) -> @@ -607,6 +616,7 @@ valfun_4({test,bs_start_match2,{f,Fail},Live,[Ctx,NeedSlots],Ctx}, Vst0) -> %% is OK as input. CtxType = get_move_term_type(Ctx, Vst0), verify_live(Live, Vst0), + verify_y_init(Vst0), Vst1 = prune_x_regs(Live, Vst0), BranchVst = case CtxType of #ms{} -> @@ -623,9 +633,10 @@ valfun_4({test,bs_start_match2,{f,Fail},Live,[Ctx,NeedSlots],Ctx}, Vst0) -> valfun_4({test,bs_start_match2,{f,Fail},Live,[Src,Slots],Dst}, Vst0) -> assert_term(Src, Vst0), verify_live(Live, Vst0), + verify_y_init(Vst0), Vst1 = prune_x_regs(Live, Vst0), Vst = branch_state(Fail, Vst1), - set_type_reg(bsm_match_state(Slots), Dst, Vst); + set_type_reg(bsm_match_state(Slots), Src, Dst, Vst); valfun_4({test,bs_match_string,{f,Fail},[Ctx,_,_]}, Vst) -> bsm_validate_context(Ctx, Vst), branch_state(Fail, Vst); @@ -650,7 +661,8 @@ valfun_4({test,bs_get_integer2,{f,Fail},Live,[Ctx,_,_,_],Dst}, Vst) -> valfun_4({test,bs_get_float2,{f,Fail},Live,[Ctx,_,_,_],Dst}, Vst) -> validate_bs_get(Fail, Ctx, Live, {float, []}, Dst, Vst); valfun_4({test,bs_get_binary2,{f,Fail},Live,[Ctx,_,_,_],Dst}, Vst) -> - validate_bs_get(Fail, Ctx, Live, term, Dst, Vst); + Type = propagate_fragility(term, [Ctx], Vst), + validate_bs_get(Fail, Ctx, Live, Type, Dst, Vst); valfun_4({test,bs_get_utf8,{f,Fail},Live,[Ctx,_],Dst}, Vst) -> validate_bs_get(Fail, Ctx, Live, {integer, []}, Dst, Vst); valfun_4({test,bs_get_utf16,{f,Fail},Live,[Ctx,_],Dst}, Vst) -> @@ -790,7 +802,7 @@ verify_get_map(Fail, Src, List, Vst0) -> Vst2 = branch_state(Fail, Vst1), Keys = extract_map_keys(List), assert_unique_map_keys(Keys), - verify_get_map_pair(List,Vst0,Vst2). + verify_get_map_pair(List, Src, Vst0, Vst2). extract_map_vals([_Key,Val|T]) -> [Val|extract_map_vals(T)]; @@ -800,10 +812,11 @@ extract_map_keys([Key,_Val|T]) -> [Key|extract_map_keys(T)]; extract_map_keys([]) -> []. -verify_get_map_pair([],_,Vst) -> Vst; -verify_get_map_pair([Src,Dst|Vs],Vst0,Vsti) -> +verify_get_map_pair([Src,Dst|Vs], Map, Vst0, Vsti0) -> assert_term(Src, Vst0), - verify_get_map_pair(Vs,Vst0,set_type_reg(term,Dst,Vsti)). + Vsti = set_type_reg(term, Map, Dst, Vsti0), + verify_get_map_pair(Vs, Map, Vst0, Vsti); +verify_get_map_pair([], _Map, _Vst0, Vst) -> Vst. verify_put_map(Fail, Src, Dst, Live, List, Vst0) -> assert_type(map, Src, Vst0), @@ -823,6 +836,7 @@ verify_put_map(Fail, Src, Dst, Live, List, Vst0) -> validate_bs_get(Fail, Ctx, Live, Type, Dst, Vst0) -> bsm_validate_context(Ctx, Vst0), verify_live(Live, Vst0), + verify_y_init(Vst0), Vst1 = prune_x_regs(Live, Vst0), Vst = branch_state(Fail, Vst1), set_type_reg(Type, Dst, Vst). @@ -832,6 +846,7 @@ validate_bs_get(Fail, Ctx, Live, Type, Dst, Vst0) -> %% validate_bs_skip_utf(Fail, Ctx, Live, Vst0) -> bsm_validate_context(Ctx, Vst0), + verify_y_init(Vst0), verify_live(Live, Vst0), Vst = prune_x_regs(Live, Vst0), branch_state(Fail, Vst). @@ -1093,10 +1108,11 @@ bsm_validate_context(Reg, Vst) -> bsm_get_context({x,X}=Reg, #vst{current=#st{x=Xs}}=_Vst) when is_integer(X) -> case gb_trees:lookup(X, Xs) of {value,#ms{}=Ctx} -> Ctx; + {value,{fragile,#ms{}=Ctx}} -> Ctx; _ -> error({no_bsm_context,Reg}) end; bsm_get_context(Reg, _) -> error({bad_source,Reg}). - + bsm_save(Reg, {atom,start}, Vst) -> %% Save point refering to where the match started. %% It is always valid. But don't forget to validate the context register. @@ -1133,13 +1149,34 @@ set_type(Type, {x,_}=Reg, Vst) -> set_type_reg(Type, Reg, Vst); set_type(Type, {y,_}=Reg, Vst) -> set_type_y(Type, Reg, Vst); set_type(_, _, #vst{}=Vst) -> Vst. -set_type_reg(Type, {x,X}=Reg, #vst{current=#st{x=Xs}=St}=Vst) - when is_integer(X), 0 =< X -> - check_limit(Reg), - Vst#vst{current=St#st{x=gb_trees:enter(X, Type, Xs)}}; +set_type_reg(Type, Src, Dst, Vst) -> + case get_term_type_1(Src, Vst) of + {fragile,_} -> + set_type_reg(make_fragile(Type), Dst, Vst); + _ -> + set_type_reg(Type, Dst, Vst) + end. + +set_type_reg(Type, {x,_}=Reg, Vst) -> + set_type_x(Type, Reg, Vst); set_type_reg(Type, Reg, Vst) -> set_type_y(Type, Reg, Vst). +set_type_x(Type, {x,X}=Reg, #vst{current=#st{x=Xs0}=St}=Vst) + when is_integer(X), 0 =< X -> + check_limit(Reg), + Xs = case gb_trees:lookup(X, Xs0) of + none -> + gb_trees:insert(X, Type, Xs0); + {value,{fragile,_}} -> + gb_trees:update(X, make_fragile(Type), Xs0); + {value,_} -> + gb_trees:update(X, Type, Xs0) + end, + Vst#vst{current=St#st{x=Xs}}; +set_type_x(Type, Reg, #vst{}) -> + error({invalid_store,Reg,Type}). + set_type_y(Type, {y,Y}=Reg, #vst{current=#st{y=Ys0}=St}=Vst) when is_integer(Y), 0 =< Y -> check_limit(Reg), @@ -1157,6 +1194,9 @@ set_type_y(Type, {y,Y}=Reg, #vst{current=#st{y=Ys0}=St}=Vst) Vst#vst{current=St#st{y=Ys}}; set_type_y(Type, Reg, #vst{}) -> error({invalid_store,Reg,Type}). +make_fragile({fragile,_}=Type) -> Type; +make_fragile(Type) -> {fragile,Type}. + set_catch_end({y,Y}, #vst{current=#st{y=Ys0}=St}=Vst) -> Ys = gb_trees:update(Y, initialized, Ys0), Vst#vst{current=St#st{y=Ys}}. @@ -1257,9 +1297,26 @@ assert_term(Src, Vst) -> %% %% map Map. %% +%% +%% +%% FRAGILITY +%% --------- +%% +%% The loop_rec/2 instruction may return a reference to a term that is +%% not part of the root set. That term or any part of it must not be +%% included in a garbage collection. Therefore, the term (or any part +%% of it) must not be stored in an Y register. +%% +%% Such terms are wrapped in a {fragile,Type} tuple, where Type is one +%% of the types described above. assert_type(WantedType, Term, Vst) -> - assert_type(WantedType, get_term_type(Term, Vst)). + case get_term_type(Term, Vst) of + {fragile,Type} -> + assert_type(WantedType, Type); + Type -> + assert_type(WantedType, Type) + end. assert_type(Correct, Correct) -> ok; assert_type(float, {float,_}) -> ok; @@ -1285,14 +1342,19 @@ assert_type(Needed, Actual) -> %% is inconsistent, and we know that some instructions will never %% be executed at run-time. -upgrade_tuple_type({tuple,[Sz]}, {tuple,[OldSz]}=T) when Sz < OldSz -> +upgrade_tuple_type(NewType, {fragile,OldType}) -> + make_fragile(upgrade_tuple_type_1(NewType, OldType)); +upgrade_tuple_type(NewType, OldType) -> + upgrade_tuple_type_1(NewType, OldType). + +upgrade_tuple_type_1({tuple,[Sz]}, {tuple,[OldSz]}=T) when Sz < OldSz -> %% The old type has a higher value for the least tuple size. T; -upgrade_tuple_type({tuple,[Sz]}, {tuple,OldSz}=T) +upgrade_tuple_type_1({tuple,[Sz]}, {tuple,OldSz}=T) when is_integer(Sz), is_integer(OldSz), Sz =< OldSz -> %% The old size is exact, and the new size is smaller than the old size. T; -upgrade_tuple_type({tuple,_}=T, _) -> +upgrade_tuple_type_1({tuple,_}=T, _) -> %% The new type information is exact or has a higher value for %% the least tuple size. %% Note that inconsistencies are also handled in this @@ -1459,6 +1521,14 @@ merge_y_regs_1(_, _, Regs) -> Regs. %% merge_types(Type1, Type2) -> Type %% Return the most specific type possible. %% Note: Type1 must NOT be the same as Type2. +merge_types({fragile,Same}=Type, Same) -> + Type; +merge_types({fragile,T1}, T2) -> + make_fragile(merge_types(T1, T2)); +merge_types(Same, {fragile,Same}=Type) -> + Type; +merge_types(T1, {fragile,T2}) -> + make_fragile(merge_types(T1, T2)); merge_types(uninitialized=I, _) -> I; merge_types(_, uninitialized=I) -> I; merge_types(initialized=I, _) -> I; @@ -1509,6 +1579,10 @@ verify_y_init(#vst{current=#st{y=Ys}}) -> verify_y_init_1([]) -> ok; verify_y_init_1([{Y,uninitialized}|_]) -> error({uninitialized_reg,{y,Y}}); +verify_y_init_1([{Y,{fragile,_}}|_]) -> + %% Unsafe. This term may be outside any heap belonging + %% to the process and would be corrupted by a GC. + error({fragile_message_reference,{y,Y}}); verify_y_init_1([{_,_}|Ys]) -> verify_y_init_1(Ys). @@ -1554,6 +1628,27 @@ eat_heap_float(#vst{current=#st{hf=HeapFloats0}=St}=Vst) -> Vst#vst{current=St#st{hf=HeapFloats}} end. +remove_fragility(#vst{current=#st{x=Xs0,y=Ys0}=St0}=Vst) -> + F = fun(_, {fragile,Type}) -> Type; + (_, Type) -> Type + end, + Xs = gb_trees:map(F, Xs0), + Ys = gb_trees:map(F, Ys0), + St = St0#st{x=Xs,y=Ys}, + Vst#vst{current=St}. + +propagate_fragility(Type, Ss, Vst) -> + F = fun(S) -> + case get_term_type_1(S, Vst) of + {fragile,_} -> true; + _ -> false + end + end, + case any(F, Ss) of + true -> make_fragile(Type); + false -> Type + end. + bif_type('-', Src, Vst) -> arith_type(Src, Vst); bif_type('+', Src, Vst) -> diff --git a/lib/compiler/src/cerl.erl b/lib/compiler/src/cerl.erl index 6b936a7687..fce23bfd68 100644 --- a/lib/compiler/src/cerl.erl +++ b/lib/compiler/src/cerl.erl @@ -433,6 +433,8 @@ is_literal_term(T) when is_tuple(T) -> is_literal_term(B) when is_bitstring(B) -> true; is_literal_term(M) when is_map(M) -> is_literal_term_list(maps:to_list(M)); +is_literal_term(F) when is_function(F) -> + erlang:fun_info(F, type) =:= {type,external}; is_literal_term(_) -> false. diff --git a/lib/compiler/src/cerl_inline.erl b/lib/compiler/src/cerl_inline.erl index f5afa75b16..caff47dbcb 100644 --- a/lib/compiler/src/cerl_inline.erl +++ b/lib/compiler/src/cerl_inline.erl @@ -1822,6 +1822,14 @@ new_var(Env) -> Name = env__new_vname(Env), c_var(Name). +%% The way a template variable is used makes it necessary +%% to make sure that it is unique in the entire function. +%% Therefore, template variables are atoms with the prefix "@i". + +new_template_var(Env) -> + Name = env__new_tname(Env), + c_var(Name). + residualize_var(R, S) -> S1 = count_size(weight(var), S), {ref_to_var(R), st__set_var_referenced(R#ref.loc, S1)}. @@ -2183,7 +2191,7 @@ make_template(E, Vs0, Env0) -> T = make_data_skel(data_type(E), Ts), E1 = update_data(E, data_type(E), [hd(get_ann(T)) || T <- Ts]), - V = new_var(Env1), + V = new_template_var(Env1), Env2 = env__bind(var_name(V), E1, Env1), {set_ann(T, [V]), [V | Vs1], Env2}; false -> @@ -2198,7 +2206,7 @@ make_template(E, Vs0, Env0) -> Env2 = env__bind(V, E1, Env1), {T, Vs1, Env2}; _ -> - V = new_var(Env0), + V = new_template_var(Env0), Env1 = env__bind(var_name(V), E, Env0), {set_ann(V, [V]), [V | Vs0], Env1} end @@ -2564,6 +2572,11 @@ env__is_defined(Key, Env) -> env__new_vname(Env) -> rec_env:new_key(Env). +env__new_tname(Env) -> + rec_env:new_key(fun(I) -> + list_to_atom("@i"++integer_to_list(I)) + end, Env). + env__new_fname(A, N, Env) -> rec_env:new_key(fun (X) -> S = integer_to_list(X), diff --git a/lib/compiler/src/cerl_trees.erl b/lib/compiler/src/cerl_trees.erl index f30a0b33ac..c7a129b42c 100644 --- a/lib/compiler/src/cerl_trees.erl +++ b/lib/compiler/src/cerl_trees.erl @@ -22,7 +22,8 @@ -module(cerl_trees). -export([depth/1, fold/3, free_variables/1, get_label/1, label/1, label/2, - map/2, mapfold/3, mapfold/4, size/1, variables/1]). + map/2, mapfold/3, mapfold/4, next_free_variable_name/1, + size/1, variables/1]). -import(cerl, [alias_pat/1, alias_var/1, ann_c_alias/3, ann_c_apply/3, ann_c_binary/2, ann_c_bitstr/6, ann_c_call/4, @@ -507,6 +508,7 @@ mapfold_pairs(_, _, S, []) -> %% well-formed Core Erlang syntax tree. %% %% @see free_variables/1 +%% @see next_free_variable_name/1 -spec variables(cerl:cerl()) -> [cerl:var_name()]. @@ -519,6 +521,7 @@ variables(T) -> %% @doc Like <code>variables/1</code>, but only includes variables %% that are free in the tree. %% +%% @see next_free_variable_name/1 %% @see variables/1 -spec free_variables(cerl:cerl()) -> [cerl:var_name()]. @@ -678,6 +681,110 @@ var_list_names([V | Vs], A) -> var_list_names([], A) -> A. +%% --------------------------------------------------------------------- + +%% @spec next_free_variable_name(Tree::cerl()) -> var_name() +%% +%% var_name() = integer() +%% +%% @doc Returns a integer variable name higher than any other integer +%% variable name in the syntax tree. An exception is thrown if +%% <code>Tree</code> does not represent a well-formed Core Erlang +%% syntax tree. +%% +%% @see variables/1 +%% @see free_variables/1 + +-spec next_free_variable_name(cerl:cerl()) -> integer(). + +next_free_variable_name(T) -> + 1 + next_free(T, -1). + +next_free(T, Max) -> + case type(T) of + literal -> + Max; + var -> + case var_name(T) of + Int when is_integer(Int) -> + max(Int, Max); + _ -> + Max + end; + values -> + next_free_in_list(values_es(T), Max); + cons -> + next_free(cons_hd(T), next_free(cons_tl(T), Max)); + tuple -> + next_free_in_list(tuple_es(T), Max); + map -> + next_free_in_list([map_arg(T)|map_es(T)], Max); + map_pair -> + next_free_in_list([map_pair_op(T),map_pair_key(T), + map_pair_val(T)], Max); + 'let' -> + Max1 = next_free(let_body(T), Max), + Max2 = next_free_in_list(let_vars(T), Max1), + next_free(let_arg(T), Max2); + seq -> + next_free(seq_arg(T), + next_free(seq_body(T), Max)); + apply -> + next_free(apply_op(T), + next_free_in_list(apply_args(T), Max)); + call -> + next_free(call_module(T), + next_free(call_name(T), + next_free_in_list( + call_args(T), Max))); + primop -> + next_free_in_list(primop_args(T), Max); + 'case' -> + next_free(case_arg(T), + next_free_in_list(case_clauses(T), Max)); + clause -> + Max1 = next_free(clause_guard(T), + next_free(clause_body(T), Max)), + next_free_in_list(clause_pats(T), Max1); + alias -> + next_free(alias_var(T), + next_free(alias_pat(T), Max)); + 'fun' -> + next_free(fun_body(T), + next_free_in_list(fun_vars(T), Max)); + 'receive' -> + Max1 = next_free_in_list(receive_clauses(T), + next_free(receive_timeout(T), Max)), + next_free(receive_action(T), Max1); + 'try' -> + Max1 = next_free(try_body(T), Max), + Max2 = next_free_in_list(try_vars(T), Max1), + Max3 = next_free(try_handler(T), Max2), + Max4 = next_free_in_list(try_evars(T), Max3), + next_free(try_arg(T), Max4); + 'catch' -> + next_free(catch_body(T), Max); + binary -> + next_free_in_list(binary_segments(T), Max); + bitstr -> + next_free(bitstr_val(T), next_free(bitstr_size(T), Max)); + letrec -> + Max1 = next_free_in_defs(letrec_defs(T), Max), + Max2 = next_free(letrec_body(T), Max1), + next_free_in_list(letrec_vars(T), Max2); + module -> + next_free_in_defs(module_defs(T), Max) + end. + +next_free_in_list([H | T], Max) -> + next_free_in_list(T, next_free(H, Max)); +next_free_in_list([], Max) -> + Max. + +next_free_in_defs([{_, Post} | Ds], Max) -> + next_free_in_defs(Ds, next_free(Post, Max)); +next_free_in_defs([], Max) -> + Max. %% --------------------------------------------------------------------- diff --git a/lib/compiler/src/core_lint.erl b/lib/compiler/src/core_lint.erl index 6e2114be56..6ded8fe78f 100644 --- a/lib/compiler/src/core_lint.erl +++ b/lib/compiler/src/core_lint.erl @@ -491,8 +491,10 @@ pattern(#c_tuple{es=Es}, Def, Ps, St) -> pattern_list(Es, Def, Ps, St); pattern(#c_map{es=Es}, Def, Ps, St) -> pattern_list(Es, Def, Ps, St); -pattern(#c_map_pair{op=#c_literal{val=exact},key=K,val=V},Def,Ps,St) -> - pattern_list([K,V],Def,Ps,St); +pattern(#c_map_pair{op=#c_literal{val=exact},key=K,val=V}, Def, Ps, St) -> + %% The key is an input. + pat_map_expr(K, Def, St), + pattern_list([V],Def,Ps,St); pattern(#c_binary{segments=Ss}, Def, Ps, St0) -> St = pat_bin_tail_check(Ss, St0), pat_bin(Ss, Def, Ps, St); @@ -555,6 +557,10 @@ pat_bit_expr(#c_binary{}, _, _Def, St) -> pat_bit_expr(_, _, _, St) -> add_error({illegal_expr,St#lint.func}, St). +pat_map_expr(#c_var{name=N}, Def, St) -> expr_var(N, Def, St); +pat_map_expr(#c_literal{}, _Def, St) -> St; +pat_map_expr(_, _, St) -> add_error({illegal_expr,St#lint.func}, St). + %% pattern_list([Var], Defined, State) -> {[PatVar],State}. %% pattern_list([Var], Defined, [PatVar], State) -> {[PatVar],State}. diff --git a/lib/compiler/src/core_parse.yrl b/lib/compiler/src/core_parse.yrl index 79a7cccd98..11c4cd8b50 100644 --- a/lib/compiler/src/core_parse.yrl +++ b/lib/compiler/src/core_parse.yrl @@ -36,7 +36,7 @@ other_pattern atomic_pattern tuple_pattern cons_pattern tail_pattern binary_pattern segment_patterns segment_pattern expression single_expression -literal literals atomic_literal tuple_literal cons_literal tail_literal +literal literals atomic_literal tuple_literal cons_literal tail_literal fun_literal nil tuple cons tail binary segments segment @@ -267,6 +267,7 @@ single_expression -> cons : '$1'. single_expression -> binary : '$1'. single_expression -> variable : '$1'. single_expression -> function_name : '$1'. +single_expression -> fun_literal : '$1'. single_expression -> fun_expr : '$1'. single_expression -> let_expr : '$1'. single_expression -> letrec_expr : '$1'. @@ -303,6 +304,9 @@ tail_literal -> ']' : #c_literal{val=[]}. tail_literal -> '|' literal ']' : '$2'. tail_literal -> ',' literal tail_literal : c_cons('$2', '$3'). +fun_literal -> 'fun' atom ':' atom '/' integer : + #c_literal{val = erlang:make_fun(tok_val('$2'), tok_val('$4'), tok_val('$6'))}. + tuple -> '{' '}' : c_tuple([]). tuple -> '{' anno_expressions '}' : c_tuple('$2'). @@ -496,7 +500,7 @@ make_lit_bin(Acc, [#c_bitstr{val=I0,size=Sz0,unit=U0,type=Type0,flags=F0}|T]) -> throw(impossible) end, if - Sz =< 8, T =:= [] -> + 0 =< Sz, Sz =< 8, T =:= [] -> <<Acc/binary,I:Sz>>; Sz =:= 8 -> make_lit_bin(<<Acc/binary,I:8>>, T); diff --git a/lib/compiler/src/core_pp.erl b/lib/compiler/src/core_pp.erl index 2516a9a1e1..f247722b4c 100644 --- a/lib/compiler/src/core_pp.erl +++ b/lib/compiler/src/core_pp.erl @@ -136,6 +136,11 @@ format_1(#c_literal{anno=A,val=M},Ctxt) when is_map(M) -> key=#c_literal{val=K}, val=#c_literal{val=V}} || {K,V} <- Pairs], format_1(#c_map{anno=A,arg=#c_literal{val=#{}},es=Cpairs},Ctxt); +format_1(#c_literal{val=F},_Ctxt) when is_function(F) -> + {module,M} = erlang:fun_info(F, module), + {name,N} = erlang:fun_info(F, name), + {arity,A} = erlang:fun_info(F, arity), + ["fun ",core_atom(M),$:,core_atom(N),$/,integer_to_list(A)]; format_1(#c_var{name={I,A}}, _) -> [core_atom(I),$/,integer_to_list(A)]; format_1(#c_var{name=V}, _) -> @@ -541,4 +546,3 @@ segs_from_bitstring(Bitstring) -> unit=#c_literal{val=1}, type=#c_literal{val=integer}, flags=#c_literal{val=[unsigned,big]}}]. - diff --git a/lib/compiler/src/erl_bifs.erl b/lib/compiler/src/erl_bifs.erl index bafa9d75b7..70b36f029e 100644 --- a/lib/compiler/src/erl_bifs.erl +++ b/lib/compiler/src/erl_bifs.erl @@ -109,6 +109,8 @@ is_pure(erlang, list_to_integer, 1) -> true; is_pure(erlang, list_to_pid, 1) -> true; is_pure(erlang, list_to_tuple, 1) -> true; is_pure(erlang, max, 2) -> true; +is_pure(erlang, make_fun, 3) -> true; +is_pure(erlang, map_get, 2) -> true; is_pure(erlang, min, 2) -> true; is_pure(erlang, phash, 2) -> false; is_pure(erlang, pid_to_list, 1) -> true; @@ -196,6 +198,7 @@ is_safe(erlang, is_port, 1) -> true; is_safe(erlang, is_reference, 1) -> true; is_safe(erlang, is_tuple, 1) -> true; is_safe(erlang, make_ref, 0) -> true; +is_safe(erlang, make_fun, 3) -> true; is_safe(erlang, max, 2) -> true; is_safe(erlang, min, 2) -> true; is_safe(erlang, node, 0) -> true; diff --git a/lib/compiler/src/sys_core_fold.erl b/lib/compiler/src/sys_core_fold.erl index a9bd363ee1..a13bdedaf9 100644 --- a/lib/compiler/src/sys_core_fold.erl +++ b/lib/compiler/src/sys_core_fold.erl @@ -108,17 +108,29 @@ module(#c_module{defs=Ds0}=Mod, Opts) -> put(no_inline_list_funcs, not member(inline_list_funcs, Opts)), - case get(new_var_num) of - undefined -> put(new_var_num, 0); - _ -> ok - end, init_warnings(), Ds1 = [function_1(D) || D <- Ds0], + erase(new_var_num), erase(no_inline_list_funcs), {ok,Mod#c_module{defs=Ds1},get_warnings()}. function_1({#c_var{name={F,Arity}}=Name,B0}) -> + %% Find a suitable starting value for the variable counter. Note + %% that this pass assumes that new_var_name/1 returns a variable + %% name distinct from any variable used in the entire body of + %% the function. We use integers as variable names to avoid + %% filling up the atom table when compiling huge functions. + Count = cerl_trees:next_free_variable_name(B0), + put(new_var_num, Count), try + %% Find a suitable starting value for the variable + %% counter. Note that this pass assumes that new_var_name/1 + %% returns a variable name distinct from any variable used in + %% the entire body of the function. We use integers as + %% variable names to avoid filling up the atom table when + %% compiling huge functions. + Count = cerl_trees:next_free_variable_name(B0), + put(new_var_num, Count), B = find_fixpoint(fun(Core) -> %% This must be a fun! expr(Core, value, sub_new()) @@ -202,6 +214,8 @@ opt_guard_try(#c_case{clauses=Cs}=Term) -> Term#c_case{clauses=opt_guard_try_list(Cs)}; opt_guard_try(#c_clause{body=B0}=Term) -> Term#c_clause{body=opt_guard_try(B0)}; +opt_guard_try(#c_let{vars=[],arg=#c_values{es=[]},body=B}) -> + B; opt_guard_try(#c_let{arg=Arg,body=B0}=Term) -> case opt_guard_try(B0) of #c_literal{}=B -> @@ -389,14 +403,15 @@ expr(#c_receive{clauses=Cs0,timeout=T0,action=A0}=Recv, Ctxt, Sub) -> T1 = expr(T0, value, Sub), A1 = body(A0, Ctxt, Sub), Recv#c_receive{clauses=Cs1,timeout=T1,action=A1}; -expr(#c_apply{anno=Anno,op=Op0,args=As0}=App, _, Sub) -> +expr(#c_apply{anno=Anno,op=Op0,args=As0}=Apply0, _, Sub) -> Op1 = expr(Op0, value, Sub), As1 = expr_list(As0, value, Sub), - case cerl:is_data(Op1) of + case cerl:is_data(Op1) andalso not is_literal_fun(Op1) of false -> - App#c_apply{op=Op1,args=As1}; + Apply = Apply0#c_apply{op=Op1,args=As1}, + fold_apply(Apply, Op1, As1); true -> - add_warning(App, invalid_call), + add_warning(Apply0, invalid_call), Err = #c_call{anno=Anno, module=#c_literal{val=erlang}, name=#c_literal{val=error}, @@ -487,6 +502,9 @@ bitstr_list(Es, Sub) -> bitstr(#c_bitstr{val=Val,size=Size}=BinSeg, Sub) -> BinSeg#c_bitstr{val=expr(Val, Sub),size=expr(Size, value, Sub)}. +is_literal_fun(#c_literal{val=F}) -> is_function(F); +is_literal_fun(_) -> false. + %% is_safe_simple(Expr, Sub) -> true | false. %% A safe simple cannot fail with badarg and is safe to use %% in a guard. @@ -751,6 +769,25 @@ make_effect_seq([H|T], Sub) -> end; make_effect_seq([], _) -> void(). +%% fold_apply(Apply, LiteraFun, Args) -> Apply. +%% Replace an apply of a literal external fun with a call. + +fold_apply(Apply, #c_literal{val=Fun}, Args) when is_function(Fun) -> + {module,Mod} = erlang:fun_info(Fun, module), + {name,Name} = erlang:fun_info(Fun, name), + {arity,Arity} = erlang:fun_info(Fun, arity), + if + Arity =:= length(Args) -> + #c_call{anno=Apply#c_apply.anno, + module=#c_literal{val=Mod}, + name=#c_literal{val=Name}, + args=Args}; + true -> + Apply + end; +fold_apply(Apply, _, _) -> Apply. + + %% Handling remote calls. The module/name fields have been processed. call(#c_call{args=As}=Call, #c_literal{val=M}=M0, #c_literal{val=N}=N0, Sub) -> @@ -788,6 +825,8 @@ fold_call(Call, #c_literal{val=M}, #c_literal{val=F}, Args, Sub) -> fold_call_1(Call, M, F, Args, Sub); fold_call(Call, _M, _N, _Args, _Sub) -> Call. +fold_call_1(Call, erlang, apply, [Fun,Args], _) -> + simplify_fun_apply(Call, Fun, Args); fold_call_1(Call, erlang, apply, [Mod,Func,Args], _) -> simplify_apply(Call, Mod, Func, Args); fold_call_1(Call, Mod, Name, Args, Sub) -> @@ -1096,24 +1135,38 @@ eval_failure(Call, Reason) -> %% Simplify an apply/3 to a call if the number of arguments %% are known at compile time. -simplify_apply(Call, Mod, Func, Args) -> +simplify_apply(Call, Mod, Func, Args0) -> case is_atom_or_var(Mod) andalso is_atom_or_var(Func) of - true -> simplify_apply_1(Args, Call, Mod, Func, []); - false -> Call + true -> + case get_fixed_args(Args0, []) of + error -> + Call; + {ok,Args} -> + Call#c_call{module=Mod,name=Func,args=Args} + end; + false -> + Call end. - -simplify_apply_1(#c_literal{val=MoreArgs0}, Call, Mod, Func, Args) - when length(MoreArgs0) >= 0 -> - MoreArgs = [#c_literal{val=Arg} || Arg <- MoreArgs0], - Call#c_call{module=Mod,name=Func,args=reverse(Args, MoreArgs)}; -simplify_apply_1(#c_cons{hd=Arg,tl=T}, Call, Mod, Func, Args) -> - simplify_apply_1(T, Call, Mod, Func, [Arg|Args]); -simplify_apply_1(_, Call, _, _, _) -> Call. - is_atom_or_var(#c_literal{val=Atom}) when is_atom(Atom) -> true; is_atom_or_var(#c_var{}) -> true; is_atom_or_var(_) -> false. +simplify_fun_apply(#c_call{anno=Anno}=Call, Fun, Args0) -> + case get_fixed_args(Args0, []) of + error -> + Call; + {ok,Args} -> + #c_apply{anno=Anno,op=Fun,args=Args} + end. + +get_fixed_args(#c_literal{val=MoreArgs0}, Args) + when length(MoreArgs0) >= 0 -> + MoreArgs = [#c_literal{val=Arg} || Arg <- MoreArgs0], + {ok,reverse(Args, MoreArgs)}; +get_fixed_args(#c_cons{hd=Arg,tl=T}, Args) -> + get_fixed_args(T, [Arg|Args]); +get_fixed_args(_, _) -> error. + %% clause(Clause, Cepxr, Context, Sub) -> Clause. clause(#c_clause{pats=Ps0}=Cl, Cexpr, Ctxt, Sub0) -> @@ -2154,7 +2207,7 @@ make_var(A) -> make_var_name() -> N = get(new_var_num), put(new_var_num, N+1), - list_to_atom("@f"++integer_to_list(N)). + N. letify(Bs, Body) -> Ann = cerl:get_ann(Body), diff --git a/lib/compiler/src/v3_codegen.erl b/lib/compiler/src/v3_codegen.erl index a8f4926e55..8e73b613a0 100644 --- a/lib/compiler/src/v3_codegen.erl +++ b/lib/compiler/src/v3_codegen.erl @@ -588,6 +588,7 @@ is_gc_bif(node, 1) -> false; is_gc_bif(element, 2) -> false; is_gc_bif(get, 1) -> false; is_gc_bif(tuple_size, 1) -> false; +is_gc_bif(map_get, 2) -> false; is_gc_bif(Bif, Arity) -> not (erl_internal:bool_op(Bif, Arity) orelse erl_internal:new_type_test(Bif, Arity) orelse @@ -1162,7 +1163,7 @@ select_binary(#k_val_clause{val=#k_binary{segs=#k_var{name=V}},body=B, {Bis0,Aft,St1} = match_cg(B, Vf, Int0, St0#cg{ctx=V}), CtxReg = fetch_var(V, Int0), Live = max_reg(Bef#sr.reg), - Bis1 = [{test,bs_start_match2,{f,Tf},Live,[CtxReg,V],CtxReg}, + Bis1 = [{test,bs_start_match2,{f,Tf},Live,[CtxReg,{context,V}],CtxReg}, {bs_save2,CtxReg,{V,V}}|Bis0], Bis = finish_select_binary(Bis1), {Bis,Aft,St1#cg{ctx=OldCtx}}; @@ -1174,7 +1175,8 @@ select_binary(#k_val_clause{val=#k_binary{segs=#k_var{name=Ivar}},body=B, {Bis0,Aft,St1} = match_cg(B, Vf, Int0, St0#cg{ctx=Ivar}), CtxReg = fetch_var(Ivar, Int0), Live = max_reg(Bef#sr.reg), - Bis1 = [{test,bs_start_match2,{f,Tf},Live,[fetch_var(V, Bef),Ivar],CtxReg}, + Bis1 = [{test,bs_start_match2,{f,Tf},Live, + [fetch_var(V, Bef),{context,Ivar}],CtxReg}, {bs_save2,CtxReg,{Ivar,Ivar}}|Bis0], Bis = finish_select_binary(Bis1), {Bis,Aft,St1#cg{ctx=OldCtx}}. diff --git a/lib/compiler/src/v3_core.erl b/lib/compiler/src/v3_core.erl index 6029b91cdc..4799105d05 100644 --- a/lib/compiler/src/v3_core.erl +++ b/lib/compiler/src/v3_core.erl @@ -1152,7 +1152,7 @@ fun_tq(Cs0, L, St0, NameInfo) -> %% lc_tq(Line, Exp, [Qualifier], Mc, State) -> {LetRec,[PreExp],State}. %% This TQ from Simon PJ pp 127-138. -lc_tq(Line, E, [#igen{anno=GAnno,ceps=Ceps, +lc_tq(Line, E, [#igen{anno=#a{anno=GA}=GAnno,ceps=Ceps, acc_pat=AccPat,acc_guard=AccGuard, skip_pat=SkipPat,tail=Tail,tail_pat=TailPat, arg={Pre,Arg}}|Qs], Mc, St0) -> @@ -1162,7 +1162,7 @@ lc_tq(Line, E, [#igen{anno=GAnno,ceps=Ceps, F = #c_var{anno=LA,name={Name,1}}, Nc = #iapply{anno=GAnno,op=F,args=[Tail]}, {Var,St2} = new_var(St1), - Fc = function_clause([Var], LA, {Name,1}), + Fc = function_clause([Var], GA, {Name,1}), TailClause = #iclause{anno=LAnno,pats=[TailPat],guard=[],body=[Mc]}, Cs0 = case {AccPat,AccGuard} of {SkipPat,[]} -> @@ -1185,9 +1185,9 @@ lc_tq(Line, E, [#igen{anno=GAnno,ceps=Ceps, body=Lps ++ [Lc]}|Cs0], St3} end, - Fun = #ifun{anno=LAnno,id=[],vars=[Var],clauses=Cs,fc=Fc}, - {#iletrec{anno=LAnno#a{anno=[list_comprehension|LA]},defs=[{{Name,1},Fun}], - body=Pre ++ [#iapply{anno=LAnno,op=F,args=[Arg]}]}, + Fun = #ifun{anno=GAnno,id=[],vars=[Var],clauses=Cs,fc=Fc}, + {#iletrec{anno=GAnno#a{anno=[list_comprehension|GA]},defs=[{{Name,1},Fun}], + body=Pre ++ [#iapply{anno=GAnno,op=F,args=[Arg]}]}, Ceps,St4}; lc_tq(Line, E, [#ifilter{}=Filter|Qs], Mc, St) -> filter_tq(Line, E, Filter, Mc, St, Qs, fun lc_tq/5); @@ -2005,7 +2005,7 @@ new_fun_name(Type, #core{fcount=C}=St) -> %% new_var_name(State) -> {VarName,State}. new_var_name(#core{vcount=C}=St) -> - {list_to_atom("@c" ++ integer_to_list(C)),St#core{vcount=C + 1}}. + {C,St#core{vcount=C + 1}}. %% new_var(State) -> {{var,Name},State}. %% new_var(LineAnno, State) -> {{var,Name},State}. diff --git a/lib/compiler/src/v3_kernel.erl b/lib/compiler/src/v3_kernel.erl index fd73e5a7dc..4e3ceedbc0 100644 --- a/lib/compiler/src/v3_kernel.erl +++ b/lib/compiler/src/v3_kernel.erl @@ -157,7 +157,13 @@ include_attribute(_) -> true. function({#c_var{name={F,Arity}=FA},Body}, St0) -> %%io:format("~w/~w~n", [F,Arity]), try - St1 = St0#kern{func=FA,ff=undefined,vcount=0,fcount=0,ds=cerl_sets:new()}, + %% Find a suitable starting value for the variable counter. Note + %% that this pass assumes that new_var_name/1 returns a variable + %% name distinct from any variable used in the entire body of + %% the function. We use integers as variable names to avoid + %% filling up the atom table when compiling huge functions. + Count = cerl_trees:next_free_variable_name(Body), + St1 = St0#kern{func=FA,ff=undefined,vcount=Count,fcount=0,ds=cerl_sets:new()}, {#ifun{anno=Ab,vars=Kvs,body=B0},[],St2} = expr(Body, new_sub(), St1), {B1,_,St3} = ubody(B0, return, St2), %%B1 = B0, St3 = St2, %Null second pass @@ -168,7 +174,6 @@ function({#c_var{name={F,Arity}=FA},Body}, St0) -> erlang:raise(Class, Error, Stack) end. - %% body(Cexpr, Sub, State) -> {Kexpr,[PreKepxr],State}. %% Do the main sequence of a body. A body ends in an atomic value or %% values. Must check if vector first so do expr. @@ -1356,7 +1361,7 @@ new_fun_name(Type, #kern{func={F,Arity},fcount=C}=St) -> %% new_var_name(State) -> {VarName,State}. new_var_name(#kern{vcount=C}=St) -> - {list_to_atom("@k" ++ integer_to_list(C)),St#kern{vcount=C+1}}. + {C,St#kern{vcount=C+1}}. %% new_var(State) -> {#k_var{},State}. @@ -2377,12 +2382,11 @@ uexpr(#k_try{anno=A,arg=A0,vars=Vs,body=B0,evars=Evs,handler=H0}, {A1,Au,St2} = ubody(A0, {break,Avs}, St1), {B1,Bu,St3} = ubody(B0, Br, St2), {H1,Hu,St4} = ubody(H0, Br, St3), - {Rs1,St5} = ensure_return_vars(Rs0, St4), Used = union([Au,subtract(Bu, lit_list_vars(Vs)), subtract(Hu, lit_list_vars(Evs))]), - {#k_try{anno=#k{us=Used,ns=lit_list_vars(Rs1),a=A}, - arg=A1,vars=Vs,body=B1,evars=Evs,handler=H1,ret=Rs1}, - Used,St5} + {#k_try{anno=#k{us=Used,ns=lit_list_vars(Rs0),a=A}, + arg=A1,vars=Vs,body=B1,evars=Evs,handler=H1,ret=Rs0}, + Used,St4} end; uexpr(#k_try{anno=A,arg=A0,vars=Vs,body=B0,evars=Evs,handler=H0}, return, St0) -> @@ -2390,13 +2394,11 @@ uexpr(#k_try{anno=A,arg=A0,vars=Vs,body=B0,evars=Evs,handler=H0}, {A1,Au,St2} = ubody(A0, {break,Avs}, St1), %Must break to clean up here! {B1,Bu,St3} = ubody(B0, return, St2), {H1,Hu,St4} = ubody(H0, return, St3), - NumNew = 1, - {Ns,St5} = new_vars(NumNew, St4), Used = union([Au,subtract(Bu, lit_list_vars(Vs)), subtract(Hu, lit_list_vars(Evs))]), - {#k_try_enter{anno=#k{us=Used,ns=Ns,a=A}, + {#k_try_enter{anno=#k{us=Used,ns=[],a=A}, arg=A1,vars=Vs,body=B1,evars=Evs,handler=H1}, - Used,St5}; + Used,St4}; uexpr(#k_catch{anno=A,body=B0}, {break,Rs0}, St0) -> {Rb,St1} = new_var(St0), {B1,Bu,St2} = ubody(B0, {break,[Rb]}, St1), diff --git a/lib/compiler/src/v3_kernel_pp.erl b/lib/compiler/src/v3_kernel_pp.erl index ac91039ae0..e9cbe81088 100644 --- a/lib/compiler/src/v3_kernel_pp.erl +++ b/lib/compiler/src/v3_kernel_pp.erl @@ -248,7 +248,7 @@ format_1(#k_put{arg=A,ret=Rs}, Ctxt) -> [format(A, Ctxt), format_ret(Rs, ctxt_bump_indent(Ctxt, 1)) ]; -format_1(#k_try{arg=A,vars=Vs,body=B,evars=Evs,handler=H}, Ctxt) -> +format_1(#k_try{arg=A,vars=Vs,body=B,evars=Evs,handler=H,ret=Rs}, Ctxt) -> Ctxt1 = ctxt_bump_indent(Ctxt, Ctxt#ctxt.body_indent), ["try", nl_indent(Ctxt1), @@ -264,7 +264,8 @@ format_1(#k_try{arg=A,vars=Vs,body=B,evars=Evs,handler=H}, Ctxt) -> nl_indent(Ctxt1), format(H, Ctxt1), nl_indent(Ctxt), - "end" + "end", + format_ret(Rs, Ctxt) ]; format_1(#k_try_enter{arg=A,vars=Vs,body=B,evars=Evs,handler=H}, Ctxt) -> Ctxt1 = ctxt_bump_indent(Ctxt, Ctxt#ctxt.body_indent), diff --git a/lib/compiler/test/beam_utils_SUITE.erl b/lib/compiler/test/beam_utils_SUITE.erl index 7686e69b63..b2a5cada3d 100644 --- a/lib/compiler/test/beam_utils_SUITE.erl +++ b/lib/compiler/test/beam_utils_SUITE.erl @@ -25,7 +25,7 @@ is_not_killed/1,is_not_used_at/1, select/1,y_catch/1,otp_8949_b/1,liveopt/1,coverage/1, y_registers/1,user_predef/1,scan_f/1,cafu/1, - receive_label/1,read_size_file_version/1]). + receive_label/1,read_size_file_version/1,not_used/1]). -export([id/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -51,7 +51,8 @@ groups() -> user_predef, scan_f, cafu, - read_size_file_version + read_size_file_version, + not_used ]}]. init_per_suite(Config) -> @@ -507,5 +508,24 @@ do_read_size_file_version(E) -> {ok,MaxFiles} end. +-record(s, { a, b }). +-record(k, { v }). + +not_used(_Config) -> + [] = not_used_p(any, #s{b=true}, #k{}, ignored), + #k{v=42} = not_used_p(any, #s{b=false}, #k{v=42}, ignored), + #k{v=42} = not_used_p(any, #s{b=bad}, #k{v=42}, ignored), + ok. + +not_used_p(_C, S, K, L) when is_record(K, k) -> + if ((S#s.b) and + (S#s.b)) -> + []; + true -> + id(L), + id(K#k.v), + id(K) + end. + %% The identity function. id(I) -> I. diff --git a/lib/compiler/test/beam_validator_SUITE.erl b/lib/compiler/test/beam_validator_SUITE.erl index b8fff7b100..3af71559ae 100644 --- a/lib/compiler/test/beam_validator_SUITE.erl +++ b/lib/compiler/test/beam_validator_SUITE.erl @@ -33,7 +33,8 @@ state_after_fault_in_catch/1,no_exception_in_catch/1, undef_label/1,illegal_instruction/1,failing_gc_guard_bif/1, map_field_lists/1,cover_bin_opt/1, - val_dsetel/1,bad_tuples/1,bad_try_catch_nesting/1]). + val_dsetel/1,bad_tuples/1,bad_try_catch_nesting/1, + receive_stacked/1]). -include_lib("common_test/include/ct.hrl"). @@ -62,7 +63,8 @@ groups() -> state_after_fault_in_catch,no_exception_in_catch, undef_label,illegal_instruction,failing_gc_guard_bif, map_field_lists,cover_bin_opt,val_dsetel, - bad_tuples,bad_try_catch_nesting]}]. + bad_tuples,bad_try_catch_nesting, + receive_stacked]}]. init_per_suite(Config) -> Config. @@ -531,6 +533,52 @@ bad_try_catch_nesting(Config) -> {bad_try_catch_nesting,{y,2},[{{y,1},{trytag,[5]}}]}}}] = Errors, ok. +receive_stacked(Config) -> + Mod = ?FUNCTION_NAME, + Errors = do_val(Mod, Config), + [{{receive_stacked,f1,0}, + {{loop_rec_end,{f,3}}, + 17, + {fragile_message_reference,{y,0}}}}, + {{receive_stacked,f2,0}, + {{test_heap,3,0},10,{fragile_message_reference,{y,1}}}}, + {{receive_stacked,f3,0}, + {{test_heap,3,0},10,{fragile_message_reference,{y,1}}}}, + {{receive_stacked,f4,0}, + {{test_heap,3,0},10,{fragile_message_reference,{y,1}}}}, + {{receive_stacked,f5,0}, + {{loop_rec_end,{f,23}}, + 23, + {fragile_message_reference,{y,1}}}}, + {{receive_stacked,f6,0}, + {{gc_bif,byte_size,{f,29},0,[{y,0}],{x,0}}, + 12, + {fragile_message_reference,{y,0}}}}, + {{receive_stacked,f7,0}, + {{loop_rec_end,{f,33}}, + 20, + {fragile_message_reference,{y,0}}}}, + {{receive_stacked,f8,0}, + {{loop_rec_end,{f,38}}, + 20, + {fragile_message_reference,{y,0}}}}, + {{receive_stacked,m1,0}, + {{loop_rec_end,{f,43}}, + 19, + {fragile_message_reference,{y,0}}}}, + {{receive_stacked,m2,0}, + {{loop_rec_end,{f,48}}, + 33, + {fragile_message_reference,{y,0}}}}] = Errors, + + %% Compile the original source code as a smoke test. + Data = proplists:get_value(data_dir, Config), + Base = atom_to_list(Mod), + File = filename:join(Data, Base), + {ok,Mod,_} = compile:file(File, [binary]), + + ok. + %%%------------------------------------------------------------------------- transform_remove(Remove, Module) -> diff --git a/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.S b/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.S new file mode 100644 index 0000000000..cca052a9c4 --- /dev/null +++ b/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.S @@ -0,0 +1,390 @@ +{module, receive_stacked}. %% version = 0 + +{exports, [{f1,0}, + {f2,0}, + {f3,0}, + {f4,0}, + {f5,0}, + {f6,0}, + {f7,0}, + {f8,0}, + {id,1}, + {m1,0}, + {m2,0}, + {module_info,0}, + {module_info,1}]}. + +{attributes, []}. + +{labels, 57}. + + +{function, f1, 0, 2}. + {label,1}. + {line,[{location,"receive_stacked.erl",15}]}. + {func_info,{atom,receive_stacked},{atom,f1},0}. + {label,2}. + {allocate_zero,1,0}. + {label,3}. + {loop_rec,{f,5},{x,0}}. + {move,{x,0},{y,0}}. + {test,is_integer,{f,4},[{y,0}]}. + remove_message. + {move,{integer,42},{x,0}}. + {line,[{location,"receive_stacked.erl",18}]}. + {call,1,{f,52}}. + {move,{y,0},{x,0}}. + {deallocate,1}. + return. + {label,4}. + {loop_rec_end,{f,3}}. + {label,5}. + {wait,{f,3}}. + + +{function, f2, 0, 7}. + {label,6}. + {line,[{location,"receive_stacked.erl",22}]}. + {func_info,{atom,receive_stacked},{atom,f2},0}. + {label,7}. + {allocate_zero,2,0}. + {label,8}. + {loop_rec,{f,10},{x,0}}. + {test,is_nonempty_list,{f,9},[{x,0}]}. + {get_list,{x,0},{y,1},{x,0}}. + {test,is_nil,{f,9},[{x,0}]}. + {test_heap,3,0}. + remove_message. + {put_tuple,2,{y,0}}. + {put,{atom,ok}}. + {put,{y,1}}. + {move,{integer,42},{x,0}}. + {line,[{location,"receive_stacked.erl",26}]}. + {call,1,{f,52}}. + {test_heap,3,0}. + {put_tuple,2,{x,0}}. + {put,{y,0}}. + {put,{y,1}}. + {deallocate,2}. + return. + {label,9}. + {loop_rec_end,{f,8}}. + {label,10}. + {wait,{f,8}}. + + +{function, f3, 0, 12}. + {label,11}. + {line,[{location,"receive_stacked.erl",30}]}. + {func_info,{atom,receive_stacked},{atom,f3},0}. + {label,12}. + {allocate_zero,2,0}. + {label,13}. + {loop_rec,{f,15},{x,0}}. + {test,is_nonempty_list,{f,14},[{x,0}]}. + {get_hd,{x,0},{y,1}}. + {test,is_integer,{f,14},[{y,1}]}. + {test_heap,3,0}. + remove_message. + {put_tuple,2,{y,0}}. + {put,{atom,ok}}. + {put,{y,1}}. + {move,{integer,42},{x,0}}. + {line,[{location,"receive_stacked.erl",34}]}. + {call,1,{f,52}}. + {test_heap,3,0}. + {put_tuple,2,{x,0}}. + {put,{y,0}}. + {put,{y,1}}. + {deallocate,2}. + return. + {label,14}. + {loop_rec_end,{f,13}}. + {label,15}. + {wait,{f,13}}. + + +{function, f4, 0, 17}. + {label,16}. + {line,[{location,"receive_stacked.erl",38}]}. + {func_info,{atom,receive_stacked},{atom,f4},0}. + {label,17}. + {allocate_zero,2,0}. + {label,18}. + {loop_rec,{f,20},{x,0}}. + {test,is_nonempty_list,{f,19},[{x,0}]}. + {get_tl,{x,0},{y,1}}. + {test,is_list,{f,19},[{y,1}]}. + {test_heap,3,0}. + remove_message. + {put_tuple,2,{y,0}}. + {put,{atom,ok}}. + {put,{y,1}}. + {move,{integer,42},{x,0}}. + {line,[{location,"receive_stacked.erl",42}]}. + {call,1,{f,52}}. + {test_heap,3,0}. + {put_tuple,2,{x,0}}. + {put,{y,0}}. + {put,{y,1}}. + {deallocate,2}. + return. + {label,19}. + {loop_rec_end,{f,18}}. + {label,20}. + {wait,{f,18}}. + + +{function, f5, 0, 22}. + {label,21}. + {line,[{location,"receive_stacked.erl",46}]}. + {func_info,{atom,receive_stacked},{atom,f5},0}. + {label,22}. + {allocate_zero,2,0}. + {label,23}. + {loop_rec,{f,25},{x,0}}. + {test,is_tuple,{f,24},[{x,0}]}. + {test,test_arity,{f,24},[{x,0},1]}. + {get_tuple_element,{x,0},0,{y,1}}. + {test,is_integer,{f,24},[{y,1}]}. + remove_message. + {put_map_assoc,{f,0},{literal,#{}},{y,0},0,{list,[{atom,key},{y,1}]}}. + {move,{integer,42},{x,0}}. + {line,[{location,"receive_stacked.erl",50}]}. + {call,1,{f,52}}. + {test_heap,3,0}. + {put_tuple,2,{x,0}}. + {put,{y,0}}. + {put,{y,1}}. + {deallocate,2}. + return. + {label,24}. + {loop_rec_end,{f,23}}. + {label,25}. + {wait,{f,23}}. + + +{function, f6, 0, 27}. + {label,26}. + {line,[{location,"receive_stacked.erl",54}]}. + {func_info,{atom,receive_stacked},{atom,f6},0}. + {label,27}. + {allocate_zero,1,0}. + {label,28}. + {loop_rec,{f,30},{x,0}}. + {test,bs_start_match2,{f,29},1,[{x,0},0],{x,0}}. + {test,bs_get_integer2, + {f,29}, + 1, + [{x,0}, + {integer,8}, + 1, + {field_flags,[{anno,[56,{file,"receive_stacked.erl"}]}, + unsigned,big]}], + {x,1}}. + {test,bs_get_binary2, + {f,29}, + 1, + [{x,0}, + {atom,all}, + 8, + {field_flags,[{anno,[56,{file,"receive_stacked.erl"}]}, + unsigned,big]}], + {y,0}}. + {'%', + {no_bin_opt, + {binary_used_in,{gc_bif,byte_size,{f,29},0,[{y,0}],{x,0}}}, + [56,{file,"receive_stacked.erl"}]}}. + {line,[{location,"receive_stacked.erl",56}]}. + {gc_bif,byte_size,{f,29},0,[{y,0}],{x,0}}. + {test,is_lt,{f,29},[{integer,8},{x,0}]}. + remove_message. + {move,{integer,42},{x,0}}. + {line,[{location,"receive_stacked.erl",57}]}. + {call,1,{f,52}}. + {move,{y,0},{x,0}}. + {deallocate,1}. + return. + {label,29}. + {loop_rec_end,{f,28}}. + {label,30}. + {wait,{f,28}}. + + +{function, f7, 0, 32}. + {label,31}. + {line,[{location,"receive_stacked.erl",61}]}. + {func_info,{atom,receive_stacked},{atom,f7},0}. + {label,32}. + {allocate_zero,1,0}. + {label,33}. + {loop_rec,{f,35},{x,0}}. + {test,bs_start_match2,{f,34},1,[{x,0},0],{x,0}}. + {test,bs_get_integer2, + {f,34}, + 1, + [{x,0}, + {integer,8}, + 1, + {field_flags,[{anno,[63,{file,"receive_stacked.erl"}]}, + unsigned,big]}], + {x,1}}. + {test,bs_get_binary2, + {f,34}, + 1, + [{x,0}, + {atom,all}, + 8, + {field_flags,[{anno,[63,{file,"receive_stacked.erl"}]}, + unsigned,big]}], + {y,0}}. + {'%',{no_bin_opt,{binary_used_in,{test,is_binary,{f,34},[{y,0}]}}, + [63,{file,"receive_stacked.erl"}]}}. + {test,is_binary,{f,34},[{y,0}]}. + remove_message. + {move,{integer,42},{x,0}}. + {line,[{location,"receive_stacked.erl",64}]}. + {call,1,{f,52}}. + {move,{y,0},{x,0}}. + {deallocate,1}. + return. + {label,34}. + {loop_rec_end,{f,33}}. + {label,35}. + {wait,{f,33}}. + + +{function, f8, 0, 37}. + {label,36}. + {line,[{location,"receive_stacked.erl",68}]}. + {func_info,{atom,receive_stacked},{atom,f8},0}. + {label,37}. + {allocate_zero,1,0}. + {label,38}. + {loop_rec,{f,40},{x,0}}. + {test,bs_start_match2,{f,39},1,[{x,0},0],{x,1}}. + {test,bs_get_integer2, + {f,39}, + 2, + [{x,1}, + {integer,8}, + 1, + {field_flags,[{anno,[70,{file,"receive_stacked.erl"}]}, + unsigned,big]}], + {x,2}}. + {test,bs_get_binary2, + {f,39}, + 2, + [{x,1}, + {atom,all}, + 8, + {field_flags,[{anno,[70,{file,"receive_stacked.erl"}]}, + unsigned,big]}], + {y,0}}. + {'%',{no_bin_opt,{[{x,1},{y,0}],{loop_rec_end,{f,38}},not_handled}, + [70,{file,"receive_stacked.erl"}]}}. + {test,is_binary,{f,39},[{x,0}]}. + remove_message. + {move,{integer,42},{x,0}}. + {line,[{location,"receive_stacked.erl",71}]}. + {call,1,{f,52}}. + {move,{y,0},{x,0}}. + {deallocate,1}. + return. + {label,39}. + {loop_rec_end,{f,38}}. + {label,40}. + {wait,{f,38}}. + + +{function, m1, 0, 42}. + {label,41}. + {line,[{location,"receive_stacked.erl",75}]}. + {func_info,{atom,receive_stacked},{atom,m1},0}. + {label,42}. + {allocate_zero,1,0}. + {label,43}. + {loop_rec,{f,45},{x,0}}. + {test,is_map,{f,44},[{x,0}]}. + {get_map_elements,{f,44},{x,0},{list,[{atom,key},{y,0}]}}. + {test,is_integer,{f,44},[{y,0}]}. + remove_message. + {move,{integer,42},{x,0}}. + {line,[{location,"receive_stacked.erl",78}]}. + {call,1,{f,52}}. + {test_heap,2,0}. + {put_list,{y,0},nil,{x,0}}. + {deallocate,1}. + return. + {label,44}. + {loop_rec_end,{f,43}}. + {label,45}. + {wait,{f,43}}. + + +{function, m2, 0, 47}. + {label,46}. + {line,[{location,"receive_stacked.erl",82}]}. + {func_info,{atom,receive_stacked},{atom,m2},0}. + {label,47}. + {allocate_zero,4,0}. + {move,{atom,key1},{x,0}}. + {line,[{location,"receive_stacked.erl",83}]}. + {call,1,{f,52}}. + {move,{x,0},{y,3}}. + {move,{atom,key2},{x,0}}. + {line,[{location,"receive_stacked.erl",84}]}. + {call,1,{f,52}}. + {move,{x,0},{y,2}}. + {label,48}. + {loop_rec,{f,50},{x,0}}. + {test,is_map,{f,49},[{x,0}]}. + {get_map_elements,{f,49},{x,0},{list,[{y,3},{y,1}]}}. + {get_map_elements,{f,49},{x,0},{list,[{y,2},{y,0}]}}. + {test,is_integer,{f,49},[{y,1}]}. + {test,is_integer,{f,49},[{y,0}]}. + remove_message. + {kill,{y,2}}. + {kill,{y,3}}. + {move,{integer,42},{x,0}}. + {line,[{location,"receive_stacked.erl",87}]}. + {call,1,{f,52}}. + {test_heap,3,0}. + {put_tuple,2,{x,0}}. + {put,{y,1}}. + {put,{y,0}}. + {deallocate,4}. + return. + {label,49}. + {loop_rec_end,{f,48}}. + {label,50}. + {wait,{f,48}}. + + +{function, id, 1, 52}. + {label,51}. + {line,[{location,"receive_stacked.erl",91}]}. + {func_info,{atom,receive_stacked},{atom,id},1}. + {label,52}. + return. + + +{function, module_info, 0, 54}. + {label,53}. + {line,[]}. + {func_info,{atom,receive_stacked},{atom,module_info},0}. + {label,54}. + {move,{atom,receive_stacked},{x,0}}. + {line,[]}. + {call_ext_only,1,{extfunc,erlang,get_module_info,1}}. + + +{function, module_info, 1, 56}. + {label,55}. + {line,[]}. + {func_info,{atom,receive_stacked},{atom,module_info},1}. + {label,56}. + {move,{x,0},{x,1}}. + {move,{atom,receive_stacked},{x,0}}. + {line,[]}. + {call_ext_only,2,{extfunc,erlang,get_module_info,2}}. diff --git a/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.erl b/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.erl new file mode 100644 index 0000000000..b95fa9ca62 --- /dev/null +++ b/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.erl @@ -0,0 +1,92 @@ +-module(receive_stacked). +-compile([export_all,nowarn_export_all]). + +%% Messages may be stored outside any process heap until they +%% have been accepted by the 'remove_message' instruction. +%% When matching of a message fails, it is not allowed to +%% leave references to the message or any part of it in +%% the Y registers. An experimental code generator could +%% do that, causing an emulator crash if there happenened to +%% be a garbage collection. +%% +%% The 'S' file corresponding to this file was compiled with +%% that experimental code generator. + +f1() -> + receive + X when is_integer(X) -> + id(42), + X + end. + +f2() -> + receive + [X] -> + Res = {ok,X}, + id(42), + {Res,X} + end. + +f3() -> + receive + [H|_] when is_integer(H) -> + Res = {ok,H}, + id(42), + {Res,H} + end. + +f4() -> + receive + [_|T] when is_list(T) -> + Res = {ok,T}, + id(42), + {Res,T} + end. + +f5() -> + receive + {X} when is_integer(X) -> + Res = #{key=>X}, + id(42), + {Res,X} + end. + +f6() -> + receive + <<_:8,T/binary>> when byte_size(T) > 8 -> + id(42), + T + end. + +f7() -> + receive + <<_:8,T/binary>> when is_binary(T) -> + id(42), + T + end. + +f8() -> + receive + <<_:8,T/binary>> = Bin when is_binary(Bin) -> + id(42), + T + end. + +m1() -> + receive + #{key:=V} when is_integer(V) -> + id(42), + [V] + end. + +m2() -> + K1 = id(key1), + K2 = id(key2), + receive + #{K1:=V1,K2:=V2} when is_integer(V1), is_integer(V2) -> + id(42), + {V1,V2} + end. + +id(I) -> + I. diff --git a/lib/compiler/test/bs_construct_SUITE.erl b/lib/compiler/test/bs_construct_SUITE.erl index da99aba346..7c5ad97f7e 100644 --- a/lib/compiler/test/bs_construct_SUITE.erl +++ b/lib/compiler/test/bs_construct_SUITE.erl @@ -303,7 +303,14 @@ fail(Config) when is_list(Config) -> {'EXIT',{badarg,_}} = (catch <<42.0/integer>>), {'EXIT',{badarg,_}} = (catch <<42/binary>>), {'EXIT',{badarg,_}} = (catch <<an_atom/integer>>), - + + %% Bad literal sizes + Bin = i(<<>>), + {'EXIT',{badarg,_}} = (catch <<0:(-1)>>), + {'EXIT',{badarg,_}} = (catch <<Bin/binary,0:(-1)>>), + {'EXIT',{badarg,_}} = (catch <<0:(-(1 bsl 100))>>), + {'EXIT',{badarg,_}} = (catch <<Bin/binary,0:(-(1 bsl 100))>>), + ok. float_bin(Config) when is_list(Config) -> diff --git a/lib/compiler/test/compile_SUITE.erl b/lib/compiler/test/compile_SUITE.erl index eee5bc733f..a1de8961bd 100644 --- a/lib/compiler/test/compile_SUITE.erl +++ b/lib/compiler/test/compile_SUITE.erl @@ -1111,10 +1111,30 @@ remove_compiler_gen(M) -> remove_compiler_gen_1(Pair) -> Op0 = cerl:map_pair_op(Pair), Op = cerl:set_ann(Op0, []), - K = cerl:map_pair_key(Pair), - V = cerl:map_pair_val(Pair), + K = map_var(cerl:map_pair_key(Pair)), + V = map_var(cerl:map_pair_val(Pair)), cerl:update_c_map_pair(Pair, Op, K, V). +map_var(Var) -> + case cerl:is_c_var(Var) of + true -> + case cerl:var_name(Var) of + Name when is_atom(Name) -> + L = atom_to_list(Name), + try list_to_integer(L) of + Int -> + cerl:update_c_var(Var, Int) + catch + error:_ -> + Var + end; + _ -> + Var + end; + false -> + Var + end. + %% Compile to Beam assembly language (.S) and then try to %% run .S through the compiler again. diff --git a/lib/compiler/test/core_fold_SUITE.erl b/lib/compiler/test/core_fold_SUITE.erl index 4fd1f84569..ab7f36abf7 100644 --- a/lib/compiler/test/core_fold_SUITE.erl +++ b/lib/compiler/test/core_fold_SUITE.erl @@ -27,7 +27,8 @@ multiple_aliases/1,redundant_boolean_clauses/1, mixed_matching_clauses/1,unnecessary_building/1, no_no_file/1,configuration/1,supplies/1, - redundant_stack_frame/1,export_from_case/1]). + redundant_stack_frame/1,export_from_case/1, + empty_values/1]). -export([foo/0,foo/1,foo/2,foo/3]). @@ -47,7 +48,8 @@ groups() -> multiple_aliases,redundant_boolean_clauses, mixed_matching_clauses,unnecessary_building, no_no_file,configuration,supplies, - redundant_stack_frame,export_from_case]}]. + redundant_stack_frame,export_from_case, + empty_values]}]. init_per_suite(Config) -> @@ -584,5 +586,17 @@ export_from_case_2(Bool, Rec) -> end, {ok,Result}. +empty_values(_Config) -> + case ?MODULE of + core_fold_inline_SUITE -> + {'EXIT',_} = (catch do_empty_values()); + _ -> + {'EXIT',{function_clause,_}} = (catch do_empty_values()) + end, + ok. + +do_empty_values() when (#{})#{} -> + c. + id(I) -> I. diff --git a/lib/compiler/test/fun_SUITE.erl b/lib/compiler/test/fun_SUITE.erl index 16474adf5b..3c272a35a6 100644 --- a/lib/compiler/test/fun_SUITE.erl +++ b/lib/compiler/test/fun_SUITE.erl @@ -194,6 +194,17 @@ external(Config) when is_list(Config) -> ?APPLY2(ListsMod, ListsMap, 2), ?APPLY2(ListsMod, ListsMap, ListsArity), + 42 = (fun erlang:abs/1)(-42), + 42 = (id(fun erlang:abs/1))(-42), + 42 = apply(fun erlang:abs/1, [-42]), + 42 = apply(id(fun erlang:abs/1), [-42]), + 6 = (fun lists:sum/1)([1,2,3]), + 6 = (id(fun lists:sum/1))([1,2,3]), + + {'EXIT',{{badarity,_},_}} = (catch (fun lists:sum/1)(1, 2, 3)), + {'EXIT',{{badarity,_},_}} = (catch (id(fun lists:sum/1))(1, 2, 3)), + {'EXIT',{{badarity,_},_}} = (catch apply(fun lists:sum/1, [1,2,3])), + ok. call_me(I) -> diff --git a/lib/compiler/test/lc_SUITE.erl b/lib/compiler/test/lc_SUITE.erl index 9ad417b09b..699081470d 100644 --- a/lib/compiler/test/lc_SUITE.erl +++ b/lib/compiler/test/lc_SUITE.erl @@ -107,6 +107,31 @@ basic(Config) when is_list(Config) -> [] = [X || X <- L1, X+1 < 2], {'EXIT',_} = (catch [X || X <- L1, odd(X)]), fc([x], catch [E || E <- id(x)]), + + %% Make sure that line numbers point out the generator. + case ?MODULE of + lc_inline_SUITE -> + ok; + _ -> + {'EXIT',{function_clause, + [{?MODULE,_,_, + [{file,"bad_lc.erl"},{line,4}]}|_]}} = + (catch bad_generator(a)), + {'EXIT',{function_clause, + [{?MODULE,_,_, + [{file,"bad_lc.erl"},{line,4}]}|_]}} = + (catch bad_generator([a|b])), + {'EXIT',{badarg, + [{erlang,length,_,_}, + {?MODULE,bad_generator_bc,1, + [{file,"bad_lc.erl"},{line,7}]}|_]}} = + (catch bad_generator_bc(a)), + {'EXIT',{badarg, + [{erlang,length,_,_}, + {?MODULE,bad_generator_bc,1, + [{file,"bad_lc.erl"},{line,7}]}|_]}} = + (catch bad_generator_bc([a|b])) + end, ok. tuple_list() -> @@ -249,3 +274,11 @@ fc(Args, {'EXIT',{function_clause,[{?MODULE,_,Arity,_}|_]}}) fc(Args, {'EXIT',{{case_clause,ActualArgs},_}}) when ?MODULE =:= lc_inline_SUITE -> Args = tuple_to_list(ActualArgs). + +-file("bad_lc.erl", 1). +bad_generator(List) -> %Line 2 + [I || %Line 3 + I <- List]. %Line 4 +bad_generator_bc(List) -> %Line 5 + << <<I:4>> || %Line 6 + I <- List>>. %Line 7 diff --git a/lib/compiler/test/map_SUITE.erl b/lib/compiler/test/map_SUITE.erl index f15917e3cb..e98c295da6 100644 --- a/lib/compiler/test/map_SUITE.erl +++ b/lib/compiler/test/map_SUITE.erl @@ -36,7 +36,7 @@ t_guard_fun/1, t_list_comprehension/1, t_map_sort_literals/1, - t_map_size/1, + t_map_size/1, t_map_get/1, t_build_and_match_aliasing/1, t_is_map/1, @@ -67,8 +67,10 @@ %% errors in 18 t_register_corruption/1, - t_bad_update/1 + t_bad_update/1, + %% new in OTP 21 + t_reused_key_variable/1 ]). suite() -> []. @@ -89,7 +91,7 @@ all() -> t_guard_receive, t_guard_receive_large, t_guard_fun, t_list_comprehension, t_map_sort_literals, - t_map_size, + t_map_size, t_map_get, t_build_and_match_aliasing, t_is_map, @@ -120,7 +122,10 @@ all() -> %% errors in 18 t_register_corruption, - t_bad_update + t_bad_update, + + %% new in OTP 21 + t_reused_key_variable ]. groups() -> []. @@ -686,6 +691,26 @@ t_map_size(Config) when is_list(Config) -> map_is_size(M,N) when map_size(M) =:= N -> true; map_is_size(_,_) -> false. +t_map_get(Config) when is_list(Config) -> + 1 = map_get(a, id(#{a=>1})), + + {'EXIT',{{badkey,a},_}} = (catch map_get(a, #{})), + {'EXIT',{{badkey,a},_}} = (catch map_get(a, #{b=>1})), + + M = #{"a"=>1, "b" => 2}, + true = check_map_value(M, "a", 1), + false = check_map_value(M, "b", 1), + true = check_map_value(M#{"c"=>2}, "c", 2), + false = check_map_value(M#{"a"=>5}, "a", 1), + + {'EXIT',{{badmap,[]},_}} = (catch map_get(a, [])), + {'EXIT',{{badmap,<<1,2,3>>},_}} = (catch map_get(a, <<1,2,3>>)), + {'EXIT',{{badmap,1},_}} = (catch map_get(a, 1)), + ok. + +check_map_value(Map, Key, Value) when map_get(Key, Map) =:= Value -> true; +check_map_value(_, _, _) -> false. + t_is_map(Config) when is_list(Config) -> true = is_map(#{}), true = is_map(#{a=>1}), @@ -1980,6 +2005,16 @@ properly(Item) -> increase(Allows) -> catch fun() -> Allows end#{[] => +Allows, "warranty" => fun id/1}. +t_reused_key_variable(Config) when is_list(Config) -> + Key = id(key), + Map1 = id(#{Key=>Config}), + Map2 = id(#{Key=>Config}), + case {Map1,Map2} of + %% core_lint treated Key as pattern variables, not input variables, + %% and complained about the variable being duplicated. + {#{Key:=Same},#{Key:=Same}} -> + ok + end. %% aux diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c index 149387bcee..dbb6bf8135 100644 --- a/lib/crypto/c_src/crypto.c +++ b/lib/crypto/c_src/crypto.c @@ -3004,16 +3004,21 @@ static ERL_NIF_TERM rsa_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF static ERL_NIF_TERM dh_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {/* (PrivKey|undefined, DHParams=[P,G], Mpint, Len|0) */ - DH* dh_params; + DH* dh_params = NULL; int pub_len, prv_len; unsigned char *pub_ptr, *prv_ptr; ERL_NIF_TERM ret, ret_pub, ret_prv, head, tail; int mpint; /* 0 or 4 */ - BIGNUM *priv_key = NULL; + BIGNUM *priv_key_in = NULL; BIGNUM *dh_p = NULL, *dh_g = NULL; unsigned long len = 0; +#ifdef HAS_EVP_PKEY_CTX + EVP_PKEY_CTX *ctx = NULL; + EVP_PKEY *dhkey = NULL, + *params = NULL; +#endif - if (!(get_bn_from_bin(env, argv[0], &priv_key) + if (!(get_bn_from_bin(env, argv[0], &priv_key_in) || argv[0] == atom_undefined) || !enif_get_list_cell(env, argv[1], &head, &tail) || !get_bn_from_bin(env, head, &dh_p) @@ -3021,40 +3026,63 @@ static ERL_NIF_TERM dh_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_ || !get_bn_from_bin(env, head, &dh_g) || !enif_is_empty_list(env, tail) || !enif_get_int(env, argv[2], &mpint) || (mpint & ~4) - || !enif_get_ulong(env, argv[3], &len) ) { - - if (priv_key) BN_free(priv_key); + || !enif_get_ulong(env, argv[3], &len) + + /* Load dh_params with values to use by the generator. + Mem mgmnt transfered from dh_p etc to dh_params */ + || !(dh_params = DH_new()) + || (priv_key_in && !DH_set0_key(dh_params, NULL, priv_key_in)) + || !DH_set0_pqg(dh_params, dh_p, NULL, dh_g) + ) { + if (priv_key_in) BN_free(priv_key_in); if (dh_p) BN_free(dh_p); if (dh_g) BN_free(dh_g); + if (dh_params) DH_free(dh_params); return enif_make_badarg(env); } - dh_params = DH_new(); - DH_set0_key(dh_params, NULL, priv_key); - DH_set0_pqg(dh_params, dh_p, NULL, dh_g); - if (len) { if (len < BN_num_bits(dh_p)) DH_set_length(dh_params, len); else { - DH_free(dh_params); + if (priv_key_in) BN_free(priv_key_in); + if (dh_p) BN_free(dh_p); + if (dh_g) BN_free(dh_g); + if (dh_params) DH_free(dh_params); return enif_make_badarg(env); } } +#ifdef HAS_EVP_PKEY_CTX + if ((dhkey = EVP_PKEY_new()) + && (params = EVP_PKEY_new()) + && EVP_PKEY_set1_DH(params, dh_params) /* set the key referenced by params to dh_params. + dh_params (and params) must be freed by us*/ + && (ctx = EVP_PKEY_CTX_new(params, NULL)) + && EVP_PKEY_keygen_init(ctx) + && EVP_PKEY_keygen(ctx, &dhkey) + && (dh_params = EVP_PKEY_get1_DH(dhkey)) /* return the referenced key. dh_params and dhkey must be freed */ + ) { +#else if (DH_generate_key(dh_params)) { - const BIGNUM *pub_key, *priv_key; - DH_get0_key(dh_params, &pub_key, &priv_key); - pub_len = BN_num_bytes(pub_key); - prv_len = BN_num_bytes(priv_key); +#endif + const BIGNUM *pub_key_gen, *priv_key_gen; + + DH_get0_key(dh_params, + &pub_key_gen, &priv_key_gen); /* Get pub_key_gen and priv_key_gen. + "The values point to the internal representation of + the public key and private key values. This memory + should not be freed directly." says man */ + pub_len = BN_num_bytes(pub_key_gen); + prv_len = BN_num_bytes(priv_key_gen); pub_ptr = enif_make_new_binary(env, pub_len+mpint, &ret_pub); prv_ptr = enif_make_new_binary(env, prv_len+mpint, &ret_prv); if (mpint) { put_int32(pub_ptr, pub_len); pub_ptr += 4; put_int32(prv_ptr, prv_len); prv_ptr += 4; } - BN_bn2bin(pub_key, pub_ptr); - BN_bn2bin(priv_key, prv_ptr); + BN_bn2bin(pub_key_gen, pub_ptr); + BN_bn2bin(priv_key_gen, prv_ptr); ERL_VALGRIND_MAKE_MEM_DEFINED(pub_ptr, pub_len); ERL_VALGRIND_MAKE_MEM_DEFINED(prv_ptr, prv_len); ret = enif_make_tuple2(env, ret_pub, ret_prv); @@ -3062,21 +3090,33 @@ static ERL_NIF_TERM dh_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_ else { ret = atom_error; } + DH_free(dh_params); +#ifdef HAS_EVP_PKEY_CTX + if (ctx) EVP_PKEY_CTX_free(ctx); + /* if (dhkey) EVP_PKEY_free(dhkey); */ + /* if (params) EVP_PKEY_free(params); */ +#endif return ret; } static ERL_NIF_TERM dh_compute_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {/* (OthersPublicKey, MyPrivateKey, DHParams=[P,G]) */ - DH* dh_params; - BIGNUM *dummy_pub_key = NULL, *priv_key = NULL; - BIGNUM *other_pub_key; - BIGNUM *dh_p = NULL, *dh_g = NULL; - int i; + BIGNUM *dummy_pub_key = NULL, + *priv_key = NULL, + *other_pub_key = NULL, + *dh_p = NULL, + *dh_g = NULL; ErlNifBinary ret_bin; ERL_NIF_TERM ret, head, tail; - - dh_params = DH_new(); + DH *dh_priv = DH_new(), *dh_pub = DH_new(); +#ifdef HAS_EVP_PKEY_CTX + EVP_PKEY_CTX *ctx = NULL; + EVP_PKEY *my_priv_key = NULL, *peer_pub_key = NULL; + size_t skeylen; +#else + int i; +#endif if (!get_bn_from_bin(env, argv[0], &other_pub_key) || !get_bn_from_bin(env, argv[1], &priv_key) @@ -3084,35 +3124,78 @@ static ERL_NIF_TERM dh_compute_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_T || !get_bn_from_bin(env, head, &dh_p) || !enif_get_list_cell(env, tail, &head, &tail) || !get_bn_from_bin(env, head, &dh_g) - || !enif_is_empty_list(env, tail)) { + || !enif_is_empty_list(env, tail) + + /* Note: DH_set0_key() does not allow setting only the + * private key, although DH_compute_key() does not use the + * public key. Work around this limitation by setting + * the public key to a copy of the private key. + */ + || !(dummy_pub_key = BN_dup(priv_key)) + || !DH_set0_key(dh_priv, dummy_pub_key, priv_key) + || !DH_set0_pqg(dh_priv, dh_p, NULL, dh_g) + ) { if (dh_p) BN_free(dh_p); if (dh_g) BN_free(dh_g); - ret = enif_make_badarg(env); + if (other_pub_key) BN_free(other_pub_key); + if (dummy_pub_key) BN_free(dummy_pub_key); + if (priv_key) BN_free(priv_key); + return enif_make_badarg(env); + } + +#ifdef HAS_EVP_PKEY_CTX + if (!(my_priv_key = EVP_PKEY_new()) + || !EVP_PKEY_set1_DH(my_priv_key, dh_priv) /* set the key referenced by my_priv_key to dh_priv. + dh_priv (and my_priv_key) must be freed by us*/ + + || !(peer_pub_key = EVP_PKEY_new()) + || !DH_set0_key(dh_pub, other_pub_key, NULL) + || !DH_set0_pqg(dh_pub, dh_p, NULL, dh_g) + || !EVP_PKEY_set1_DH(peer_pub_key, dh_pub) + + || !(ctx = EVP_PKEY_CTX_new(my_priv_key, NULL)) + || (EVP_PKEY_derive_init(ctx) <= 0) + || (EVP_PKEY_derive_set_peer(ctx, peer_pub_key) <= 0) + || (EVP_PKEY_derive(ctx, NULL, &skeylen) <= 0)) { + + ret = atom_error; } else { - /* Note: DH_set0_key() does not allow setting only the - * private key, although DH_compute_key() does not use the - * public key. Work around this limitation by setting - * the public key to a copy of the private key. - */ - dummy_pub_key = BN_dup(priv_key); - DH_set0_key(dh_params, dummy_pub_key, priv_key); - DH_set0_pqg(dh_params, dh_p, NULL, dh_g); - enif_alloc_binary(DH_size(dh_params), &ret_bin); - i = DH_compute_key(ret_bin.data, other_pub_key, dh_params); - if (i > 0) { - if (i != ret_bin.size) { - enif_realloc_binary(&ret_bin, i); - } - ret = enif_make_binary(env, &ret_bin); - } - else { + enif_alloc_binary(skeylen, &ret_bin); + + if ((EVP_PKEY_derive(ctx, ret_bin.data, &skeylen) > 0) + && (ret_bin.size >= skeylen)) { + /* Derivation succeded */ + if (ret_bin.size > skeylen) enif_realloc_binary(&ret_bin, skeylen); + ret = enif_make_binary(env, &ret_bin); + } + else { enif_release_binary(&ret_bin); - ret = atom_error; - } + ret = atom_error; + } } + +#else + enif_alloc_binary(DH_size(dh_priv), &ret_bin); + i = DH_compute_key(ret_bin.data, other_pub_key, dh_priv); + if (i > 0) { + if (i != ret_bin.size) enif_realloc_binary(&ret_bin, i); + ret = enif_make_binary(env, &ret_bin); + } + else { + enif_release_binary(&ret_bin); + ret = atom_error; + } +#endif + if (other_pub_key) BN_free(other_pub_key); - DH_free(dh_params); + if (dh_priv) DH_free(dh_priv); + if (dh_pub) DH_free(dh_pub); +#ifdef HAS_EVP_PKEY_CTX + if (ctx) EVP_PKEY_CTX_free(ctx); + /* if (my_priv_key) EVP_PKEY_free(my_priv_key); */ + /* if (peer_pub_key) EVP_PKEY_free(peer_pub_key); */ +#endif return ret; } diff --git a/lib/crypto/doc/src/crypto.xml b/lib/crypto/doc/src/crypto.xml index 3a5efd0bea..af676d9021 100644 --- a/lib/crypto/doc/src/crypto.xml +++ b/lib/crypto/doc/src/crypto.xml @@ -136,6 +136,7 @@ See also <seealso marker="#supports-0">crypto:supports/0</seealso> </p> + <marker id="type-engine_key_ref"/> <marker id="engine_key_ref_type"/> <code>engine_key_ref() = #{engine := engine_ref(), key_id := key_id(), diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index ec2a1dba0a..46775989ae 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -847,8 +847,13 @@ on_load() -> case Status of ok -> ok; {error, {E, Str}} -> - error_logger:error_msg("Unable to load crypto library. Failed with error:~n\"~p, ~s\"~n" - "OpenSSL might not be installed on this system.~n",[E,Str]), + Fmt = "Unable to load crypto library. Failed with error:~n\"~p, ~s\"~n~s", + Extra = case E of + load_failed -> + "OpenSSL might not be installed on this system.\n"; + _ -> "" + end, + error_logger:error_msg(Fmt, [E,Str,Extra]), Status end. diff --git a/lib/crypto/test/crypto_SUITE.erl b/lib/crypto/test/crypto_SUITE.erl index 6dab459df6..d148fa3856 100644 --- a/lib/crypto/test/crypto_SUITE.erl +++ b/lib/crypto/test/crypto_SUITE.erl @@ -131,7 +131,8 @@ groups() -> {ecdsa, [], [sign_verify %% Does not work yet: ,public_encrypt, private_encrypt ]}, - {dh, [], [generate_compute]}, + {dh, [], [generate_compute, + compute_bug]}, {ecdh, [], [compute, generate]}, {srp, [], [generate_compute]}, {des_cbc, [], [block]}, @@ -463,6 +464,24 @@ generate_compute(Config) when is_list(Config) -> GenCom = proplists:get_value(generate_compute, Config), lists:foreach(fun do_generate_compute/1, GenCom). %%-------------------------------------------------------------------- +compute_bug() -> + [{doc, "Test that it works even if the Secret is smaller than expected"}]. +compute_bug(Config) -> + ExpectedSecret = <<118,89,171,16,156,18,156,103,189,134,130,49,28,144,111,241,247,82,79,32,228,11,209,141,119,176,251,80,105,143,235,251,203,121,223,211,129,3,233,133,45,2,31,157,24,111,5,75,153,66,135,185,128,115,229,178,216,39,73,52,80,151,8,241,34,52,226,71,137,167,53,48,59,224,175,154,89,110,76,83,24,117,149,21,72,6,186,78,149,74,188,56,98,244,30,77,108,248,88,194,195,237,23,51,20,242,254,123,21,12,209,74,217,168,230,65,7,60,211,139,128,239,234,153,22,229,180,59,159,121,41,156,121,200,177,130,163,162,54,224,93,1,94,11,177,254,118,28,156,26,116,10,207,145,219,166,214,189,214,230,221,170,228,15,69,88,31,68,94,255,113,58,49,82,86,192,248,176,131,133,39,186,194,172,206,84,184,16,66,68,153,128,178,227,27,118,52,130,122,92,24,222,102,195,221,207,255,13,152,175,65,32,167,84,54,244,243,109,244,18,234,16,159,224,188,2,106,123,27,17,131,171,226,34,111,251,62,119,155,124,221,124,254,62,97,167,1,105,116,98,98,19,197,30,72,180,79,221,100,134,120,117,124,85,73,132,224,223,222,41,155,137,218,130,238,237,157,161,134,150,69,206,91,141,17,89,120,218,235,229,37,150,76,197,7,157,56,144,42,203,137,100,200,72,141,194,239,1,67,236,238,183,48,214,75,76,108,235,3,237,67,40,137,45,182,236,246,37,116,103,144,237,142,211,88,233,11,24,21,218,41,245,250,51,130,250,104,74,189,17,69,145,70,50,50,215,253,155,10,128,41,114,185,211,82,164,72,92,17,145,104,66,6,140,226,80,43,62,1,166,216,153,118,96,15,147,126,137,118,191,192,75,149,241,206,18,92,17,154,215,219,18,6,139,190,103,210,156,184,29,224,213,157,60,112,189,104,220,125,40,186,50,119,17,143,136,149,38,74,107,21,192,59,61,59,42,231,144,59,175,3,176,87,23,16,122,54,31,82,34,230,211,44,81,41,47,86,37,228,175,130,148,88,136,131,254,241,202,99,199,175,1,141,215,124,155,120,43,141,89,11,140,120,141,29,35,82,219,155,204,75,12,66,241,253,33,250,84,24,85,68,13,80,85,142,227,34,139,26,146,24>>, + OthersPublicKey = 635619632099733175381667940709387641100492974601603060984753028943194386334921787463327680809776598322996634648015962954045728174069768874873236397421720142610982770302060309928552098274817978606093380781524199673890631795310930242601197479471368910519338301177304682162189801040921618559902948819107531088646753320486728060005223263561551402855338732899079439899705951063999951507319258050864346087428042978411873495523439615429804957374639092580169417598963105885529553632847023899713490485619763926900318508906706745060947269748612049634207985438016935262521715769812475329234748426647554362991758104620357149045960316987533503707855364806010494793980069245562784050236811004893018183726397041999426883788660276453352521120006817370050691205529335316794439089316232980047277245051173281601960196573681285904611182521967067911862467395705665888521948321299521549941618586026714676885890192323289343756440666276226084448279082483536164085883288884231665240707495770544705648564889889198060417915693315346959170105413290799314390963124178046425737828369059171472978294050322371452255088799865552038756937873388385970088906560408959959429398326288750834357514847891423941047433478384621074116184703014798814515161475596555032391555842, + MyPrivateKey = 387759582879975726965038486537011291913744975764132199838375902680222019267527675651273586836110220500657652661706223760165097275862806031329642160439090779625708664007910974206651834216043397115514725827856461492311499129200688538220719685637154290305617686974719521885238198226075381217068175824097878445476010193039590876624464274744156624589136789060427283492343902761765833713520850870233407503430180028104167029073459918756981323130062648615262139444306321256382009848217866984408901761817655567071716275177768316006340055589170095799943481591033461616307776069027985761229636731465482676467627154100912586936231051371168178564599296638350391246393336702334311781595616786107810962134407697848002331639021101685320844880636050048769216986088652236979636019052557155807310341483407890060105599892252118584570558049301477535792498672552850760356632076013402382600669875697284264329434950712239302528367835155163504374877787288116104285944993818319105835423479332617802010952731990182088670508346704423006877514817882782443833997288652405892920173712497948376815825396272381214976859009518623799156300136570204539240675245115597412280078940442452936425561984312708387584800789375684525365060589104566195610526570099527133097201479, + P = 818034524162384276004384029858643530286875094391273833506734966261806257117433972760379103507630310628953496150318170372254219924175532996281953750642804369831900894594960807970232131410638888573275563720690293481410915588408505771183615664441221559618326229227448328879290185035795866796496147000467456347856187771645103733376761936369144682074588463621004219054111244232031965820058057143484947957179035662640791007685559024477920075136419228662974090561346329074935312181886940693299380892129818458511403741106419480550004799657220331973244248753744516935739033770420884365608406478656143540532371463443324228730693491647103274058971797182813283112583029849186056551355376851686616057869624968077484471229044125401535456699914745876082047459812392122562460031611344154642406382436701361983114768023990405077450124649862159757605118611426368650203370143674925598905779061402007525955196464201496773278952462368223659263492419274489020447849336502432222101793313731259141617677580646998184158969477474527427664187763741360356528830301163614618231141541403007931347398186427059736520580903587497382362610721261644208653717495736748724114113311672504064943864203789205551568648546606356374830209356446449765364678719909024329058480379, + G = 2, + DHParameters = [P, G], + case crypto:compute_key(dh, OthersPublicKey, MyPrivateKey, DHParameters) of + ExpectedSecret -> + ok; + Others -> + ct:log("Got ~p",[Others]), + {fail, "crypto:compute_key(dh,...) failed for the bug test"} + end. + +%%-------------------------------------------------------------------- no_generate_compute() -> [{doc, "Test crypto:genarate_key and crypto:compute_key " "for disabled algorithms"}]. diff --git a/lib/dialyzer/test/small_SUITE_data/results/fun_arity b/lib/dialyzer/test/small_SUITE_data/results/fun_arity index e916b2483f..8b7a538758 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/fun_arity +++ b/lib/dialyzer/test/small_SUITE_data/results/fun_arity @@ -1,37 +1,37 @@ -fun_arity.erl:100: Fun application will fail since _@c1 :: fun(() -> any()) is not a function of arity 1 +fun_arity.erl:100: Fun application will fail since _1 :: fun(() -> any()) is not a function of arity 1 fun_arity.erl:100: Function 'Mfa_0_ko'/1 has no local return -fun_arity.erl:104: Fun application will fail since _@c1 :: fun((_) -> any()) is not a function of arity 0 +fun_arity.erl:104: Fun application will fail since _1 :: fun((_) -> any()) is not a function of arity 0 fun_arity.erl:104: Function 'Mfa_1_ko'/1 has no local return -fun_arity.erl:111: Fun application will fail since _@c1 :: fun(() -> any()) is not a function of arity 1 +fun_arity.erl:111: Fun application will fail since _1 :: fun(() -> any()) is not a function of arity 1 fun_arity.erl:111: Function mFa_0_ko/1 has no local return -fun_arity.erl:115: Fun application will fail since _@c1 :: fun((_) -> any()) is not a function of arity 0 +fun_arity.erl:115: Fun application will fail since _1 :: fun((_) -> any()) is not a function of arity 0 fun_arity.erl:115: Function mFa_1_ko/1 has no local return -fun_arity.erl:122: Fun application will fail since _@c2 :: fun(() -> any()) is not a function of arity 1 +fun_arity.erl:122: Fun application will fail since _2 :: fun(() -> any()) is not a function of arity 1 fun_arity.erl:122: Function 'MFa_0_ko'/2 has no local return -fun_arity.erl:126: Fun application will fail since _@c2 :: fun((_) -> any()) is not a function of arity 0 +fun_arity.erl:126: Fun application will fail since _2 :: fun((_) -> any()) is not a function of arity 0 fun_arity.erl:126: Function 'MFa_1_ko'/2 has no local return -fun_arity.erl:35: Fun application will fail since _@c0 :: fun(() -> 'ok') is not a function of arity 1 +fun_arity.erl:35: Fun application will fail since _0 :: fun(() -> 'ok') is not a function of arity 1 fun_arity.erl:35: Function f_0_ko/0 has no local return -fun_arity.erl:39: Fun application will fail since _@c0 :: fun((_) -> 'ok') is not a function of arity 0 +fun_arity.erl:39: Fun application will fail since _0 :: fun((_) -> 'ok') is not a function of arity 0 fun_arity.erl:39: Function f_1_ko/0 has no local return -fun_arity.erl:48: Fun application will fail since _@c0 :: fun(() -> 'ok') is not a function of arity 1 +fun_arity.erl:48: Fun application will fail since _0 :: fun(() -> 'ok') is not a function of arity 1 fun_arity.erl:48: Function fa_0_ko/0 has no local return -fun_arity.erl:53: Fun application will fail since _@c0 :: fun((_) -> 'ok') is not a function of arity 0 +fun_arity.erl:53: Fun application will fail since _0 :: fun((_) -> 'ok') is not a function of arity 0 fun_arity.erl:53: Function fa_1_ko/0 has no local return -fun_arity.erl:63: Fun application will fail since _@c0 :: fun(() -> any()) is not a function of arity 1 +fun_arity.erl:63: Fun application will fail since _0 :: fun(() -> any()) is not a function of arity 1 fun_arity.erl:63: Function mfa_0_ko/0 has no local return -fun_arity.erl:68: Fun application will fail since _@c0 :: fun((_) -> any()) is not a function of arity 0 +fun_arity.erl:68: Fun application will fail since _0 :: fun((_) -> any()) is not a function of arity 0 fun_arity.erl:68: Function mfa_1_ko/0 has no local return -fun_arity.erl:76: Fun application will fail since _@c0 :: fun(() -> any()) is not a function of arity 1 +fun_arity.erl:76: Fun application will fail since _0 :: fun(() -> any()) is not a function of arity 1 fun_arity.erl:76: Function mfa_ne_0_ko/0 has no local return fun_arity.erl:78: Function mf_ne/0 will never be called -fun_arity.erl:81: Fun application will fail since _@c0 :: fun((_) -> any()) is not a function of arity 0 +fun_arity.erl:81: Fun application will fail since _0 :: fun((_) -> any()) is not a function of arity 0 fun_arity.erl:81: Function mfa_ne_1_ko/0 has no local return fun_arity.erl:83: Function mf_ne/1 will never be called -fun_arity.erl:89: Fun application will fail since _@c0 :: fun(() -> any()) is not a function of arity 1 +fun_arity.erl:89: Fun application will fail since _0 :: fun(() -> any()) is not a function of arity 1 fun_arity.erl:89: Function mfa_nd_0_ko/0 has no local return fun_arity.erl:90: Call to missing or unexported function fun_arity:mf_nd/0 -fun_arity.erl:93: Fun application will fail since _@c0 :: fun((_) -> any()) is not a function of arity 0 +fun_arity.erl:93: Fun application will fail since _0 :: fun((_) -> any()) is not a function of arity 0 fun_arity.erl:93: Function mfa_nd_1_ko/0 has no local return fun_arity.erl:94: Call to missing or unexported function fun_arity:mf_nd/1 diff --git a/lib/diameter/doc/src/diameter.xml b/lib/diameter/doc/src/diameter.xml index 6bc7d147c0..dfa4c803ed 100644 --- a/lib/diameter/doc/src/diameter.xml +++ b/lib/diameter/doc/src/diameter.xml @@ -135,7 +135,7 @@ along with any extra arguments to be appended to those documented. Note that extra arguments specific to an outgoing request can be specified to &call;, in which case -those are are appended to any module-specific extra arguments.</p> +those are appended to any module-specific extra arguments.</p> <p> Specifying a <c>#diameter_callback{}</c> record allows individual @@ -150,7 +150,7 @@ See <c>diameter_callback.erl</c> for details.</p> <p> Options defining a Diameter application. -Has one the following types.</p> +Has one of the following types.</p> <taglist> @@ -2221,7 +2221,7 @@ Stop a diameter service.</p> <p> Stopping a service causes all associated transport connections to be broken. -A DPR message with be sent as in the case of &remove_transport;.</p> +A DPR message will be sent as in the case of &remove_transport;.</p> <note> <p> diff --git a/lib/diameter/doc/src/diameter_codec.xml b/lib/diameter/doc/src/diameter_codec.xml index 5124b49484..0a34dd7ec7 100644 --- a/lib/diameter/doc/src/diameter_codec.xml +++ b/lib/diameter/doc/src/diameter_codec.xml @@ -255,7 +255,7 @@ Another list-valued representation allows a message to be specified as a list whose head is a &header; and whose tail is an &avp; list. This representation is used by diameter itself when relaying requests as directed by the return value of a &app_handle_request; callback. -It differs from the other other two in that it bypasses the checks for +It differs from the other two in that it bypasses the checks for messages that do not agree with their definitions in the dictionary in question: messages are sent exactly as specified.</p> diff --git a/lib/diameter/doc/src/diameter_dict.xml b/lib/diameter/doc/src/diameter_dict.xml index 94016d9466..37d30b709b 100644 --- a/lib/diameter/doc/src/diameter_dict.xml +++ b/lib/diameter/doc/src/diameter_dict.xml @@ -486,9 +486,9 @@ will result in the following record definition given an empty prefix.</p> <pre> --record('SIP-Deregistration-Reason' {'SIP-Reason-Code', - 'SIP-Reason-Info', - 'AVP'}). +-record('SIP-Deregistration-Reason', {'SIP-Reason-Code', + 'SIP-Reason-Info', + 'AVP'}). </pre> <p> diff --git a/lib/diameter/doc/src/diameter_soc.xml b/lib/diameter/doc/src/diameter_soc.xml index 28e01ff1be..2d2d66a243 100644 --- a/lib/diameter/doc/src/diameter_soc.xml +++ b/lib/diameter/doc/src/diameter_soc.xml @@ -137,7 +137,7 @@ Capitalized <em>Diameter</em> refers to the protocol, lowercase <cell>Creating New AVPs</cell> <cell>&NA;</cell> <cell>New AVPs can be defined using the dictionary interface. - Both both RFC data formats and extensions are supported.</cell> + Both RFC data formats and extensions are supported.</cell> </row> <row> <cell>1.3.3</cell> diff --git a/lib/diameter/doc/src/notes.xml b/lib/diameter/doc/src/notes.xml index fa1be39b5b..f74a4ca2a2 100644 --- a/lib/diameter/doc/src/notes.xml +++ b/lib/diameter/doc/src/notes.xml @@ -75,7 +75,7 @@ first.</p> <list> <item> <p> - Fix documentation typo: peer_down/3 was written where + Fix documentation typo: peer_up/3 was written where peer_down/3 was intended.</p> <p> Own Id: OTP-14805</p> diff --git a/lib/diameter/src/diameter.appup.src b/lib/diameter/src/diameter.appup.src index 05a8c9378e..3389f11937 100644 --- a/lib/diameter/src/diameter.appup.src +++ b/lib/diameter/src/diameter.appup.src @@ -57,7 +57,8 @@ {"2.1", [{restart_application, diameter}]}, %% 20.1 {"2.1.1", [{restart_application, diameter}]}, %% 20.1.2 {"2.1.2", [{restart_application, diameter}]}, %% 20.1.3 - {"2.1.3", [{restart_application, diameter}]} %% 20.2 + {"2.1.3", [{restart_application, diameter}]}, %% 20.2 + {"2.1.4", [{restart_application, diameter}]} %% 20.3 ], [ {"0.9", [{restart_application, diameter}]}, @@ -96,6 +97,7 @@ {"2.1", [{restart_application, diameter}]}, {"2.1.1", [{restart_application, diameter}]}, {"2.1.2", [{restart_application, diameter}]}, - {"2.1.3", [{restart_application, diameter}]} + {"2.1.3", [{restart_application, diameter}]}, + {"2.1.4", [{restart_application, diameter}]} ] }. diff --git a/lib/diameter/vsn.mk b/lib/diameter/vsn.mk index b0fb4ada28..3081034df9 100644 --- a/lib/diameter/vsn.mk +++ b/lib/diameter/vsn.mk @@ -17,5 +17,5 @@ # %CopyrightEnd% APPLICATION = diameter -DIAMETER_VSN = 2.1.4 +DIAMETER_VSN = 2.1.5 APP_VSN = $(APPLICATION)-$(DIAMETER_VSN)$(PRE_VSN) diff --git a/lib/edoc/doc/src/Makefile b/lib/edoc/doc/src/Makefile index ca9ea66e3c..71de42795a 100644 --- a/lib/edoc/doc/src/Makefile +++ b/lib/edoc/doc/src/Makefile @@ -54,9 +54,10 @@ XML_NOTES_FILES = notes.xml BOOK_FILES = book.xml XML_FILES=\ - $(BOOK_FILES) $(XML_CHAPTER_FILES) \ - $(XML_PART_FILES) $(XML_REF3_FILES) $(XML_APPLICATION_FILES) \ - $(XML_NOTES_FILES) + $(BOOK_FILES) $(XML_APPLICATION_FILES) \ + $(XML_PART_FILES) $(XML_NOTES_FILES) + +XML_GEN_FILES=$(XML_REF3_FILES:%=$(XMLDIR)/%) $(XML_CHAPTER_FILES:%=$(XMLDIR)/%) # ---------------------------------------------------- INFO_FILE = ../../info @@ -101,11 +102,11 @@ html: gifs $(HTML_REF_MAN_FILE) man: $(MAN3_FILES) -$(XML_REF3_FILES): - escript $(DOCGEN)/priv/bin/xml_from_edoc.escript -def vsn $(EDOC_VSN) -i $(ERL_TOP)/lib/edoc/include $(SRC_DIR)/$(@:%.xml=%.erl) +$(XML_REF3_FILES:%=$(XMLDIR)/%): + $(gen_verbose)escript $(DOCGEN)/priv/bin/xml_from_edoc.escript -def vsn $(EDOC_VSN) -i $(ERL_TOP)/lib/edoc/include -dir $(XMLDIR) $(SRC_DIR)/$(@:$(XMLDIR)/%.xml=%.erl) -$(XML_CHAPTER_FILES): ../overview.edoc - escript $(DOCGEN)/priv/bin/xml_from_edoc.escript -def vsn $(EDOC_VSN) -chapter ../overview.edoc +$(XML_CHAPTER_FILES:%=$(XMLDIR)/%): ../overview.edoc + $(gen_verbose)escript $(DOCGEN)/priv/bin/xml_from_edoc.escript -def vsn $(EDOC_VSN) -chapter -dir $(XMLDIR) $< gifs: $(GIF_FILES:%=$(HTMLDIR)/%) diff --git a/lib/edoc/src/edoc_doclet.erl b/lib/edoc/src/edoc_doclet.erl index 0e084e619e..f55cffe158 100644 --- a/lib/edoc/src/edoc_doclet.erl +++ b/lib/edoc/src/edoc_doclet.erl @@ -255,7 +255,7 @@ modules_frame(Dir, Ms, Title, CSS) -> ?NL, {table, [{width, "100%"}, {border, 0}, {summary, "list of modules"}], - lists:concat( + lists:append( [[?NL, {tr, [{td, [], [{a, [{href, module_ref(M)}, @@ -448,7 +448,7 @@ application_frame(Dir, Apps, Title, CSS) -> {h2, ["Applications"]}, ?NL, {table, [{width, "100%"}, {border, 0}], - lists:concat( + lists:append( [[{tr, [{td, [], [{a, [{href,app_ref(Path,App)}, {target,"_top"}], [App]}]}]}] diff --git a/lib/erl_docgen/priv/bin/codeline_preprocessing.escript b/lib/erl_docgen/priv/bin/codeline_preprocessing.escript index 8e1e35bcdd..67966b79e6 100755 --- a/lib/erl_docgen/priv/bin/codeline_preprocessing.escript +++ b/lib/erl_docgen/priv/bin/codeline_preprocessing.escript @@ -30,7 +30,7 @@ %% Function: main/1 %% Description: %%---------------------------------------------------------------------- -main([InFile, OutFile]) -> +main([CPath, InFile, OutFile]) -> InDev = case file:open(InFile, [read]) of {ok,ID} -> @@ -38,7 +38,6 @@ main([InFile, OutFile]) -> _ -> halt(5) end, - CPath=filename:dirname(InFile), OutDev = case file:open(OutFile, [write]) of {ok,OD} -> diff --git a/lib/erl_docgen/priv/bin/github_link.escript b/lib/erl_docgen/priv/bin/github_link.escript new file mode 100755 index 0000000000..1b36fca202 --- /dev/null +++ b/lib/erl_docgen/priv/bin/github_link.escript @@ -0,0 +1,51 @@ +#!/usr/bin/env escript +%% -*- erlang -*- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%%---------------------------------------------------------------------- +%% File : github_link.escript +%% +%% Created : 12 Dec 2017 by Lukas Larsson +%%---------------------------------------------------------------------- + +main([In, Filename, Sha, Out]) -> + {ok, Bin} = file:read_file(In), + + TagsToAnnotate = ["description", "func", "datatype", "section"], + + Subs = subs(TagsToAnnotate, Filename, Sha, re:split(Bin,[$\n])), + + file:write_file(Out, Subs). + +subs([], _, _, Bin) -> + lists:join("\n", Bin); +subs([Pat|Pats], Fn, Sha, Bin) -> + subs(Pats, Fn, Sha, sub(Bin, Pat, Fn, Sha)). + +sub(Bin, Pat, Fn, Sha) -> + sub(Bin, Pat, Fn, Sha, 1). +sub([], _Pat, _Fn, _Sha, _Cnt) -> + []; +sub([H|T], Pat, Fn, Sha, Cnt) -> + %% We use the maint branch here, it is not as exact as the tag, + %% but it is the best we can do as github does not allow doing + %% pullrequests on anything but branches. + [re:replace(H,["<",Pat,">"], + ["<",Pat," ghlink=\"maint/",Fn,"#L", + integer_to_list(Cnt),"\">"],[{return,list}]) | + sub(T, Pat, Fn, Sha, Cnt+1)]. diff --git a/lib/erl_docgen/priv/bin/xml_from_edoc.escript b/lib/erl_docgen/priv/bin/xml_from_edoc.escript index b930ae3818..b0e3764fae 100755 --- a/lib/erl_docgen/priv/bin/xml_from_edoc.escript +++ b/lib/erl_docgen/priv/bin/xml_from_edoc.escript @@ -28,6 +28,7 @@ %% Records %%====================================================================== -record(args, {suffix=".xml", + dir=".", layout=docgen_edoc_xml_cb, def=[], includes=[], @@ -85,7 +86,7 @@ module(File, Args) -> {app_default, "OTPROOT"}, {file_suffix, Args#args.suffix}, - {dir, "."}, + {dir, Args#args.dir}, {layout, Args#args.layout}], edoc:file(File, Opts); false -> @@ -118,7 +119,7 @@ users_guide(File, Args) -> Text = edoc_lib:run_layout(F, Opts), OutFile = "chapter" ++ Args#args.suffix, - edoc_lib:write_file(Text, ".", OutFile, Encoding); + edoc_lib:write_file(Text, Args#args.dir, OutFile, Encoding); false -> io:format("~s: not a regular file\n", [File]), usage() @@ -139,6 +140,8 @@ parse(["-def", Key, Val |RawOpts], Type, Args) -> parse(["-i", Dir |RawOpts], Type, Args) -> Args2 = Args#args{includes=Args#args.includes++[Dir]}, parse(RawOpts, Type, Args2); +parse(["-dir", Dir |RawOpts], Type, Args) -> + parse(RawOpts, Type, Args#args{dir=Dir}); parse(["-preprocess", Bool |RawOpts], Type, Args) when Bool == "true"; Bool == "false" -> parse(RawOpts, Type, Args#args{preprocess=list_to_atom(Bool)}); diff --git a/lib/erl_docgen/priv/css/otp_doc.css b/lib/erl_docgen/priv/css/otp_doc.css index 844aad2945..34c6befb0e 100644 --- a/lib/erl_docgen/priv/css/otp_doc.css +++ b/lib/erl_docgen/priv/css/otp_doc.css @@ -242,8 +242,25 @@ th { font-size: small; } -h3>a{ - color: #1a1a1a !important; +.title_link { + color: #1a1a1a !important; + outline: none; +} + +.ghlink { + margin-left: -2.7em; /* .pencil.font-size + .pencil.padding.left + .pencil.padding.right = 2.7 */ + visibility: hidden; +} + +.pencil:before { + transform: rotateZ(90deg); + content: "\270E"; + color: #1a1a1a !important; + font-weight: bold; + font-size: 1.5em; + padding: .3em .6em .6em; + line-height: 1em; + font-family: mono; } hr{ diff --git a/lib/erl_docgen/priv/dtd/chapter.dtd b/lib/erl_docgen/priv/dtd/chapter.dtd index 8d940b90f7..3e9113d798 100644 --- a/lib/erl_docgen/priv/dtd/chapter.dtd +++ b/lib/erl_docgen/priv/dtd/chapter.dtd @@ -35,3 +35,4 @@ <!ELEMENT section (marker*,title, (%block;|quote|warning|note|dont|do|br|image|marker| table|section)*) > +<!ATTLIST section ghlink CDATA #IMPLIED> diff --git a/lib/erl_docgen/priv/dtd/common.image.dtd b/lib/erl_docgen/priv/dtd/common.image.dtd index d97057590e..138da3609b 100644 --- a/lib/erl_docgen/priv/dtd/common.image.dtd +++ b/lib/erl_docgen/priv/dtd/common.image.dtd @@ -18,5 +18,7 @@ $Id$ --> <!ELEMENT image (icaption) > -<!ATTLIST image file CDATA #REQUIRED > +<!ATTLIST image + file CDATA #REQUIRED + width CDATA #IMPLIED > <!ELEMENT icaption (#PCDATA) > diff --git a/lib/erl_docgen/priv/dtd/common.refs.dtd b/lib/erl_docgen/priv/dtd/common.refs.dtd index 4f87007a09..07c876a17f 100644 --- a/lib/erl_docgen/priv/dtd/common.refs.dtd +++ b/lib/erl_docgen/priv/dtd/common.refs.dtd @@ -26,8 +26,10 @@ %common.header; <!ELEMENT description (%block;|quote|br|marker|warning|note|dont|do)* > +<!ATTLIST description ghlink CDATA #IMPLIED> <!ELEMENT funcs (func)+ > <!ELEMENT func (name+,fsummary,(type|type_desc)*,desc?) > +<!ATTLIST func ghlink CDATA #IMPLIED> <!-- ELEMENT name is defined in each ref dtd --> <!ELEMENT fsummary (#PCDATA|c|i|em|anno)* > <!ELEMENT type (v,d?)* > @@ -42,8 +44,11 @@ <!ELEMENT email (#PCDATA) > <!ELEMENT section (marker*,title,(%block;|quote|br|marker| warning|note|dont|do|section)*) > -<!ELEMENT datatypes (datatype)+ > +<!ATTLIST section ghlink CDATA #IMPLIED> +<!ELEMENT datatypes (datatype_title?,datatype)+ > +<!ELEMENT datatype_title (#PCDATA) > <!ELEMENT datatype (name+,desc?) > +<!ATTLIST datatype ghlink CDATA #IMPLIED> <!ELEMENT type_desc (#PCDATA|anno|c|seealso)* > <!ATTLIST type_desc variable CDATA #IMPLIED name CDATA #IMPLIED> diff --git a/lib/erl_docgen/priv/xsl/db_html.xsl b/lib/erl_docgen/priv/xsl/db_html.xsl index 5b7eae4f73..a0a922216b 100644 --- a/lib/erl_docgen/priv/xsl/db_html.xsl +++ b/lib/erl_docgen/priv/xsl/db_html.xsl @@ -3,7 +3,7 @@ # # %CopyrightBegin% # - # Copyright Ericsson AB 2009-2017. All Rights Reserved. + # Copyright Ericsson AB 2009-2018. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -54,6 +54,24 @@ <func:result select="$result"/> </func:function> + <func:function name="erl:lower-case"> + <xsl:param name="str"/> + + <xsl:variable name="uppercase" select="'ABCDEFGHIJKLMNOPQRSTUVWXYZ'"/> + <xsl:variable name="lowercase" select="'abcdefghijklmnopqrstuvwxyz'"/> + + <xsl:variable name="result"> + <xsl:value-of select="translate($str, $uppercase, $lowercase)"/> + </xsl:variable> + + <func:result select="$result"/> + </func:function> + + <func:function name="erl:to-link"> + <xsl:param name="text"/> + <func:result select="translate(erl:lower-case($text),'?: /()" ','--------')"/> + </func:function> + <!-- Used from template menu.funcs to sort a module's functions for the lefthand index list, from the module's .xml file. Returns a value on which to sort the entity in question (a <name> element). @@ -208,6 +226,7 @@ <xsl:variable name="local_types" select="../type[string-length(@name) > 0]"/> <xsl:apply-templates select="$spec/contract/clause/head"> + <xsl:with-param name="ghlink" select="ancestor-or-self::*[@ghlink]/@ghlink"/> <xsl:with-param name="local_types" select="$local_types"/> <xsl:with-param name="global_types" select="$global_types"/> </xsl:apply-templates> @@ -216,9 +235,17 @@ </xsl:template> <xsl:template match="head"> + <xsl:param name="ghlink"/> <xsl:param name="local_types"/> <xsl:param name="global_types"/> - <div class="bold_code func-head"> + <xsl:variable name="id" select="concat(concat(concat(concat(../../../name,'-'),../../../arity),'-'),generate-id(.))"/> + <div class="bold_code func-head" + onMouseOver="document.getElementById('ghlink-{$id}').style.visibility = 'visible';" + onMouseOut="document.getElementById('ghlink-{$id}').style.visibility = 'hidden';"> + <xsl:call-template name="ghlink"> + <xsl:with-param name="ghlink" select="$ghlink"/> + <xsl:with-param name="id" select="$id"/> + </xsl:call-template> <xsl:apply-templates mode="local_type"> <xsl:with-param name="local_types" select="$local_types"/> <xsl:with-param name="global_types" select="$global_types"/> @@ -403,18 +430,37 @@ <!-- Datatypes --> <xsl:template match="datatypes"> - <h3> - <a name="data-types" href="#data-types"><xsl:text>Data Types</xsl:text></a> - </h3> - <div class="data-types-body"> - <xsl:apply-templates/> - </div> + <xsl:call-template name="h3_title_link"> + <xsl:with-param name="title">Data Types</xsl:with-param> + </xsl:call-template> + <xsl:apply-templates/> + </xsl:template> + + <!-- Datatype Title, is the really needed? not used by anything --> + <xsl:template match="datatype_title"> + <xsl:variable name="title" select="."/> + <h4> + <xsl:call-template name="title_link"> + <xsl:with-param name="title"><xsl:apply-templates/></xsl:with-param> + <xsl:with-param name="link" select="$title"/> + </xsl:call-template> + </h4> </xsl:template> <!-- Datatype --> <xsl:template match="datatype"> - <div class="data-type-name"><xsl:apply-templates select="name"/></div> - <div class="data-type-desc"><xsl:apply-templates select="desc"/></div> + <xsl:variable name="id" select="concat('type-',name/@name)"/> + <div class="data-types-body"> + <div class="data-type-name" + onMouseOver="document.getElementById('ghlink-{$id}').style.visibility = 'visible';" + onMouseOut="document.getElementById('ghlink-{$id}').style.visibility = 'hidden';"> + <xsl:call-template name="ghlink"> + <xsl:with-param name="id" select="$id"/> + </xsl:call-template> + <xsl:apply-templates select="name"/> + </div> + <div class="data-type-desc"><xsl:apply-templates select="desc"/></div> + </div> </xsl:template> <!-- The "mode" attribute of apply has been used to separate the case @@ -896,7 +942,7 @@ <!-- Header --> <xsl:template match="header"/> - + <!-- Section/Title --> <xsl:template match="section/title"/> @@ -909,10 +955,12 @@ <xsl:for-each select="marker"> <xsl:call-template name="marker-before-title"/> </xsl:for-each> - <a name="{generate-id(title)}"> - <xsl:value-of select="$chapnum"/>.<xsl:number/>  - <xsl:value-of select="title"/> - </a> + <xsl:call-template name="title_link"> + <xsl:with-param name="title"> + <xsl:value-of select="$chapnum"/>.<xsl:number/>  + <xsl:value-of select="title"/> + </xsl:with-param> + </xsl:call-template> </h3> <xsl:apply-templates> <xsl:with-param name="chapnum" select="$chapnum"/> @@ -929,7 +977,9 @@ <xsl:call-template name="marker-before-title"/> </xsl:for-each> <!-- xsl:value-of select="$partnum"/>.<xsl:value-of select="$chapnum"/>.<xsl:value-of select="$sectnum"/>.<xsl:number/ --> - <xsl:value-of select="title"/> + <xsl:call-template name="title_link"> + <xsl:with-param name="title" select="title"/> + </xsl:call-template> </h4> <xsl:apply-templates> <xsl:with-param name="chapnum" select="$chapnum"/> @@ -959,9 +1009,9 @@ <xsl:for-each select="marker"> <xsl:call-template name="marker-before-title"/> </xsl:for-each> - <a name="{generate-id(title)}"> - <xsl:value-of select="title"/> - </a> + <xsl:call-template name="title_link"> + <xsl:with-param name="title" select="title"/> + </xsl:call-template> </h3> <div class="REFBODY rb-3"> <xsl:apply-templates> @@ -1214,7 +1264,14 @@ </xsl:variable> <div class="doc-image-wrapper"> - <img alt="IMAGE MISSING" src="{@file}" class="doc-image"/> + <xsl:choose> + <xsl:when test="@width"> + <img alt="IMAGE MISSING" width="{@width}" src="{@file}" class="doc-image"/> + </xsl:when> + <xsl:otherwise> + <img alt="IMAGE MISSING" src="{@file}" class="doc-image"/> + </xsl:otherwise> + </xsl:choose> <xsl:apply-templates> <xsl:with-param name="chapnum" select="$chapnum"/> @@ -1351,7 +1408,7 @@ <xsl:param name="chapter_file"/> <xsl:for-each select="$entries"> <li title="{title}"> - <a href="{$chapter_file}.html#{generate-id(title)}"> + <a href="{$chapter_file}.html#{erl:to-link(title)}"> <xsl:value-of select="title"/> </a> </li> @@ -1805,7 +1862,9 @@ <!-- Module --> <xsl:template match="module"> <xsl:param name="partnum"/> - <h3><a name="module" href="#module">Module</a></h3> + <xsl:call-template name="h3_title_link"> + <xsl:with-param name="title">Module</xsl:with-param> + </xsl:call-template> <div class="REFBODY module-body"> <xsl:apply-templates> <xsl:with-param name="partnum" select="$partnum"/> @@ -1817,7 +1876,9 @@ <!-- Modulesummary --> <xsl:template match="modulesummary"> <xsl:param name="partnum"/> - <h3><a name="module-summary" href="#module-summary">Module Summary</a></h3> + <xsl:call-template name="h3_title_link"> + <xsl:with-param name="title">Module Summary</xsl:with-param> + </xsl:call-template> <div class="REFBODY module-summary-body"> <xsl:apply-templates> <xsl:with-param name="partnum" select="$partnum"/> @@ -1828,7 +1889,9 @@ <!-- Lib --> <xsl:template match="lib"> <xsl:param name="partnum"/> - <h3><a name="c-library" href="#c-library">C Library</a></h3> + <xsl:call-template name="h3_title_link"> + <xsl:with-param name="title">C Library</xsl:with-param> + </xsl:call-template> <div class="REFBODY c-library-body"> <xsl:apply-templates> <xsl:with-param name="partnum" select="$partnum"/> @@ -1840,7 +1903,9 @@ <!-- Libsummary --> <xsl:template match="libsummary"> <xsl:param name="partnum"/> - <h3><a name="library-summary" href="#library-summary">Library Summary</a></h3> + <xsl:call-template name="h3_title_link"> + <xsl:with-param name="title">Library Summary</xsl:with-param> + </xsl:call-template> <div class="REFBODY library-summary-body"> <xsl:apply-templates> <xsl:with-param name="partnum" select="$partnum"/> @@ -1851,7 +1916,9 @@ <!-- Com --> <xsl:template match="com"> <xsl:param name="partnum"/> - <h3><a name="command" href="#command">Command</a></h3> + <xsl:call-template name="h3_title_link"> + <xsl:with-param name="title">Command</xsl:with-param> + </xsl:call-template> <div class="REFBODY command-body"> <xsl:apply-templates> <xsl:with-param name="partnum" select="$partnum"/> @@ -1863,7 +1930,9 @@ <!-- Comsummary --> <xsl:template match="comsummary"> <xsl:param name="partnum"/> - <h3><a name="command-summary" href="#command-summary">Command Summary</a></h3> + <xsl:call-template name="h3_title_link"> + <xsl:with-param name="title">Command Summary</xsl:with-param> + </xsl:call-template> <div class="REFBODY command-summary-body"> <xsl:apply-templates> <xsl:with-param name="partnum" select="$partnum"/> @@ -1874,7 +1943,9 @@ <!-- File --> <xsl:template match="file"> <xsl:param name="partnum"/> - <h3><a name="file" href="#file">File</a></h3> + <xsl:call-template name="h3_title_link"> + <xsl:with-param name="title">File</xsl:with-param> + </xsl:call-template> <div class="REFBODY file-body"> <xsl:apply-templates> <xsl:with-param name="partnum" select="$partnum"/> @@ -1886,7 +1957,9 @@ <!-- Filesummary --> <xsl:template match="filesummary"> <xsl:param name="partnum"/> - <h3><a name="file-summary" href="#file-summary">File Summary</a></h3> + <xsl:call-template name="h3_title_link"> + <xsl:with-param name="title">File Summary</xsl:with-param> + </xsl:call-template> <div class="REFBODY file-summary-body"> <xsl:apply-templates> <xsl:with-param name="partnum" select="$partnum"/> @@ -1898,7 +1971,9 @@ <!-- App --> <xsl:template match="app"> <xsl:param name="partnum"/> - <h3><a name="application" href="#application">Application</a></h3> + <xsl:call-template name="h3_title_link"> + <xsl:with-param name="title">Application</xsl:with-param> + </xsl:call-template> <div class="REFBODY application-body"> <xsl:apply-templates> <xsl:with-param name="partnum" select="$partnum"/> @@ -1910,7 +1985,9 @@ <!-- Appsummary --> <xsl:template match="appsummary"> <xsl:param name="partnum"/> - <h3><a name="application-summary" href="#application-summary">Application Summary</a></h3> + <xsl:call-template name="h3_title_link"> + <xsl:with-param name="title">Application Summary</xsl:with-param> + </xsl:call-template> <div class="REFBODY application-summary-body"> <xsl:apply-templates> <xsl:with-param name="partnum" select="$partnum"/> @@ -1921,7 +1998,9 @@ <!-- Description --> <xsl:template match="description"> <xsl:param name="partnum"/> - <h3><a name="description" href="#description">Description</a></h3> + <xsl:call-template name="h3_title_link"> + <xsl:with-param name="title">Description</xsl:with-param> + </xsl:call-template> <div class="REFBODY description-body"> <p> <xsl:apply-templates> @@ -1935,7 +2014,9 @@ <xsl:template match="funcs"> <xsl:param name="partnum"/> - <h3><a name="exports" href="#exports"><xsl:text>Exports</xsl:text></a></h3> + <xsl:call-template name="h3_title_link"> + <xsl:with-param name="title">Exports</xsl:with-param> + </xsl:call-template> <div class="exports-body"> <xsl:apply-templates> @@ -1952,7 +2033,8 @@ <p><xsl:apply-templates select="name"/> <xsl:apply-templates select="name[string-length(@arity) > 0 and position()=last()]" - mode="types"/></p> + mode="types"/> + </p> <xsl:apply-templates select="fsummary|type|desc"> <xsl:with-param name="partnum" select="$partnum"/> @@ -2011,14 +2093,19 @@ <xsl:choose> <xsl:when test="ancestor::cref"> - <a name="{substring-before(nametext, '(')}"> - <span class="bold_code bc-7"> - <xsl:value-of select="ret"/> - <xsl:call-template name="maybe-space-after-ret"> - <xsl:with-param name="s" select="ret"/> - </xsl:call-template> - <xsl:value-of select="nametext"/> - </span></a><br/> + <span class="bold_code bc-7"> + <xsl:call-template name="title_link"> + <xsl:with-param name="link" select="substring-before(nametext, '(')"/> + <xsl:with-param name="title"> + <xsl:value-of select="ret"/> + <xsl:call-template name="maybe-space-after-ret"> + <xsl:with-param name="s" select="ret"/> + </xsl:call-template> + <xsl:value-of select="nametext"/> + </xsl:with-param> + </xsl:call-template> + </span> + <br/> </xsl:when> <xsl:when test="ancestor::erlref"> <xsl:variable name="fname"> @@ -2039,15 +2126,29 @@ </xsl:variable> <xsl:choose> <xsl:when test="ancestor::datatype"> - <a name="type-{$fname}"></a><span class="bold_code bc-8"><xsl:apply-templates/></span><br/> + <div class="bold_code bc-8"> + <xsl:call-template name="title_link"> + <xsl:with-param name="link" select="concat('type-',$fname)"/> + <xsl:with-param name="title"> + <xsl:apply-templates/> + </xsl:with-param> + </xsl:call-template> + </div> </xsl:when> <xsl:otherwise> - <a name="{$fname}-{$arity}"></a><span class="bold_code fun-type"><xsl:apply-templates/></span><br/> + <div class="bold_code fun-type"> + <xsl:call-template name="title_link"> + <xsl:with-param name="link" select="concat(concat($fname,'-'),$arity)"/> + <xsl:with-param name="title"> + <xsl:apply-templates/> + </xsl:with-param> + </xsl:call-template> + </div> </xsl:otherwise> </xsl:choose> </xsl:when> <xsl:otherwise> - <span class="bold_code bc-10"><xsl:value-of select="."/></span> + <div class="bold_code bc-10"><xsl:value-of select="."/></div> </xsl:otherwise> </xsl:choose> @@ -2107,6 +2208,49 @@ </div> </xsl:template> + <xsl:template name="h3_title_link"> + <xsl:param name="title"/> + <h3> + <xsl:call-template name="title_link"> + <xsl:with-param name="title" select="$title"/> + <xsl:with-param name="link" select="erl:to-link($title)"/> + </xsl:call-template> + </h3> + </xsl:template> + + <xsl:template name="title_link"> + <xsl:param name="title"/> + <xsl:param name="link" select="erl:to-link(title)"/> + <xsl:param name="ghlink" select="ancestor-or-self::*[@ghlink][position() = 1]/@ghlink"/> + <xsl:variable name="id" select="concat(concat($link,'-'), generate-id(.))"/> + <span onMouseOver="document.getElementById('ghlink-{$id}').style.visibility = 'visible';" + onMouseOut="document.getElementById('ghlink-{$id}').style.visibility = 'hidden';"> + <xsl:call-template name="ghlink"> + <xsl:with-param name="id" select="$id"/> + <xsl:with-param name="ghlink" select="$ghlink"/> + </xsl:call-template> + <a class="title_link" name="{$link}" href="#{$link}"><xsl:value-of select="$title"/></a> + </span> + </xsl:template> + + <xsl:template name="ghlink"> + <xsl:param name="id"/> + <xsl:param name="ghlink" select="ancestor-or-self::*[@ghlink][position() = 1]/@ghlink"/> + <xsl:choose> + <xsl:when test="string-length($ghlink) > 0"> + <span id="ghlink-{$id}" class="ghlink"> + <a href="https://github.com/erlang/otp/edit/{$ghlink}" + title="Found an issue with the documentation? Fix it by clicking here!"> + <span class="pencil"/> + </a> + </span> + </xsl:when> + <xsl:otherwise> + <span id="ghlink-{$id}"/> + </xsl:otherwise> + </xsl:choose> + </xsl:template> + <!-- Desc --> <xsl:template match="desc"> <xsl:param name="partnum"/> diff --git a/lib/erl_docgen/priv/xsl/db_man.xsl b/lib/erl_docgen/priv/xsl/db_man.xsl index 03b6b0691d..a5ad7ed5ae 100644 --- a/lib/erl_docgen/priv/xsl/db_man.xsl +++ b/lib/erl_docgen/priv/xsl/db_man.xsl @@ -271,6 +271,12 @@ <xsl:apply-templates/> </xsl:template> + <!-- Datatype Title--> + <xsl:template match="datatype_title"> + <xsl:text> .SS </xsl:text> + <xsl:apply-templates/> + </xsl:template> + <!-- Datatype --> <xsl:template match="datatype"> <xsl:apply-templates/> diff --git a/lib/erl_docgen/priv/xsl/db_pdf.xsl b/lib/erl_docgen/priv/xsl/db_pdf.xsl index 46de66bcd8..1b91d768e3 100644 --- a/lib/erl_docgen/priv/xsl/db_pdf.xsl +++ b/lib/erl_docgen/priv/xsl/db_pdf.xsl @@ -3,7 +3,7 @@ # # %CopyrightBegin% # - # Copyright Ericsson AB 2009-2016. All Rights Reserved. + # Copyright Ericsson AB 2009-2018. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -299,6 +299,13 @@ <xsl:apply-templates/> </xsl:template> + <!-- Datatype Title--> + <xsl:template match="datatype_title"> + <fo:block xsl:use-attribute-sets="h4"> + <xsl:apply-templates/> + </fo:block> + </xsl:template> + <!-- Datatype --> <xsl:template match="datatype"> <fo:block xsl:use-attribute-sets="function-name"> @@ -1649,8 +1656,14 @@ </xsl:variable> <fo:block xsl:use-attribute-sets="image"> - <fo:external-graphic content-width="scale-down-to-fit" inline-progression-dimension.maximum="100%" src="{@file}"/> - + <xsl:choose> + <xsl:when test="@width"> + <fo:external-graphic content-width="scale-to-fit" width="{@width}" inline-progression-dimension.maximum="100%" src="{@file}"/> + </xsl:when> + <xsl:otherwise> + <fo:external-graphic content-width="scale-down-to-fit" inline-progression-dimension.maximum="100%" src="{@file}"/> + </xsl:otherwise> + </xsl:choose> <xsl:apply-templates> <xsl:with-param name="chapnum" select="$chapnum"/> <xsl:with-param name="fignum" select="$fignum"/> diff --git a/lib/erl_docgen/priv/xsl/db_pdf_params.xsl b/lib/erl_docgen/priv/xsl/db_pdf_params.xsl index 99da29c2ac..9bfa991b54 100644 --- a/lib/erl_docgen/priv/xsl/db_pdf_params.xsl +++ b/lib/erl_docgen/priv/xsl/db_pdf_params.xsl @@ -3,7 +3,7 @@ # # %CopyrightBegin% # - # Copyright Ericsson AB 2009-2017. All Rights Reserved. + # Copyright Ericsson AB 2009-2018. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -139,6 +139,7 @@ <xsl:attribute-set name="image"> <xsl:attribute name="space-after">0.5em</xsl:attribute> <xsl:attribute name="space-before">0.5em</xsl:attribute> + <xsl:attribute name="text-align">center</xsl:attribute> </xsl:attribute-set> <xsl:attribute-set name="listblock"> diff --git a/lib/erl_interface/configure.in b/lib/erl_interface/configure.in index 0a8fbf513c..696ebf5ca0 100644 --- a/lib/erl_interface/configure.in +++ b/lib/erl_interface/configure.in @@ -106,6 +106,19 @@ if test $ac_cv_sizeof_long = 8; then CFLAGS="$CFLAGS -DEI_64BIT" fi +LM_HARDWARE_ARCH + +AC_MSG_CHECKING(for unaligned word access) +case "$ARCH" in + x86|amd64) + AC_MSG_RESULT(yes: x86 or amd64) + AC_DEFINE(HAVE_UNALIGNED_WORD_ACCESS, 1, [Define if hw supports unaligned word access]) + ;; + *) + AC_MSG_RESULT(no) + ;; +esac + AC_CHECK_TOOL(AR, ar, false) if test "$AR" = false; then AC_MSG_ERROR([No 'ar' command found in PATH]) diff --git a/lib/erl_interface/doc/src/notes.xml b/lib/erl_interface/doc/src/notes.xml index 641a3de13f..f165dde259 100644 --- a/lib/erl_interface/doc/src/notes.xml +++ b/lib/erl_interface/doc/src/notes.xml @@ -31,6 +31,29 @@ </header> <p>This document describes the changes made to the Erl_interface application.</p> +<section><title>Erl_Interface 3.10.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix bug in <c>ei_connect</c> functions that may cause + failure due to insufficient buffer space for + gethostbyname_r.</p> + <p> + Own Id: OTP-15022 Aux Id: ERIERL-163 </p> + </item> + <item> + <p> + Optimize encoding/decoding for pure 7-bit ascii atoms.</p> + <p> + Own Id: OTP-15023 Aux Id: ERIERL-150 </p> + </item> + </list> + </section> + +</section> + <section><title>Erl_Interface 3.10.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/erl_interface/src/connect/ei_connect.c b/lib/erl_interface/src/connect/ei_connect.c index ea9ecb31d5..5c01223e3d 100644 --- a/lib/erl_interface/src/connect/ei_connect.c +++ b/lib/erl_interface/src/connect/ei_connect.c @@ -583,6 +583,54 @@ static int cnct(uint16 port, struct in_addr *ip_addr, int addr_len, unsigned ms) return s; } /* cnct */ + +/* + * Same as ei_gethostbyname_r, but also handles ERANGE error + * and may allocate larger buffer with malloc. + */ +static +struct hostent *dyn_gethostbyname_r(const char *name, + struct hostent *hostp, + char **buffer_p, + int buflen, + int *h_errnop) +{ + char* buf = *buffer_p; + struct hostent *hp; + + while (1) { + hp = ei_gethostbyname_r(name, hostp, buf, buflen, h_errnop); + if (hp) { + *buffer_p = buf; + break; + } + + if (*h_errnop != ERANGE) { + if (buf != *buffer_p) + free(buf); + break; + } + + buflen *= 2; + if (buf == *buffer_p) + buf = malloc(buflen); + else { + char* buf2 = realloc(buf, buflen); + if (buf2) + buf = buf2; + else { + free(buf); + buf = NULL; + } + } + if (!buf) { + *h_errnop = ENOMEM; + break; + } + } + return hp; +} + /* * Set up a connection to a given Node, and * interchange hand shake messages with it. @@ -597,8 +645,10 @@ int ei_connect_tmo(ei_cnode* ec, char *nodename, unsigned ms) /* these are needed for the call to gethostbyname_r */ struct hostent host; char buffer[1024]; + char *buf = buffer; int ei_h_errno; #endif /* !win32 */ + int res; /* extract the host and alive parts from nodename */ if (!(hostname = strchr(nodename,'@'))) { @@ -611,7 +661,7 @@ int ei_connect_tmo(ei_cnode* ec, char *nodename, unsigned ms) } #ifndef __WIN32__ - hp = ei_gethostbyname_r(hostname,&host,buffer,1024,&ei_h_errno); + hp = dyn_gethostbyname_r(hostname,&host,&buf,sizeof(buffer),&ei_h_errno); if (hp == NULL) { char thishostname[EI_MAXHOSTNAMELEN+1]; /* gethostname requies len to be max(hostname) + 1*/ @@ -627,7 +677,7 @@ int ei_connect_tmo(ei_cnode* ec, char *nodename, unsigned ms) } if (strcmp(hostname,thishostname) == 0) /* Both nodes on same standalone host, use loopback */ - hp = ei_gethostbyname_r("localhost",&host,buffer,1024,&ei_h_errno); + hp = dyn_gethostbyname_r("localhost",&host,&buf,sizeof(buffer),&ei_h_errno); if (hp == NULL) { EI_TRACE_ERR2("ei_connect", "Can't find host for %s: %d\n",nodename,ei_h_errno); @@ -663,7 +713,14 @@ int ei_connect_tmo(ei_cnode* ec, char *nodename, unsigned ms) } } #endif /* win32 */ - return ei_xconnect_tmo(ec, (Erl_IpAddr) *hp->h_addr_list, alivename, ms); + + res = ei_xconnect_tmo(ec, (Erl_IpAddr) *hp->h_addr_list, alivename, ms); + +#ifndef __WIN32__ + if (buf != buffer) + free(buf); +#endif + return res; } /* ei_connect */ int ei_connect(ei_cnode* ec, char *nodename) diff --git a/lib/erl_interface/src/connect/ei_resolve.c b/lib/erl_interface/src/connect/ei_resolve.c index fd0c659373..2757735d39 100644 --- a/lib/erl_interface/src/connect/ei_resolve.c +++ b/lib/erl_interface/src/connect/ei_resolve.c @@ -645,8 +645,11 @@ struct hostent *ei_gethostbyname_r(const char *name, #else #if (defined(__GLIBC__) || defined(__linux__) || (__FreeBSD_version >= 602000) || defined(__DragonFly__) || defined(__ANDROID__)) struct hostent *result; + int err; - gethostbyname_r(name, hostp, buffer, buflen, &result, h_errnop); + err = gethostbyname_r(name, hostp, buffer, buflen, &result, h_errnop); + if (err == ERANGE) + *h_errnop = err; return result; #else diff --git a/lib/erl_interface/src/decode/decode_atom.c b/lib/erl_interface/src/decode/decode_atom.c index b3bba82434..87cd75b1be 100644 --- a/lib/erl_interface/src/decode/decode_atom.c +++ b/lib/erl_interface/src/decode/decode_atom.c @@ -92,6 +92,51 @@ int ei_decode_atom_as(const char *buf, int *index, char* p, int destlen, } + +#ifdef HAVE_UNALIGNED_WORD_ACCESS + +#if SIZEOF_VOID_P == SIZEOF_LONG +typedef unsigned long AsciiWord; +#elif SIZEOF_VOID_P == SIZEOF_LONG_LONG +typedef unsigned long long AsciiWord; +#else +# error "Uknown word type" +#endif + +#if SIZEOF_VOID_P == 4 +# define ASCII_CHECK_MASK ((AsciiWord)0x80808080U) +#elif SIZEOF_VOID_P == 8 +# define ASCII_CHECK_MASK ((AsciiWord)0x8080808080808080U) +#endif + +static int ascii_fast_track(char* dst, const char* src, int slen, int destlen) +{ + const AsciiWord* src_word = (AsciiWord*) src; + const AsciiWord* const src_word_end = src_word + (slen / sizeof(AsciiWord)); + + if (destlen < slen) + return 0; + + if (dst) { + AsciiWord* dst_word = (AsciiWord*)dst; + + while (src_word < src_word_end) { + if ((*src_word & ASCII_CHECK_MASK) != 0) + break; + *dst_word++ = *src_word++; + } + } + else { + while (src_word < src_word_end) { + if ((*src_word & ASCII_CHECK_MASK) != 0) + break; + src_word++; + } + } + return (char*)src_word - src; +} +#endif /* HAVE_UNALIGNED_WORD_ACCESS */ + int utf8_to_latin1(char* dst, const char* src, int slen, int destlen, erlang_char_encoding* res_encp) { @@ -99,6 +144,15 @@ int utf8_to_latin1(char* dst, const char* src, int slen, int destlen, const char* const dst_end = dst + destlen; int found_non_ascii = 0; +#ifdef HAVE_UNALIGNED_WORD_ACCESS + { + int aft = ascii_fast_track(dst, src, slen, destlen); + src += aft; + slen -= aft; + dst += aft; + } +#endif + while (slen > 0) { if (dst >= dst_end) return -1; if ((src[0] & 0x80) == 0) { @@ -136,6 +190,14 @@ int latin1_to_utf8(char* dst, const char* src, int slen, int destlen, const char* const dst_end = dst + destlen; int found_non_ascii = 0; +#ifdef HAVE_UNALIGNED_WORD_ACCESS + { + int aft = ascii_fast_track(dst, src, slen, destlen); + dst += aft; + src += aft; + } +#endif + while (src < src_end) { if (dst >= dst_end) return -1; if ((src[0] & 0x80) == 0) { diff --git a/lib/erl_interface/src/legacy/erl_marshal.c b/lib/erl_interface/src/legacy/erl_marshal.c index caa171858d..6435754823 100644 --- a/lib/erl_interface/src/legacy/erl_marshal.c +++ b/lib/erl_interface/src/legacy/erl_marshal.c @@ -107,7 +107,7 @@ static int init_cmp_num_class_p=1; /* initialize array, the first time */ void erl_init_marshal(void) { if (init_cmp_array_p) { - memset(cmp_array, 0, CMP_ARRAY_SIZE); + memset(cmp_array, 0, sizeof cmp_array); cmp_array[ERL_SMALL_INTEGER_EXT] = ERL_NUM_CMP; cmp_array[ERL_INTEGER_EXT] = ERL_NUM_CMP; cmp_array[ERL_FLOAT_EXT] = ERL_NUM_CMP; diff --git a/lib/erl_interface/test/ei_tmo_SUITE_data/ei_tmo_test.c b/lib/erl_interface/test/ei_tmo_SUITE_data/ei_tmo_test.c index 0079ef8c86..8e0f2807a4 100644 --- a/lib/erl_interface/test/ei_tmo_SUITE_data/ei_tmo_test.c +++ b/lib/erl_interface/test/ei_tmo_SUITE_data/ei_tmo_test.c @@ -19,6 +19,7 @@ */ #include <stdio.h> +#include <stdlib.h> #include <string.h> #ifdef VXWORKS #include "reclaim.h" diff --git a/lib/erl_interface/vsn.mk b/lib/erl_interface/vsn.mk index d76d110afd..8b6e91757d 100644 --- a/lib/erl_interface/vsn.mk +++ b/lib/erl_interface/vsn.mk @@ -1,2 +1,2 @@ -EI_VSN = 3.10.1 +EI_VSN = 3.10.2 ERL_INTERFACE_VSN = $(EI_VSN) diff --git a/lib/et/doc/src/Makefile b/lib/et/doc/src/Makefile index 0257a8f817..162d36e274 100644 --- a/lib/et/doc/src/Makefile +++ b/lib/et/doc/src/Makefile @@ -45,8 +45,11 @@ include files.mk XML_FILES = $(BOOK_FILES) $(XML_APPLICATION_FILES) $(XML_REF3_FILES) \ $(XML_PART_FILES) $(XML_CHAPTER_FILES) +XML_GEN_FILES = $(GEN_XML:%=$(XMLDIR)/%) + HTML_FILES = $(XML_APPLICATION_FILES:%.xml=$(HTMLDIR)/%.html) \ - $(XML_PART_FILES:%.xml=$(HTMLDIR)/%.html) + $(XML_PART_FILES:%.xml=$(HTMLDIR)/%.html) \ + $(GEN_XML:%.xml=$(HTMLDIR)/%.html) INFO_FILE = ../../info diff --git a/lib/et/doc/src/files.mk b/lib/et/doc/src/files.mk index e0ea9b0b76..7437da7ce3 100644 --- a/lib/et/doc/src/files.mk +++ b/lib/et/doc/src/files.mk @@ -31,10 +31,13 @@ XML_PART_FILES = \ XML_CHAPTER_FILES = \ et_intro.xml \ + notes.xml + +GEN_XML = \ et_tutorial.xml \ et_desc.xml \ - et_examples.xml \ - notes.xml + et_examples.xml + BOOK_FILES = book.xml diff --git a/lib/eunit/doc/src/Makefile b/lib/eunit/doc/src/Makefile index 610e575af6..e91d947592 100644 --- a/lib/eunit/doc/src/Makefile +++ b/lib/eunit/doc/src/Makefile @@ -70,9 +70,10 @@ HTML_STYLESHEET_FILES = \ BOOK_FILES = book.xml XML_FILES = \ - $(BOOK_FILES) $(XML_CHAPTER_FILES) $(XML_NOTES_FILES) \ - $(XML_PART_FILES) $(XML_REF3_FILES) $(XML_APPLICATION_FILES) + $(BOOK_FILES) $(XML_NOTES_FILES) \ + $(XML_PART_FILES) $(XML_APPLICATION_FILES) +XML_GEN_FILES = $(XML_REF3_FILES:%=$(XMLDIR)/%) $(XML_CHAPTER_FILES:%=$(XMLDIR)/%) # ---------------------------------------------------- INFO_FILE = ../../info @@ -122,11 +123,11 @@ man: $(MAN3_FILES) gifs: $(GIF_FILES:%=$(HTMLDIR)/%) -$(XML_REF3_FILES): - escript $(DOCGEN)/priv/bin/xml_from_edoc.escript -def vsn $(EUNIT_VSN) -i $(EUNIT_INC_DIR) $(EUNIT_DIR)/$(@:%.xml=%.erl) +$(XML_REF3_FILES:%=$(XMLDIR)/%): + $(gen_verbose)escript $(DOCGEN)/priv/bin/xml_from_edoc.escript -def vsn $(EUNIT_VSN) -i $(EUNIT_INC_DIR) -dir $(XMLDIR) $(EUNIT_DIR)/$(@:$(XMLDIR)/%.xml=%.erl) -$(XML_CHAPTER_FILES): - escript $(DOCGEN)/priv/bin/xml_from_edoc.escript -def vsn $(EUNIT_VSN) -chapter ../overview.edoc +$(XML_CHAPTER_FILES:%=$(XMLDIR)/%): + $(gen_verbose)escript $(DOCGEN)/priv/bin/xml_from_edoc.escript -def vsn $(EUNIT_VSN) -chapter -dir $(XMLDIR) ../overview.edoc info: @echo "XML_PART_FILES: $(XML_PART_FILES)" diff --git a/lib/ftp/AUTHORS b/lib/ftp/AUTHORS new file mode 100644 index 0000000000..88dcbf602a --- /dev/null +++ b/lib/ftp/AUTHORS @@ -0,0 +1,11 @@ +Original Authors: + +Peter Högfeldt - first version of ftp + +Contributors: + +Ingela Anderton Andin +Martin Gustafsson +Johan Blom +Torbjörn Törnkvist +Joe Armstrong
\ No newline at end of file diff --git a/lib/ftp/Makefile b/lib/ftp/Makefile new file mode 100644 index 0000000000..555f8b0dea --- /dev/null +++ b/lib/ftp/Makefile @@ -0,0 +1,78 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 1996-2016. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# %CopyrightEnd% +# +# +include $(ERL_TOP)/make/target.mk +include $(ERL_TOP)/make/$(TARGET)/otp.mk + +# ---------------------------------------------------- +# Macros +# ---------------------------------------------------- + +SUB_DIRECTORIES = src doc/src + +include vsn.mk +VSN = $(FTP_VSN) + +SPECIAL_TARGETS = + +DIA_PLT = ./priv/plt/$(APPLICATION).plt +DIA_ANALYSIS = $(basename $(DIA_PLT)).dialyzer_analysis + + +# ---------------------------------------------------- +# Default Subdir Targets +# ---------------------------------------------------- +include $(ERL_TOP)/make/otp_subdir.mk + +.PHONY: info gclean dialyzer dialyzer_plt dclean + +info: + @echo "OS: $(OS)" + @echo "DOCB: $(DOCB)" + @echo "" + @echo "FTP_VSN: $(FTP_VSN)" + @echo "APP_VSN: $(APP_VSN)" + @echo "" + @echo "DIA_PLT: $(DIA_PLT)" + @echo "DIA_ANALYSIS: $(DIA_ANALYSIS)" + @echo "" + +gclean: + git clean -fXd + +dclean: + rm -f $(DIA_PLT) + rm -f $(DIA_ANALYSIS) + +dialyzer_plt: $(DIA_PLT) + +$(DIA_PLT): + @echo "Building $(APPLICATION) plt file" + @dialyzer --build_plt \ + --output_plt $@ \ + -r ../$(APPLICATION)/ebin \ + --output $(DIA_ANALYSIS) \ + --verbose + +dialyzer: $(DIA_PLT) + @echo "Running dialyzer on $(APPLICATION)" + @dialyzer --plt $< \ + ../$(APPLICATION)/ebin \ + --verbose diff --git a/lib/inets/doc/archive/rfc2428.txt b/lib/ftp/doc/archive/rfc2428.txt index a6ec3535ed..a6ec3535ed 100644 --- a/lib/inets/doc/archive/rfc2428.txt +++ b/lib/ftp/doc/archive/rfc2428.txt diff --git a/lib/inets/doc/archive/rfc2577.txt b/lib/ftp/doc/archive/rfc2577.txt index 83ba203130..83ba203130 100644 --- a/lib/inets/doc/archive/rfc2577.txt +++ b/lib/ftp/doc/archive/rfc2577.txt diff --git a/lib/inets/doc/archive/rfc959.txt b/lib/ftp/doc/archive/rfc959.txt index 5c9f11af5d..5c9f11af5d 100644 --- a/lib/inets/doc/archive/rfc959.txt +++ b/lib/ftp/doc/archive/rfc959.txt diff --git a/lib/ftp/doc/html/.gitignore b/lib/ftp/doc/html/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/ftp/doc/html/.gitignore diff --git a/lib/ftp/doc/man3/.gitignore b/lib/ftp/doc/man3/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/ftp/doc/man3/.gitignore diff --git a/lib/ftp/doc/man6/.gitignore b/lib/ftp/doc/man6/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/ftp/doc/man6/.gitignore diff --git a/lib/ftp/doc/pdf/.gitignore b/lib/ftp/doc/pdf/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/ftp/doc/pdf/.gitignore diff --git a/lib/ftp/doc/src/Makefile b/lib/ftp/doc/src/Makefile new file mode 100644 index 0000000000..e96a9c032f --- /dev/null +++ b/lib/ftp/doc/src/Makefile @@ -0,0 +1,154 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 1997-2018. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# %CopyrightEnd% +# +# +include $(ERL_TOP)/make/target.mk +include $(ERL_TOP)/make/$(TARGET)/otp.mk + +# ---------------------------------------------------- +# Application version +# ---------------------------------------------------- +include ../../vsn.mk +VSN=$(FTP_VSN) + +# ---------------------------------------------------- +# Release directory specification +# ---------------------------------------------------- +RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN) + + +# ---------------------------------------------------- +# Target Specs +# ---------------------------------------------------- +XML_APPLICATION_FILES = ref_man.xml + +XML_CHAPTER_FILES = \ + introduction.xml \ + ftp_client.xml \ + notes.xml + +XML_REF3_FILES = \ + ftp.xml + +XML_PART_FILES = \ + part.xml + +BOOK_FILES = book.xml + +XML_FILES = \ + $(BOOK_FILES) \ + $(XML_CHAPTER_FILES) \ + $(XML_PART_FILES) \ + $(XML_REF6_FILES) \ + $(XML_REF3_FILES) \ + $(XML_APPLICATION_FILES) + +# GIF_FILES = ftp.gif + + +# ---------------------------------------------------- + +HTML_FILES = \ + $(XML_APPLICATION_FILES:%.xml=$(HTMLDIR)/%.html) \ + $(XML_PART_FILES:%.xml=$(HTMLDIR)/%.html) + +INFO_FILE = ../../info +EXTRA_FILES = \ + $(XML_REF3_FILES:%.xml=$(HTMLDIR)/%.html) \ + $(XML_REF6_FILES:%.xml=$(HTMLDIR)/%.html) \ + $(XML_CHAPTER_FILES:%.xml=$(HTMLDIR)/%.html) + +MAN3_FILES = $(XML_REF3_FILES:%.xml=$(MAN3DIR)/%.3) + +HTML_REF_MAN_FILE = $(HTMLDIR)/index.html + +TOP_PDF_FILE = $(PDFDIR)/$(APPLICATION)-$(VSN).pdf + +# ---------------------------------------------------- +# FLAGS +# ---------------------------------------------------- +XML_FLAGS += +DVIPS_FLAGS += + +# ---------------------------------------------------- +# Targets +# ---------------------------------------------------- +$(HTMLDIR)/%.gif: %.gif + $(INSTALL_DATA) $< $@ + +docs: pdf html man + +ldocs: local_docs + +$(TOP_PDF_FILE): $(XML_FILES) + +pdf: $(TOP_PDF_FILE) + +html: gifs $(HTML_REF_MAN_FILE) + +clean clean_docs: clean_html clean_man clean_pdf + rm -f errs core *~ + +man: $(MAN3_FILES) + +gifs: $(GIF_FILES:%=$(HTMLDIR)/%) + +debug opt: + +clean_pdf: + rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) + +clean_html: + rm -rf $(TOP_HTML_FILES) $(HTMLDIR)/* + +clean_man: + rm -f $(MAN3_FILES) + + +# ---------------------------------------------------- +# Release Target +# ---------------------------------------------------- +include $(ERL_TOP)/make/otp_release_targets.mk + +release_docs_spec: docs + $(INSTALL_DIR) "$(RELSYSDIR)/doc/pdf" + $(INSTALL_DATA) $(TOP_PDF_FILE) "$(RELSYSDIR)/doc/pdf" + $(INSTALL_DIR) "$(RELSYSDIR)/doc/html" + $(INSTALL_DATA) $(HTMLDIR)/* "$(RELSYSDIR)/doc/html" + $(INSTALL_DATA) $(INFO_FILE) "$(RELSYSDIR)" + $(INSTALL_DIR) "$(RELEASE_PATH)/man/man3" + $(INSTALL_DATA) $(MAN3DIR)/* "$(RELEASE_PATH)/man/man3" + +release_spec: + +info: + @echo "GIF_FILES:\n$(GIF_FILES)" + @echo "" + @echo "EXTRA_FILES:\n$(EXTRA_FILES)" + @echo "" + @echo "HTML_FILES:\n$(HTML_FILES)" + @echo "" + @echo "TOP_HTML_FILES:\n$(TOP_HTML_FILES)" + @echo "" + @echo "XML_REF3_FILES:\n$(XML_REF3_FILES)" + @echo "" + @echo "XML_REF6_FILES:\n$(XML_REF6_FILES)" + @echo "" + @echo "XML_CHAPTER_FILES:\n$(XML_CHAPTER_FILES)" + @echo "" diff --git a/lib/ftp/doc/src/book.xml b/lib/ftp/doc/src/book.xml new file mode 100644 index 0000000000..1268af64bf --- /dev/null +++ b/lib/ftp/doc/src/book.xml @@ -0,0 +1,49 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE book SYSTEM "book.dtd"> + +<book xmlns:xi="http://www.w3.org/2001/XInclude"> + <header titlestyle="normal"> + <copyright> + <year>1997</year><year>2018</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>FTP</title> + <prepared>Péter Dimitrov</prepared> + <docno></docno> + <date>2018-02-26</date> + <rev>1.0</rev> + <file>book.sgml</file> + </header> + <insidecover> + </insidecover> + <pagetext>FTP</pagetext> + <preamble> + <contents level="2"></contents> + </preamble> + <parts lift="no"> + <xi:include href="part.xml"/> + </parts> + <applications> + <xi:include href="ref_man.xml"/> + </applications> + <releasenotes> + <xi:include href="notes.xml"/> + </releasenotes> + <listofterms></listofterms> + <index></index> +</book> diff --git a/lib/inets/doc/src/ftp.xml b/lib/ftp/doc/src/ftp.xml index 42bece4d38..18770ebcb4 100644 --- a/lib/inets/doc/src/ftp.xml +++ b/lib/ftp/doc/src/ftp.xml @@ -38,20 +38,19 @@ according to a subset of the File Transfer Protocol (FTP), see <url href="http://www.ietf.org/rfc/rfc959.txt">RFC 959</url>.</p> - <p>As from <c>Inets</c> 4.4.1, the FTP - client always tries to use passive FTP mode and only resort + <p>The FTP client always tries to use passive FTP mode and only resort to active FTP mode if this fails. This default behavior can be changed by start option <seealso marker="#mode">mode</seealso>.</p> <marker id="two_start"></marker> <p>An FTP client can be started in two ways. One is using the - <seealso marker="#service_start">Inets service framework</seealso>, - the other is to start it directly as a standalone process + <seealso marker="#service_start">service_start</seealso> function, + the other is to start it directly as a standalone process using function <seealso marker="#open">open</seealso>.</p> <p>For a simple example of an FTP session, see - <seealso marker="ftp_client">Inets User's Guide</seealso>.</p> + <seealso marker="ftp_client">FTP User's Guide</seealso>.</p> <p>In addition to the ordinary functions for receiving and sending files (see <c>recv/2</c>, <c>recv/3</c>, <c>send/2</c>, and @@ -82,11 +81,9 @@ <title>FTP CLIENT SERVICE START/STOP</title> <p>The FTP client can be started and stopped dynamically in runtime by - calling the <c>Inets</c> application API - <c>inets:start(ftpc, ServiceConfig)</c>, - or <c>inets:start(ftpc, ServiceConfig, How)</c>, and - <c>inets:stop(ftpc, Pid)</c>. - For details, see <seealso marker="inets">inets(3)</seealso>.</p> + calling the <c>ftp</c> application API + <c>ftp:start_service(ServiceConfig)</c> and + <c>ftp:stop_service(Pid)</c>.</p> <p>The available configuration options are as follows:</p> @@ -273,6 +270,7 @@ </section> <funcs> + <func> <name>account(Pid, Account) -> ok | {error, Reason}</name> <fsummary>Specifies which account to use.</fsummary> @@ -564,7 +562,7 @@ <desc> <p>Starts a standalone FTP client process - (without the <c>Inets</c> service framework) and + (without the <c>ftp</c> service framework) and opens a session with the FTP server at <c>Host</c>. </p> <p>If option <c>{tls, tls_options()}</c> is present, the FTP session @@ -797,6 +795,37 @@ </func> <func> + <name>start_service(ServiceConfig) -> {ok, Pid} | {error, Reason}</name> + <fsummary>Dynamically starts an <c>FTP</c> + session after the <c>ftp</c> application has been started.</fsummary> + <type> + <v>ServiceConfig = [{Option, Value}]</v> + <v>Option = property()</v> + <v>Value = term()</v> + </type> + <desc> + <p>Dynamically starts an <c>FTP</c> session after the <c>ftp</c> + application has been started.</p> + <note> + <p>As long as the <c>ftp</c> application is operational, + the FTP sessions are supervised and can be soft code upgraded.</p> + </note> + </desc> + </func> + + <func> + <name>stop_service(Reference) -> ok | {error, Reason} </name> + <fsummary>Stops an FTP session.</fsummary> + <type> + <v>Reference = pid() | term() - service-specified reference</v> + <v>Reason = term()</v> + </type> + <desc> + <p>Stops a started FTP session.</p> + </desc> + </func> + + <func> <name>type(Pid, Type) -> ok | {error, Reason}</name> <fsummary>Sets transfer type to <c>ascii</c>or <c>binary</c>.</fsummary> <type> diff --git a/lib/inets/doc/src/ftp_client.xml b/lib/ftp/doc/src/ftp_client.xml index 990dd68604..047b055be7 100644 --- a/lib/inets/doc/src/ftp_client.xml +++ b/lib/ftp/doc/src/ftp_client.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2016</year> + <year>2004</year><year>2018</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -23,7 +23,7 @@ </legalnotice> <title>FTP Client</title> - <prepared>Ingela Anderton Andin</prepared> + <prepared>Péter Dimitrov</prepared> <responsible></responsible> <docno></docno> <approved></approved> @@ -53,9 +53,9 @@ the user <c>guest</c> with password <c>password</c> logs on to the remote host <c>erlang.org</c>:</p> <code type="erl"><![CDATA[ - 1> inets:start(). + 1> ftp:start(). ok - 2> {ok, Pid} = inets:start(ftpc, [{host, "erlang.org"}]). + 2> {ok, Pid} = ftp:start_service([{host, "erlang.org"}]). {ok,<0.22.0>} 3> ftp:user(Pid, "guest", "password"). ok @@ -69,7 +69,9 @@ ok 8> ftp:recv(Pid, "appl.erl"). ok - 9> inets:stop(ftpc, Pid). + 9> ftp:stop_service(Pid). + ok + 10> ftp:stop(). ok ]]></code> <p> The file @@ -82,5 +84,3 @@ <c>/home/guest/appl/examples</c>.</p> </section> </chapter> - - diff --git a/lib/ftp/doc/src/introduction.xml b/lib/ftp/doc/src/introduction.xml new file mode 100644 index 0000000000..cc3673a0fc --- /dev/null +++ b/lib/ftp/doc/src/introduction.xml @@ -0,0 +1,46 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>1997</year><year>2018</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + </legalnotice> + + <title>Introduction</title> + <prepared>Péter Dimitrov</prepared> + <responsible></responsible> + <docno></docno> + <approved></approved> + <checked></checked> + <date>2018-02-26</date> + <rev>A</rev> + <file>introduction.xml</file> + </header> + + <section> + <title>Purpose</title> + <p>An <c>FTP</c> client.</p> + </section> + + <section> + <title>Prerequisites</title> + <p>It is assumed that the reader is familiar with the Erlang + programming language, concepts of OTP, and has a basic + understanding of the FTP protocol.</p> + </section> +</chapter> diff --git a/lib/ftp/doc/src/notes.xml b/lib/ftp/doc/src/notes.xml new file mode 100644 index 0000000000..50f38941e5 --- /dev/null +++ b/lib/ftp/doc/src/notes.xml @@ -0,0 +1,53 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>2002</year><year>2018</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>FTP Release Notes</title> + <prepared></prepared> + <responsible></responsible> + <docno></docno> + <approved></approved> + <checked></checked> + <date>2018-02-26</date> + <rev>A</rev> + <file>notes.xml</file> + </header> + + <section><title>FTP 1.0</title> + + <section><title>First released version</title> + <list> + <item> + <p> + Inets application was split into multiple smaller protocol specific applications. + The FTP application is a standalone FTP client with the same functionality as + FTP client in Inets.</p> + <p> + Own Id: OTP-14113</p> + </item> + </list> + </section> + +</section> + +</chapter> diff --git a/lib/ftp/doc/src/part.xml b/lib/ftp/doc/src/part.xml new file mode 100644 index 0000000000..ec05f5ac76 --- /dev/null +++ b/lib/ftp/doc/src/part.xml @@ -0,0 +1,37 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE part SYSTEM "part.dtd"> + +<part xmlns:xi="http://www.w3.org/2001/XInclude"> + <header> + <copyright> + <year>2004</year><year>2018</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>FTP User's Guide</title> + <prepared>Péter Dimitrov</prepared> + <docno></docno> + <date>2018-02-26</date> + <rev>A</rev> + <file>part.sgml</file> + </header> + <description> + <p>The <c>FTP</c> application provides an FTP client.</p> + </description> + <xi:include href="introduction.xml"/> + <xi:include href="ftp_client.xml"/> +</part> diff --git a/lib/ftp/doc/src/ref_man.xml b/lib/ftp/doc/src/ref_man.xml new file mode 100644 index 0000000000..925842610d --- /dev/null +++ b/lib/ftp/doc/src/ref_man.xml @@ -0,0 +1,36 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE application SYSTEM "application.dtd"> + +<application xmlns:xi="http://www.w3.org/2001/XInclude"> + <header> + <copyright> + <year>1997</year><year>2018</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>FTP Reference Manual</title> + <prepared>Péter Dimitrov</prepared> + <docno></docno> + <date>2018-03-09</date> + <rev>1.0</rev> + <file>ref_man.xml</file> + </header> + <description> + <p>An <c>FTP</c> client.</p> + </description> + <xi:include href="ftp.xml"/> +</application> diff --git a/lib/ftp/ebin/.gitignore b/lib/ftp/ebin/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/ftp/ebin/.gitignore diff --git a/lib/ftp/info b/lib/ftp/info new file mode 100644 index 0000000000..f6c62d19c2 --- /dev/null +++ b/lib/ftp/info @@ -0,0 +1,2 @@ +group: comm +short: FTP client diff --git a/lib/inets/src/ftp/Makefile b/lib/ftp/src/Makefile index 6b99694ea7..6a6df6bde4 100644 --- a/lib/inets/src/ftp/Makefile +++ b/lib/ftp/src/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2005-2016. All Rights Reserved. +# Copyright Ericsson AB 1999-2017. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,88 +17,102 @@ # # %CopyrightEnd% # + # include $(ERL_TOP)/make/target.mk -EBIN = ../../ebin include $(ERL_TOP)/make/$(TARGET)/otp.mk - # ---------------------------------------------------- # Application version # ---------------------------------------------------- -include ../../vsn.mk - -VSN = $(INETS_VSN) - +include ../vsn.mk +VSN=$(FTP_VSN) # ---------------------------------------------------- # Release directory specification # ---------------------------------------------------- -RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN) - +RELSYSDIR = $(RELEASE_PATH)/lib/ftp-$(VSN) # ---------------------------------------------------- -# Target Specs +# Common Macros # ---------------------------------------------------- -MODULES = \ + +BEHAVIOUR_MODULES= + +MODULES= \ ftp \ + ftp_app \ ftp_progress \ ftp_response \ - ftp_sup + ftp_sup + + +INTERNAL_HRL_FILES = -HRL_FILES = ftp_internal.hrl +ERL_FILES= \ + $(MODULES:%=%.erl) \ + $(BEHAVIOUR_MODULES:%=%.erl) -ERL_FILES = $(MODULES:%=%.erl) TARGET_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) +BEHAVIOUR_TARGET_FILES= $(BEHAVIOUR_MODULES:%=$(EBIN)/%.$(EMULATOR)) + +APP_FILE= ftp.app +APPUP_FILE= ftp.appup + +APP_SRC= $(APP_FILE).src +APP_TARGET= $(EBIN)/$(APP_FILE) +APPUP_SRC= $(APPUP_FILE).src +APPUP_TARGET= $(EBIN)/$(APPUP_FILE) # ---------------------------------------------------- # FLAGS # ---------------------------------------------------- - -include ../inets_app/inets.mk - -ifeq ($(FTP_DEBUG),true) - INETS_FLAGS += -Dftp_debug -endif - -ERL_COMPILE_FLAGS += \ - $(INETS_FLAGS) \ - $(INETS_ERL_COMPILE_FLAGS) \ - -I../../include \ - -I../inets_app +EXTRA_ERLC_FLAGS = +warn_unused_vars +ERL_COMPILE_FLAGS += -I$(ERL_TOP)/lib/kernel/src \ + -pz $(EBIN) \ + -pz $(ERL_TOP)/lib/public_key/ebin \ + $(EXTRA_ERLC_FLAGS) -DVSN=\"$(VSN)\" # ---------------------------------------------------- # Targets # ---------------------------------------------------- -debug opt: $(TARGET_FILES) +$(TARGET_FILES): $(BEHAVIOUR_TARGET_FILES) + +debug opt: $(TARGET_FILES) $(APP_TARGET) $(APPUP_TARGET) clean: - rm -f $(TARGET_FILES) - rm -f core + rm -f $(TARGET_FILES) $(APP_TARGET) $(APPUP_TARGET) $(BEHAVIOUR_TARGET_FILES) + rm -f errs core *~ + +$(APP_TARGET): $(APP_SRC) ../vsn.mk + $(vsn_verbose)sed -e 's;%VSN%;$(VSN);' $< > $@ + +$(APPUP_TARGET): $(APPUP_SRC) ../vsn.mk + $(vsn_verbose)sed -e 's;%VSN%;$(VSN);' $< > $@ docs: + # ---------------------------------------------------- # Release Target # ---------------------------------------------------- include $(ERL_TOP)/make/otp_release_targets.mk release_spec: opt - $(INSTALL_DIR) "$(RELSYSDIR)/src" - $(INSTALL_DIR) "$(RELSYSDIR)/src/ftp" - $(INSTALL_DATA) $(HRL_FILES) $(ERL_FILES) "$(RELSYSDIR)/src/ftp" - $(INSTALL_DIR) "$(RELSYSDIR)/ebin" - $(INSTALL_DATA) $(TARGET_FILES) "$(RELSYSDIR)/ebin" + $(INSTALL_DIR) "$(RELSYSDIR)/src" + $(INSTALL_DATA) $(ERL_FILES) $(INTERNAL_HRL_FILES) "$(RELSYSDIR)/src" + $(INSTALL_DIR) "$(RELSYSDIR)/ebin" + $(INSTALL_DATA) $(BEHAVIOUR_TARGET_FILES) $(TARGET_FILES) $(APP_TARGET) \ + $(APPUP_TARGET) "$(RELSYSDIR)/ebin" release_docs_spec: -info: - @echo "APPLICATION = $(APPLICATION)" - @echo "INETS_DEBUG = $(INETS_DEBUG)" - @echo "INETS_FLAGS = $(INETS_FLAGS)" - @echo "ERL_COMPILE_FLAGS = $(ERL_COMPILE_FLAGS)" +# ---------------------------------------------------- +# Dependencies +# ---------------------------------------------------- + diff --git a/lib/ftp/src/ftp.app.src b/lib/ftp/src/ftp.app.src new file mode 100644 index 0000000000..237174358f --- /dev/null +++ b/lib/ftp/src/ftp.app.src @@ -0,0 +1,19 @@ +{application, ftp, + [{description, "FTP client"}, + {vsn, "1.0"}, + {registered, []}, + {mod, { ftp_app, []}}, + {applications, + [kernel, + stdlib + ]}, + {env,[]}, + {modules, [ + ftp, + ftp_app, + ftp_progress, + ftp_response, + ftp_sup + ]}, + {runtime_dependencies, ["erts-7.0","stdlib-3.5","kernel-6.0"]} + ]}. diff --git a/lib/ssh/src/ssh_dbg.hrl b/lib/ftp/src/ftp.appup.src index e94664737b..d79c7b60ff 100644 --- a/lib/ssh/src/ssh_dbg.hrl +++ b/lib/ftp/src/ftp.appup.src @@ -1,7 +1,7 @@ -%% +%% -*- erlang -*- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2016. All Rights Reserved. +%% Copyright Ericsson AB 1999-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -16,12 +16,11 @@ %% limitations under the License. %% %% %CopyrightEnd% -%% - --ifndef(SSH_DBG_HRL). --define(SSH_DBG_HRL, 1). - --define(formatrec(RecName,R), - ssh_dbg:wr_record(R, record_info(fields,RecName), [])). - --endif. % SSH_DBG_HRL defined +{"%VSN%", + [ + {<<".*">>,[{restart_application, ftp}]} + ], + [ + {<<".*">>,[{restart_application, ftp}]} + ] +}. diff --git a/lib/inets/src/ftp/ftp.erl b/lib/ftp/src/ftp.erl index e0430654eb..8790bfec13 100644 --- a/lib/inets/src/ftp/ftp.erl +++ b/lib/ftp/src/ftp.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2017. All Rights Reserved. +%% Copyright Ericsson AB 2002-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -18,14 +18,23 @@ %% %CopyrightEnd% %% %% -%% Description: This module implements an ftp client, RFC 959. -%% It also supports ipv6 RFC 2428 and starttls RFC 4217. -module(ftp). -behaviour(gen_server). --behaviour(inets_service). +-export([start/0, + start_service/1, + stop/0, + stop_service/1, + services/0, + service_info/1 + ]). + +%% Added for backward compatibility +-export([start_standalone/1]). + +-export([start_link/1, start_link/2]). %% API - Client interface -export([cd/2, close/1, delete/2, formaterror/1, @@ -47,13 +56,6 @@ -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -%% supervisor callbacks --export([start_link/1, start_link/2]). - -%% Behavior callbacks --export([start_standalone/1, start_service/1, - stop_service/1, services/0, service_info/1]). - -include("ftp_internal.hrl"). %% Constants used in internal state definition @@ -67,6 +69,11 @@ -define(FTP_PORT, 21). -define(FILE_BUFSIZE, 4096). + +%%%========================================================================= +%%% Data Types +%%%========================================================================= + %% Internal state -record(state, { csock = undefined, % socket() - Control connection socket @@ -116,6 +123,61 @@ %%-define(DBG(F,A), io:format(F,A)). %%-define(DBG(F,A), ct:pal("~p:~p " ++ if is_list(F) -> F; is_atom(F) -> atom_to_list(F) end, [?MODULE,?LINE|A])). + +%%%========================================================================= +%%% API +%%%========================================================================= + +start() -> + application:start(ftp). + +start_standalone(Options) -> + try + {ok, StartOptions} = start_options(Options), + {ok, OpenOptions} = open_options(Options), + case start_link(StartOptions, []) of + {ok, Pid} -> + call(Pid, {open, ip_comm, OpenOptions}, plain); + Error1 -> + Error1 + end + catch + throw:Error2 -> + Error2 + end. + +start_service(Options) -> + try + {ok, StartOptions} = start_options(Options), + {ok, OpenOptions} = open_options(Options), + case ftp_sup:start_child([[[{client, self()} | StartOptions], []]]) of + {ok, Pid} -> + call(Pid, {open, ip_comm, OpenOptions}, plain); + Error1 -> + Error1 + end + catch + throw:Error2 -> + Error2 + end. + +stop() -> + application:stop(ftp). + +stop_service(Pid) -> + close(Pid). + +services() -> + [{ftpc, Pid} || {_, Pid, _, _} <- + supervisor:which_children(ftp_sup)]. +service_info(Pid) -> + {ok, Info} = call(Pid, info, list), + {ok, [proplists:lookup(mode, Info), + proplists:lookup(local_port, Info), + proplists:lookup(peer, Info), + proplists:lookup(peer_port, Info)]}. + + %%%========================================================================= %%% API - CLIENT FUNCTIONS %%%========================================================================= @@ -162,22 +224,17 @@ open(Host, Port) when is_integer(Port) -> %% </BACKWARD-COMPATIBILLITY> open(Host, Opts) when is_list(Opts) -> - ?fcrt("open", [{host, Host}, {opts, Opts}]), try {ok, StartOptions} = start_options(Opts), - ?fcrt("open", [{start_options, StartOptions}]), {ok, OpenOptions} = open_options([{host, Host}|Opts]), - ?fcrt("open", [{open_options, OpenOptions}]), case start_link(StartOptions, []) of {ok, Pid} -> do_open(Pid, OpenOptions, tls_options(Opts)); Error1 -> - ?fcrt("open - error", [{error1, Error1}]), Error1 end catch throw:Error2 -> - ?fcrt("open - error", [{error2, Error2}]), Error2 end. @@ -872,219 +929,6 @@ info(Pid) -> latest_ctrl_response(Pid) -> call(Pid, latest_ctrl_response, string). -%%%======================================================================== -%%% Behavior callbacks -%%%======================================================================== -start_standalone(Options) -> - try - {ok, StartOptions} = start_options(Options), - {ok, OpenOptions} = open_options(Options), - case start_link(StartOptions, []) of - {ok, Pid} -> - call(Pid, {open, ip_comm, OpenOptions}, plain); - Error1 -> - Error1 - end - catch - throw:Error2 -> - Error2 - end. - -start_service(Options) -> - try - {ok, StartOptions} = start_options(Options), - {ok, OpenOptions} = open_options(Options), - case ftp_sup:start_child([[[{client, self()} | StartOptions], []]]) of - {ok, Pid} -> - call(Pid, {open, ip_comm, OpenOptions}, plain); - Error1 -> - Error1 - end - catch - throw:Error2 -> - Error2 - end. - -stop_service(Pid) -> - close(Pid). - -services() -> - [{ftpc, Pid} || {_, Pid, _, _} <- - supervisor:which_children(ftp_sup)]. -service_info(Pid) -> - {ok, Info} = call(Pid, info, list), - {ok, [proplists:lookup(mode, Info), - proplists:lookup(local_port, Info), - proplists:lookup(peer, Info), - proplists:lookup(peer_port, Info)]}. - - -%% This function extracts the start options from the -%% Valid options: -%% debug, -%% verbose -%% ipfamily -%% priority -%% flags (for backward compatibillity) -start_options(Options) -> - ?fcrt("start_options", [{options, Options}]), - case lists:keysearch(flags, 1, Options) of - {value, {flags, Flags}} -> - Verbose = lists:member(verbose, Flags), - IsTrace = lists:member(trace, Flags), - IsDebug = lists:member(debug, Flags), - DebugLevel = - if - (IsTrace =:= true) -> - trace; - IsDebug =:= true -> - debug; - true -> - disable - end, - {ok, [{verbose, Verbose}, - {debug, DebugLevel}, - {priority, low}]}; - false -> - ValidateVerbose = - fun(true) -> true; - (false) -> true; - (_) -> false - end, - ValidateDebug = - fun(trace) -> true; - (debug) -> true; - (disable) -> true; - (_) -> false - end, - ValidatePriority = - fun(low) -> true; - (normal) -> true; - (high) -> true; - (_) -> false - end, - ValidOptions = - [{verbose, ValidateVerbose, false, false}, - {debug, ValidateDebug, false, disable}, - {priority, ValidatePriority, false, low}], - validate_options(Options, ValidOptions, []) - end. - - -%% This function extracts and validates the open options from the -%% Valid options: -%% mode -%% host -%% port -%% timeout -%% dtimeout -%% progress -%% ftp_extension - -open_options(Options) -> - ?fcrt("open_options", [{options, Options}]), - ValidateMode = - fun(active) -> true; - (passive) -> true; - (_) -> false - end, - ValidateHost = - fun(Host) when is_list(Host) -> - true; - (Host) when is_tuple(Host) andalso - ((size(Host) =:= 4) orelse (size(Host) =:= 8)) -> - true; - (_) -> - false - end, - ValidatePort = - fun(Port) when is_integer(Port) andalso (Port > 0) -> true; - (_) -> false - end, - ValidateIpFamily = - fun(inet) -> true; - (inet6) -> true; - (inet6fb4) -> true; - (_) -> false - end, - ValidateTimeout = - fun(Timeout) when is_integer(Timeout) andalso (Timeout >= 0) -> true; - (_) -> false - end, - ValidateDTimeout = - fun(DTimeout) when is_integer(DTimeout) andalso (DTimeout >= 0) -> true; - (infinity) -> true; - (_) -> false - end, - ValidateProgress = - fun(ignore) -> - true; - ({Mod, Func, _InitProgress}) when is_atom(Mod) andalso - is_atom(Func) -> - true; - (_) -> - false - end, - ValidateFtpExtension = - fun(true) -> true; - (false) -> true; - (_) -> false - end, - ValidOptions = - [{mode, ValidateMode, false, ?DEFAULT_MODE}, - {host, ValidateHost, true, ehost}, - {port, ValidatePort, false, ?FTP_PORT}, - {ipfamily, ValidateIpFamily, false, inet}, - {timeout, ValidateTimeout, false, ?CONNECTION_TIMEOUT}, - {dtimeout, ValidateDTimeout, false, ?DATA_ACCEPT_TIMEOUT}, - {progress, ValidateProgress, false, ?PROGRESS_DEFAULT}, - {ftp_extension, ValidateFtpExtension, false, ?FTP_EXT_DEFAULT}], - validate_options(Options, ValidOptions, []). - -tls_options(Options) -> - %% Options will be validated by ssl application - proplists:get_value(tls, Options, undefined). - -validate_options([], [], Acc) -> - ?fcrt("validate_options -> done", [{acc, Acc}]), - {ok, lists:reverse(Acc)}; -validate_options([], ValidOptions, Acc) -> - ?fcrt("validate_options -> done", - [{valid_options, ValidOptions}, {acc, Acc}]), - %% Check if any mandatory options are missing! - case [{Key, Reason} || {Key, _, true, Reason} <- ValidOptions] of - [] -> - Defaults = - [{Key, Default} || {Key, _, _, Default} <- ValidOptions], - {ok, lists:reverse(Defaults ++ Acc)}; - [{_, Reason}|_Missing] -> - throw({error, Reason}) - end; -validate_options([{Key, Value}|Options], ValidOptions, Acc) -> - ?fcrt("validate_options -> check", - [{key, Key}, {value, Value}, {acc, Acc}]), - case lists:keysearch(Key, 1, ValidOptions) of - {value, {Key, Validate, _, Default}} -> - case (catch Validate(Value)) of - true -> - ?fcrt("validate_options -> check - accept", []), - NewValidOptions = lists:keydelete(Key, 1, ValidOptions), - validate_options(Options, NewValidOptions, - [{Key, Value} | Acc]); - _ -> - ?fcrt("validate_options -> check - reject", - [{default, Default}]), - NewValidOptions = lists:keydelete(Key, 1, ValidOptions), - validate_options(Options, NewValidOptions, - [{Key, Default} | Acc]) - end; - false -> - validate_options(Options, ValidOptions, Acc) - end; -validate_options([_|Options], ValidOptions, Acc) -> - validate_options(Options, ValidOptions, Acc). - - %%%======================================================================== %%% gen_server callback functions @@ -1183,7 +1027,6 @@ handle_call({Pid, _}, _, #state{owner = Owner} = State) when Owner =/= Pid -> {reply, {error, not_connection_owner}, State}; handle_call({_, {open, ip_comm, Opts}}, From, State) -> - ?fcrd("handle_call(open)", [{opts, Opts}]), case key_search(host, Opts, undefined) of undefined -> {stop, normal, {error, ehost}, State}; @@ -1203,16 +1046,10 @@ handle_call({_, {open, ip_comm, Opts}}, From, State) -> dtimeout = DTimeout, ftp_extension = FtpExt}, - ?fcrd("handle_call(open) -> setup ctrl connection with", - [{host, Host}, {port, Port}, {timeout, Timeout}]), case setup_ctrl_connection(Host, Port, Timeout, State2) of {ok, State3, WaitTimeout} -> - ?fcrd("handle_call(open) -> ctrl connection setup done", - [{waittimeout, WaitTimeout}]), {noreply, State3, WaitTimeout}; - {error, Reason} -> - ?fcrd("handle_call(open) -> ctrl connection setup failed", - [{reason, Reason}]), + {error, _Reason} -> gen_server:reply(From, {error, ehost}), {stop, normal, State2#state{client = undefined}} end @@ -1241,7 +1078,7 @@ handle_call({_, {open, ip_comm, Host, Opts}}, From, State) -> end; handle_call({_, {open, tls_upgrade, TLSOptions}}, From, State) -> - send_ctrl_message(State, mk_cmd("AUTH TLS", [])), + _ = send_ctrl_message(State, mk_cmd("AUTH TLS", [])), activate_ctrl_connection(State), {noreply, State#state{client = From, caller = open, tls_options = TLSOptions}}; @@ -1257,7 +1094,7 @@ handle_call({_, {account, Acc}}, From, State)-> handle_user_account(Acc, State#state{client = From}); handle_call({_, pwd}, From, #state{chunk = false} = State) -> - send_ctrl_message(State, mk_cmd("PWD", [])), + _ = send_ctrl_message(State, mk_cmd("PWD", [])), activate_ctrl_connection(State), {noreply, State#state{client = From, caller = pwd}}; @@ -1265,7 +1102,7 @@ handle_call({_, lpwd}, From, #state{ldir = LDir} = State) -> {reply, {ok, LDir}, State#state{client = From}}; handle_call({_, {cd, Dir}}, From, #state{chunk = false} = State) -> - send_ctrl_message(State, mk_cmd("CWD ~s", [Dir])), + _ = send_ctrl_message(State, mk_cmd("CWD ~s", [Dir])), activate_ctrl_connection(State), {noreply, State#state{client = From, caller = cd}}; @@ -1284,35 +1121,35 @@ handle_call({_, {dir, Len, Dir}}, {_Pid, _} = From, client = From}); handle_call({_, {rename, CurrFile, NewFile}}, From, #state{chunk = false} = State) -> - send_ctrl_message(State, mk_cmd("RNFR ~s", [CurrFile])), + _ = send_ctrl_message(State, mk_cmd("RNFR ~s", [CurrFile])), activate_ctrl_connection(State), {noreply, State#state{caller = {rename, NewFile}, client = From}}; handle_call({_, {delete, File}}, {_Pid, _} = From, #state{chunk = false} = State) -> - send_ctrl_message(State, mk_cmd("DELE ~s", [File])), + _ = send_ctrl_message(State, mk_cmd("DELE ~s", [File])), activate_ctrl_connection(State), {noreply, State#state{client = From}}; handle_call({_, {mkdir, Dir}}, From, #state{chunk = false} = State) -> - send_ctrl_message(State, mk_cmd("MKD ~s", [Dir])), + _ = send_ctrl_message(State, mk_cmd("MKD ~s", [Dir])), activate_ctrl_connection(State), {noreply, State#state{client = From}}; handle_call({_,{rmdir, Dir}}, From, #state{chunk = false} = State) -> - send_ctrl_message(State, mk_cmd("RMD ~s", [Dir])), + _ = send_ctrl_message(State, mk_cmd("RMD ~s", [Dir])), activate_ctrl_connection(State), {noreply, State#state{client = From}}; handle_call({_,{type, Type}}, From, #state{chunk = false} = State) -> case Type of ascii -> - send_ctrl_message(State, mk_cmd("TYPE A", [])), + _ = send_ctrl_message(State, mk_cmd("TYPE A", [])), activate_ctrl_connection(State), {noreply, State#state{caller = type, type = ascii, client = From}}; binary -> - send_ctrl_message(State, mk_cmd("TYPE I", [])), + _ = send_ctrl_message(State, mk_cmd("TYPE I", [])), activate_ctrl_connection(State), {noreply, State#state{caller = type, type = binary, client = From}}; @@ -1423,7 +1260,7 @@ handle_call({_, chunk_end}, _, #state{chunk = false} = State) -> {reply, {error, echunk}, State}; handle_call({_, {quote, Cmd}}, From, #state{chunk = false} = State) -> - send_ctrl_message(State, mk_cmd(Cmd, [])), + _ = send_ctrl_message(State, mk_cmd(Cmd, [])), activate_ctrl_connection(State), {noreply, State#state{client = From, caller = quote}}; @@ -1447,7 +1284,7 @@ handle_call(Request, _Timeout, State) -> %% Description: Handles cast messages. %%------------------------------------------------------------------------- handle_cast({Pid, close}, #state{owner = Pid} = State) -> - send_ctrl_message(State, mk_cmd("QUIT", [])), + _ = send_ctrl_message(State, mk_cmd("QUIT", [])), close_ctrl_connection(State), close_data_connection(State), {stop, normal, State#state{csock = undefined, dsock = undefined}}; @@ -1743,17 +1580,17 @@ start_link(Opts, GenServerOptions) -> %%-------------------------------------------------------------------------- %% User handling handle_user(User, Password, Acc, State) -> - send_ctrl_message(State, mk_cmd("USER ~s", [User])), + _ = send_ctrl_message(State, mk_cmd("USER ~s", [User])), activate_ctrl_connection(State), {noreply, State#state{caller = {handle_user, Password, Acc}}}. handle_user_passwd(Password, Acc, State) -> - send_ctrl_message(State, mk_cmd("PASS ~s", [Password])), + _ = send_ctrl_message(State, mk_cmd("PASS ~s", [Password])), activate_ctrl_connection(State), {noreply, State#state{caller = {handle_user_passwd, Acc}}}. handle_user_account(Acc, State) -> - send_ctrl_message(State, mk_cmd("ACCT ~s", [Acc])), + _ = send_ctrl_message(State, mk_cmd("ACCT ~s", [Acc])), activate_ctrl_connection(State), {noreply, State#state{caller = handle_user_account}}. @@ -1770,7 +1607,7 @@ handle_ctrl_result({tls_upgrade, _}, #state{csock = {tcp, Socket}, case ssl:connect(Socket, TLSOptions, Timeout) of {ok, TLSSocket} -> State = State0#state{csock = {ssl,TLSSocket}}, - send_ctrl_message(State, mk_cmd("PBSZ 0", [])), + _ = send_ctrl_message(State, mk_cmd("PBSZ 0", [])), activate_ctrl_connection(State), {noreply, State#state{tls_upgrading_data_connection = {true, pbsz}} }; {error, _} = Error -> @@ -1781,7 +1618,7 @@ handle_ctrl_result({tls_upgrade, _}, #state{csock = {tcp, Socket}, end; handle_ctrl_result({pos_compl, _}, #state{tls_upgrading_data_connection = {true, pbsz}} = State) -> - send_ctrl_message(State, mk_cmd("PROT P", [])), + _ = send_ctrl_message(State, mk_cmd("PROT P", [])), activate_ctrl_connection(State), {noreply, State#state{tls_upgrading_data_connection = {true, prot}}}; @@ -1975,7 +1812,7 @@ handle_ctrl_result({pos_compl, Lines}, #state{caller = {handle_dir_data, Dir, DirData}} = State) -> OldDir = pwd_result(Lines), - send_ctrl_message(State, mk_cmd("CWD ~s", [Dir])), + _ = send_ctrl_message(State, mk_cmd("CWD ~s", [Dir])), activate_ctrl_connection(State), {noreply, State#state{caller = {handle_dir_data_second_phase, OldDir, DirData}}}; @@ -1991,7 +1828,7 @@ handle_ctrl_result(S={_Status, _}, handle_ctrl_result({pos_compl, _}, #state{caller = {handle_dir_data_second_phase, OldDir, DirData}} = State) -> - send_ctrl_message(State, mk_cmd("CWD ~s", [OldDir])), + _ = send_ctrl_message(State, mk_cmd("CWD ~s", [OldDir])), activate_ctrl_connection(State), {noreply, State#state{caller = {handle_dir_data_third_phase, DirData}}}; handle_ctrl_result({Status, _}, @@ -2013,7 +1850,7 @@ handle_ctrl_result(Status={epath, _}, #state{caller = {dir,_}} = State) -> %% File renaming handle_ctrl_result({pos_interm, _}, #state{caller = {rename, NewFile}} = State) -> - send_ctrl_message(State, mk_cmd("RNTO ~s", [NewFile])), + _ = send_ctrl_message(State, mk_cmd("RNTO ~s", [NewFile])), activate_ctrl_connection(State), {noreply, State#state{caller = rename_second_phase}}; @@ -2190,28 +2027,28 @@ handle_caller(#state{caller = {dir, Dir, Len}} = State) -> short -> "NLST"; long -> "LIST" end, - case Dir of - "" -> - send_ctrl_message(State, mk_cmd(Cmd, "")); - _ -> - send_ctrl_message(State, mk_cmd(Cmd ++ " ~s", [Dir])) - end, + _ = case Dir of + "" -> + send_ctrl_message(State, mk_cmd(Cmd, "")); + _ -> + send_ctrl_message(State, mk_cmd(Cmd ++ " ~s", [Dir])) + end, activate_ctrl_connection(State), {noreply, State#state{caller = {dir, Dir}}}; handle_caller(#state{caller = {recv_bin, RemoteFile}} = State) -> - send_ctrl_message(State, mk_cmd("RETR ~s", [RemoteFile])), + _ = send_ctrl_message(State, mk_cmd("RETR ~s", [RemoteFile])), activate_ctrl_connection(State), {noreply, State#state{caller = recv_bin}}; handle_caller(#state{caller = {start_chunk_transfer, Cmd, RemoteFile}} = State) -> - send_ctrl_message(State, mk_cmd("~s ~s", [Cmd, RemoteFile])), + _ = send_ctrl_message(State, mk_cmd("~s ~s", [Cmd, RemoteFile])), activate_ctrl_connection(State), {noreply, State#state{caller = start_chunk_transfer}}; handle_caller(#state{caller = {recv_file, RemoteFile, Fd}} = State) -> - send_ctrl_message(State, mk_cmd("RETR ~s", [RemoteFile])), + _ = send_ctrl_message(State, mk_cmd("RETR ~s", [RemoteFile])), activate_ctrl_connection(State), {noreply, State#state{caller = {recv_file, Fd}}}; @@ -2219,7 +2056,7 @@ handle_caller(#state{caller = {transfer_file, {Cmd, LocalFile, RemoteFile}}, ldir = LocalDir, client = From} = State) -> case file_open(filename:absname(LocalFile, LocalDir), read) of {ok, Fd} -> - send_ctrl_message(State, mk_cmd("~s ~s", [Cmd, RemoteFile])), + _ = send_ctrl_message(State, mk_cmd("~s ~s", [Cmd, RemoteFile])), activate_ctrl_connection(State), {noreply, State#state{caller = {transfer_file, Fd}}}; {error, _} -> @@ -2230,7 +2067,7 @@ handle_caller(#state{caller = {transfer_file, {Cmd, LocalFile, RemoteFile}}, handle_caller(#state{caller = {transfer_data, {Cmd, Bin, RemoteFile}}} = State) -> - send_ctrl_message(State, mk_cmd("~s ~s", [Cmd, RemoteFile])), + _ = send_ctrl_message(State, mk_cmd("~s ~s", [Cmd, RemoteFile])), activate_ctrl_connection(State), {noreply, State#state{caller = {transfer_data, Bin}}}. @@ -2244,7 +2081,7 @@ setup_ctrl_connection(Host, Port, Timeout, State) -> {ok, IpFam, CSock} -> NewState = State#state{csock = {tcp, CSock}, ipfamily = IpFam}, activate_ctrl_connection(NewState), - case Timeout - inets_lib:millisec_passed(MsTime) of + case Timeout - millisec_passed(MsTime) of Timeout2 when (Timeout2 >= 0) -> {ok, NewState#state{caller = open}, Timeout2}; _ -> @@ -2267,7 +2104,7 @@ setup_data_connection(#state{mode = active, {ok, {_, Port}} = sockname({tcp,LSock}), IpAddress = inet_parse:ntoa(IP), Cmd = mk_cmd("EPRT |2|~s|~p|", [IpAddress, Port]), - send_ctrl_message(State, Cmd), + _ = send_ctrl_message(State, Cmd), activate_ctrl_connection(State), {noreply, State#state{caller = {setup_data_connection, {LSock, Caller}}}}; @@ -2275,18 +2112,18 @@ setup_data_connection(#state{mode = active, {ok, LSock} = gen_tcp:listen(0, [{ip, IP}, {active, false}, binary, {packet, 0}]), {ok, Port} = inet:port(LSock), - case FtpExt of - false -> - {IP1, IP2, IP3, IP4} = IP, - {Port1, Port2} = {Port div 256, Port rem 256}, - send_ctrl_message(State, - mk_cmd("PORT ~w,~w,~w,~w,~w,~w", - [IP1, IP2, IP3, IP4, Port1, Port2])); - true -> - IpAddress = inet_parse:ntoa(IP), - Cmd = mk_cmd("EPRT |1|~s|~p|", [IpAddress, Port]), - send_ctrl_message(State, Cmd) - end, + _ = case FtpExt of + false -> + {IP1, IP2, IP3, IP4} = IP, + {Port1, Port2} = {Port div 256, Port rem 256}, + send_ctrl_message(State, + mk_cmd("PORT ~w,~w,~w,~w,~w,~w", + [IP1, IP2, IP3, IP4, Port1, Port2])); + true -> + IpAddress = inet_parse:ntoa(IP), + Cmd = mk_cmd("EPRT |1|~s|~p|", [IpAddress, Port]), + send_ctrl_message(State, Cmd) + end, activate_ctrl_connection(State), {noreply, State#state{caller = {setup_data_connection, {LSock, Caller}}}} @@ -2294,21 +2131,21 @@ setup_data_connection(#state{mode = active, setup_data_connection(#state{mode = passive, ipfamily = inet6, caller = Caller} = State) -> - send_ctrl_message(State, mk_cmd("EPSV", [])), + _ = send_ctrl_message(State, mk_cmd("EPSV", [])), activate_ctrl_connection(State), {noreply, State#state{caller = {setup_data_connection, Caller}}}; setup_data_connection(#state{mode = passive, ipfamily = inet, caller = Caller, ftp_extension = false} = State) -> - send_ctrl_message(State, mk_cmd("PASV", [])), + _ = send_ctrl_message(State, mk_cmd("PASV", [])), activate_ctrl_connection(State), {noreply, State#state{caller = {setup_data_connection, Caller}}}; setup_data_connection(#state{mode = passive, ipfamily = inet, caller = Caller, ftp_extension = true} = State) -> - send_ctrl_message(State, mk_cmd("EPSV", [])), + _ = send_ctrl_message(State, mk_cmd("EPSV", [])), activate_ctrl_connection(State), {noreply, State#state{caller = {setup_data_connection, Caller}}}. @@ -2594,3 +2431,166 @@ start_chunk(#state{client = From} = State) -> State#state{chunk = true, client = undefined, caller = undefined}. + + +%% This function extracts the start options from the +%% Valid options: +%% debug, +%% verbose +%% ipfamily +%% priority +%% flags (for backward compatibillity) +start_options(Options) -> + case lists:keysearch(flags, 1, Options) of + {value, {flags, Flags}} -> + Verbose = lists:member(verbose, Flags), + IsTrace = lists:member(trace, Flags), + IsDebug = lists:member(debug, Flags), + DebugLevel = + if + (IsTrace =:= true) -> + trace; + IsDebug =:= true -> + debug; + true -> + disable + end, + {ok, [{verbose, Verbose}, + {debug, DebugLevel}, + {priority, low}]}; + false -> + ValidateVerbose = + fun(true) -> true; + (false) -> true; + (_) -> false + end, + ValidateDebug = + fun(trace) -> true; + (debug) -> true; + (disable) -> true; + (_) -> false + end, + ValidatePriority = + fun(low) -> true; + (normal) -> true; + (high) -> true; + (_) -> false + end, + ValidOptions = + [{verbose, ValidateVerbose, false, false}, + {debug, ValidateDebug, false, disable}, + {priority, ValidatePriority, false, low}], + validate_options(Options, ValidOptions, []) + end. + + +%% This function extracts and validates the open options from the +%% Valid options: +%% mode +%% host +%% port +%% timeout +%% dtimeout +%% progress +%% ftp_extension + +open_options(Options) -> + ValidateMode = + fun(active) -> true; + (passive) -> true; + (_) -> false + end, + ValidateHost = + fun(Host) when is_list(Host) -> + true; + (Host) when is_tuple(Host) andalso + ((size(Host) =:= 4) orelse (size(Host) =:= 8)) -> + true; + (_) -> + false + end, + ValidatePort = + fun(Port) when is_integer(Port) andalso (Port > 0) -> true; + (_) -> false + end, + ValidateIpFamily = + fun(inet) -> true; + (inet6) -> true; + (inet6fb4) -> true; + (_) -> false + end, + ValidateTimeout = + fun(Timeout) when is_integer(Timeout) andalso (Timeout >= 0) -> true; + (_) -> false + end, + ValidateDTimeout = + fun(DTimeout) when is_integer(DTimeout) andalso (DTimeout >= 0) -> true; + (infinity) -> true; + (_) -> false + end, + ValidateProgress = + fun(ignore) -> + true; + ({Mod, Func, _InitProgress}) when is_atom(Mod) andalso + is_atom(Func) -> + true; + (_) -> + false + end, + ValidateFtpExtension = + fun(true) -> true; + (false) -> true; + (_) -> false + end, + ValidOptions = + [{mode, ValidateMode, false, ?DEFAULT_MODE}, + {host, ValidateHost, true, ehost}, + {port, ValidatePort, false, ?FTP_PORT}, + {ipfamily, ValidateIpFamily, false, inet}, + {timeout, ValidateTimeout, false, ?CONNECTION_TIMEOUT}, + {dtimeout, ValidateDTimeout, false, ?DATA_ACCEPT_TIMEOUT}, + {progress, ValidateProgress, false, ?PROGRESS_DEFAULT}, + {ftp_extension, ValidateFtpExtension, false, ?FTP_EXT_DEFAULT}], + validate_options(Options, ValidOptions, []). + +tls_options(Options) -> + %% Options will be validated by ssl application + proplists:get_value(tls, Options, undefined). + +validate_options([], [], Acc) -> + {ok, lists:reverse(Acc)}; +validate_options([], ValidOptions, Acc) -> + %% Check if any mandatory options are missing! + case [{Key, Reason} || {Key, _, true, Reason} <- ValidOptions] of + [] -> + Defaults = + [{Key, Default} || {Key, _, _, Default} <- ValidOptions], + {ok, lists:reverse(Defaults ++ Acc)}; + [{_, Reason}|_Missing] -> + throw({error, Reason}) + end; +validate_options([{Key, Value}|Options], ValidOptions, Acc) -> + case lists:keysearch(Key, 1, ValidOptions) of + {value, {Key, Validate, _, Default}} -> + case (catch Validate(Value)) of + true -> + NewValidOptions = lists:keydelete(Key, 1, ValidOptions), + validate_options(Options, NewValidOptions, + [{Key, Value} | Acc]); + _ -> + NewValidOptions = lists:keydelete(Key, 1, ValidOptions), + validate_options(Options, NewValidOptions, + [{Key, Default} | Acc]) + end; + false -> + validate_options(Options, ValidOptions, Acc) + end; +validate_options([_|Options], ValidOptions, Acc) -> + validate_options(Options, ValidOptions, Acc). + +%% Help function, elapsed milliseconds since T0 +millisec_passed(T0) -> + %% OTP 18 + erlang:convert_time_unit(erlang:monotonic_time() - T0, + native, + micro_seconds) div 1000. diff --git a/lib/ftp/src/ftp_app.erl b/lib/ftp/src/ftp_app.erl new file mode 100644 index 0000000000..d647d9fce3 --- /dev/null +++ b/lib/ftp/src/ftp_app.erl @@ -0,0 +1,47 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2002-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% + +%%%------------------------------------------------------------------- +%% @doc ftp public API +%% @end +%%%------------------------------------------------------------------- + +-module(ftp_app). + +-behaviour(application). + +%% Application callbacks +-export([start/2, stop/1]). + +%%==================================================================== +%% API +%%==================================================================== + +start(_StartType, _StartArgs) -> + ftp_sup:start_link(). + +%%-------------------------------------------------------------------- +stop(_State) -> + ok. + +%%==================================================================== +%% Internal functions +%%==================================================================== diff --git a/lib/inets/src/ftp/ftp_internal.hrl b/lib/ftp/src/ftp_internal.hrl index f29bb4a099..84f980e8fd 100644 --- a/lib/inets/src/ftp/ftp_internal.hrl +++ b/lib/ftp/src/ftp_internal.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2016. All Rights Reserved. +%% Copyright Ericsson AB 2005-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -22,12 +22,14 @@ -ifndef(ftp_internal_hrl). -define(ftp_internal_hrl, true). --include_lib("inets/src/inets_app/inets_internal.hrl"). - --define(SERVICE, ftpc). --define(fcri(Label, Content), ?report_important(Label, ?SERVICE, Content)). --define(fcrv(Label, Content), ?report_verbose(Label, ?SERVICE, Content)). --define(fcrd(Label, Content), ?report_debug(Label, ?SERVICE, Content)). --define(fcrt(Label, Content), ?report_trace(Label, ?SERVICE, Content)). +-define(CR, $\r). +-define(LF, $\n). +-define(CRLF, [$\r,$\n]). +-define(SP, $\s). +-define(TAB, $\t). +-define(LEFT_PAREN, $(). +-define(RIGHT_PAREN, $)). +-define(WHITE_SPACE, $ ). +-define(DOUBLE_QUOTE, $"). -endif. % -ifdef(ftp_internal_hrl). diff --git a/lib/inets/src/ftp/ftp_progress.erl b/lib/ftp/src/ftp_progress.erl index a6263e5cd7..a6263e5cd7 100644 --- a/lib/inets/src/ftp/ftp_progress.erl +++ b/lib/ftp/src/ftp_progress.erl diff --git a/lib/inets/src/ftp/ftp_response.erl b/lib/ftp/src/ftp_response.erl index d54d97dc91..d54d97dc91 100644 --- a/lib/inets/src/ftp/ftp_response.erl +++ b/lib/ftp/src/ftp_response.erl diff --git a/lib/ftp/src/ftp_sup.erl b/lib/ftp/src/ftp_sup.erl new file mode 100644 index 0000000000..f30046802f --- /dev/null +++ b/lib/ftp/src/ftp_sup.erl @@ -0,0 +1,68 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2002-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% + +%%%------------------------------------------------------------------- +%% @doc ftp top level supervisor. +%% @end +%%%------------------------------------------------------------------- + +-module(ftp_sup). + +-behaviour(supervisor). + +%% API +-export([start_child/1, start_link/0]). + +%% Supervisor callbacks +-export([init/1]). + +-define(SERVER, ?MODULE). + +%%==================================================================== +%% API functions +%%==================================================================== + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +start_child(Args) -> + supervisor:start_child(?MODULE, Args). + +%%==================================================================== +%% Supervisor callbacks +%%==================================================================== +init(_) -> + SupFlags = #{strategy => simple_one_for_one, + intensity => 0, + period => 3600}, + {ok, {SupFlags, child_specs()}}. + + +%%==================================================================== +%% Internal functions +%%==================================================================== +child_specs() -> + [#{id => undefined, + start => {ftp, start_link, []}, + restart => temporary, + shutdown => 4000, + type => worker, + modules => [ftp]}]. diff --git a/lib/ftp/test/Makefile b/lib/ftp/test/Makefile new file mode 100644 index 0000000000..147f8e5dd6 --- /dev/null +++ b/lib/ftp/test/Makefile @@ -0,0 +1,251 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 1997-2018. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# %CopyrightEnd% +# +# +# For an outline of how this all_SUITE_data stuff works, see the +# make file ../../ssl/test/Makefile. +# +include $(ERL_TOP)/make/target.mk +include $(ERL_TOP)/make/$(TARGET)/otp.mk + +# ---------------------------------------------------- +# Application version +# ---------------------------------------------------- +include ../vsn.mk +VSN = $(FTP_VSN) + + +# ---------------------------------------------------- +# Release directory specification +# ---------------------------------------------------- +RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN) + + +# ---------------------------------------------------- +# Target Specs +# ---------------------------------------------------- +INCLUDES = -I. \ + -I$(ERL_TOP)/lib/ftp/src + +CP = cp + +ifeq ($(TESTROOT_DIR),) +TESTROOT_DIR = /ldisk/tests/$(USER)/ftp +endif + +ifeq ($(FTP_DATA_DIR),) +FTP_DATA_DIR = $(TESTROOT_DIR)/data_dir +endif + +ifeq ($(FTP_PRIV_DIR),) +FTP_PRIV_DIR = $(TESTROOT_DIR)/priv_dir +endif + +FTP_FLAGS = -Dftp__data_dir='"$(FTP_DATA_DIR)"' \ + -Dftp_priv_dir='"$(FTP_PRIV_DIR)"' + + +### +### test suite debug flags +### +ifeq ($(FTP_DEBUG_CLIENT),) + FTP_DEBUG_CLIENT = y +endif + +ifeq ($(FTP_DEBUG_CLIENT),) + FTP_FLAGS += -Dftp_debug_client +endif + +ifeq ($(FTP_TRACE_CLIENT),) + FTP_DEBUG_CLIENT = y +endif + +ifeq ($(FTP_TRACE_CLIENT),y) + FTP_FLAGS += -Dftp_trace_client +endif + +ifneq ($(FTP_DEBUG),) + FTP_DEBUG = s +endif + +ifeq ($(FTP_DEBUG),l) + FTP_FLAGS += -Dftp_log +endif + +ifeq ($(FTP_DEBUG),d) + FTP_FLAGS += -Dftp_debug -Dftp_log +endif + + +FTP_FLAGS += -pa ../ftp/ebin + +FTP_ROOT = ../ftp + +MODULES = \ + erl_make_certs \ + ftp_SUITE \ + ftp_format_SUITE \ + ftp_test_lib + + +EBIN = . + +HRL_FILES = \ + ftp_internal.hrl + +ERL_FILES = $(MODULES:%=%.erl) + +SOURCE = $(ERL_FILES) $(HRL_FILES) + +TARGET_FILES = $(MODULES:%=$(EBIN)/%.$(EMULATOR)) + +FTP_SPECS = ftp.spec ftp_bench.spec +COVER_FILE = ftp.cover +FTP_FILES = ftp.config $(FTP_SPECS) + + +FTP_DATADIRS = ftp_SUITE_data + +DATADIRS = $(FTP_DATADIRS) + +EMAKEFILE = Emakefile +MAKE_EMAKE = $(wildcard $(ERL_TOP)/make/make_emakefile) + +ifeq ($(MAKE_EMAKE),) +BUILDTARGET = $(TARGET_FILES) +RELTEST_FILES = $(COVER_FILE) $(FTP_SPECS) $(SOURCE) +else +BUILDTARGET = emakebuild +RELTEST_FILES = $(EMAKEFILE) $(COVER_FILE) $(FTP_SPECS) $(SOURCE) +endif + + +# ---------------------------------------------------- +# Release directory specification +# ---------------------------------------------------- + +RELTESTSYSDIR = "$(RELEASE_PATH)/ftp_test" +RELTESTSYSALLDATADIR = $(RELTESTSYSDIR)/all_SUITE_data +RELTESTSYSBINDIR = $(RELTESTSYSALLDATADIR)/bin + + +# ---------------------------------------------------- +# FLAGS +# The path to the test_server ebin dir is needed when +# running the target "targets". +# ---------------------------------------------------- +ERL_COMPILE_FLAGS += \ + $(INCLUDES) \ + $(FTP_FLAGS) + +# ---------------------------------------------------- +# Targets +# erl -sname kalle -pa ../ebin +# If you intend to run the test suite locally (private), then +# there is some requirements: +# 1) FTP_PRIV_DIR must be created +# ---------------------------------------------------- + +tests debug opt: $(BUILDTARGET) + +targets: $(TARGET_FILES) + +.PHONY: emakebuild + +emakebuild: $(EMAKEFILE) + +$(EMAKEFILE): + $(MAKE_EMAKE) $(ERL_COMPILE_FLAGS) -o$(EBIN) '*_SUITE_make' | grep -v Warning > $(EMAKEFILE) + $(MAKE_EMAKE) $(ERL_COMPILE_FLAGS) -o$(EBIN) $(MODULES) | grep -v Warning >> $(EMAKEFILE) + +clean: + rm -f $(EMAKEFILE) + rm -f $(TARGET_FILES) + rm -f core *~ + +docs: + + +# ---------------------------------------------------- +# Release Target +# ---------------------------------------------------- +include $(ERL_TOP)/make/otp_release_targets.mk + +release_spec: opt + $(INSTALL_DIR) "$(RELSYSDIR)/test" + $(INSTALL_DATA) $(HRL_FILES) $(ERL_FILES) "$(RELSYSDIR)/test" + $(INSTALL_DATA) $(FTP_FILES) "$(RELSYSDIR)/test" + @for d in $(DATADIRS); do \ + echo "installing data dir $$d"; \ + if test -f $$d/TAR.exclude; then \ + echo $$d/TAR.exclude2 > $$d/TAR.exclude2; \ + cat $$d/TAR.exclude >> $$d/TAR.exclude2; \ + find $$d -name '*.contrib*' >> $$d/TAR.exclude2; \ + find $$d -name '*.keep*' >> $$d/TAR.exclude2; \ + find $$d -name '*.mkelem*' >> $$d/TAR.exclude2; \ + find $$d -name '*~' >> $$d/TAR.exclude2; \ + find $$d -name 'erl_crash.dump' >> $$d/TAR.exclude2; \ + find $$d -name 'core' >> $$d/TAR.exclude2; \ + find $$d -name '.cmake.state' >> $$d/TAR.exclude2; \ + tar cfX - $$d/TAR.exclude2 $$d | (cd "$(RELSYSDIR)/test"; tar xf -); \ + else \ + tar cf - $$d | (cd "$(RELSYSDIR)/test"; tar xf -); \ + fi; \ + done + +release_tests_spec: opt + $(INSTALL_DIR) $(RELTESTSYSDIR) + $(INSTALL_DATA) $(RELTEST_FILES) $(RELTESTSYSDIR) + chmod -R u+w $(RELTESTSYSDIR) + tar chf - $(DATADIRS) | (cd $(RELTESTSYSDIR); tar xf -) + $(INSTALL_DIR) $(RELTESTSYSALLDATADIR) + $(INSTALL_DIR) $(RELTESTSYSBINDIR) + chmod -R +x $(RELTESTSYSBINDIR) + $(INSTALL_DIR) $(RELTESTSYSALLDATADIR)/win32/lib + +release_docs_spec: + +info: + @echo "MAKE_EMAKE = $(MAKE_EMAKE)" + @echo "EMAKEFILE = $(EMAKEFILE)" + @echo "BUILDTARGET = $(BUILDTARGET)" + @echo "" + @echo "MODULES = $(MODULES)" + @echo "ERL_FILES = $(ERL_FILES)" + @echo "SOURCE = $(SOURCE)" + @echo "TARGET_FILES = $(TARGET_FILES)" + @echo "" + @echo "FTP_SPECS = $(FTP_SPECS)" + @echo "FTP_FILES = $(FTP_FILES)" + @echo "" + @echo "RELEASE_PATH = "$(RELEASE_PATH)"" + @echo "RELSYSDIR = "$(RELSYSDIR)"" + @echo "RELTESTSYSDIR = $(RELTESTSYSDIR)" + @echo "RELTESTSYSALLDATADIR = $(RELTESTSYSALLDATADIR)" + @echo "RELTESTSYSBINDIR = $(RELTESTSYSBINDIR)" + @echo "" + @echo "DATADIRS = $(DATADIRS)" + @echo "REL_DATADIRS = $(REL_DATADIRS)" + @echo "" + @echo "FTP_DATA_DIR = $(FTP_DATA_DIR)" + @echo "FTP_PRIV_DIR = $(FTP_PRIV_DIR)" + @echo "FTP_ROOT = $(FTP_ROOT)" + @echo "FTP_FLAGS = $(FTP_FLAGS)" + + diff --git a/lib/ftp/test/erl_make_certs.erl b/lib/ftp/test/erl_make_certs.erl new file mode 100644 index 0000000000..2db95825bc --- /dev/null +++ b/lib/ftp/test/erl_make_certs.erl @@ -0,0 +1,475 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2011-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% Create test certificates + +-module(erl_make_certs). +-include_lib("public_key/include/public_key.hrl"). + +-export([make_cert/1, gen_rsa/1, verify_signature/3, write_pem/3]). +-compile(export_all). + +%%-------------------------------------------------------------------- +%% @doc Create and return a der encoded certificate +%% Option Default +%% ------------------------------------------------------- +%% digest sha1 +%% validity {date(), date() + week()} +%% version 3 +%% subject [] list of the following content +%% {name, Name} +%% {email, Email} +%% {city, City} +%% {state, State} +%% {org, Org} +%% {org_unit, OrgUnit} +%% {country, Country} +%% {serial, Serial} +%% {title, Title} +%% {dnQualifer, DnQ} +%% issuer = {Issuer, IssuerKey} true (i.e. a ca cert is created) +%% (obs IssuerKey migth be {Key, Password} +%% key = KeyFile|KeyBin|rsa|dsa|ec Subject PublicKey rsa, dsa or ec generates key +%% +%% +%% (OBS: The generated keys are for testing only) +%% @spec ([{::atom(), ::term()}]) -> {Cert::binary(), Key::binary()} +%% @end +%%-------------------------------------------------------------------- + +make_cert(Opts) -> + SubjectPrivateKey = get_key(Opts), + {TBSCert, IssuerKey} = make_tbs(SubjectPrivateKey, Opts), + Cert = public_key:pkix_sign(TBSCert, IssuerKey), + true = verify_signature(Cert, IssuerKey, undef), %% verify that the keys where ok + {Cert, encode_key(SubjectPrivateKey)}. + +%%-------------------------------------------------------------------- +%% @doc Writes pem files in Dir with FileName ++ ".pem" and FileName ++ "_key.pem" +%% @spec (::string(), ::string(), {Cert,Key}) -> ok +%% @end +%%-------------------------------------------------------------------- +write_pem(Dir, FileName, {Cert, Key = {_,_,not_encrypted}}) when is_binary(Cert) -> + ok = der_to_pem(filename:join(Dir, FileName ++ ".pem"), + [{'Certificate', Cert, not_encrypted}]), + ok = der_to_pem(filename:join(Dir, FileName ++ "_key.pem"), [Key]). + +%%-------------------------------------------------------------------- +%% @doc Creates a rsa key (OBS: for testing only) +%% the size are in bytes +%% @spec (::integer()) -> {::atom(), ::binary(), ::opaque()} +%% @end +%%-------------------------------------------------------------------- +gen_rsa(Size) when is_integer(Size) -> + Key = gen_rsa2(Size), + {Key, encode_key(Key)}. + +%%-------------------------------------------------------------------- +%% @doc Creates a dsa key (OBS: for testing only) +%% the sizes are in bytes +%% @spec (::integer()) -> {::atom(), ::binary(), ::opaque()} +%% @end +%%-------------------------------------------------------------------- +gen_dsa(LSize,NSize) when is_integer(LSize), is_integer(NSize) -> + Key = gen_dsa2(LSize, NSize), + {Key, encode_key(Key)}. + +%%-------------------------------------------------------------------- +%% @doc Creates a ec key (OBS: for testing only) +%% the sizes are in bytes +%% @spec (::integer()) -> {::atom(), ::binary(), ::opaque()} +%% @end +%%-------------------------------------------------------------------- +gen_ec(Curve) when is_atom(Curve) -> + Key = gen_ec2(Curve), + {Key, encode_key(Key)}. + +%%-------------------------------------------------------------------- +%% @doc Verifies cert signatures +%% @spec (::binary(), ::tuple()) -> ::boolean() +%% @end +%%-------------------------------------------------------------------- +verify_signature(DerEncodedCert, DerKey, _KeyParams) -> + Key = decode_key(DerKey), + case Key of + #'RSAPrivateKey'{modulus=Mod, publicExponent=Exp} -> + public_key:pkix_verify(DerEncodedCert, + #'RSAPublicKey'{modulus=Mod, publicExponent=Exp}); + #'DSAPrivateKey'{p=P, q=Q, g=G, y=Y} -> + public_key:pkix_verify(DerEncodedCert, {Y, #'Dss-Parms'{p=P, q=Q, g=G}}); + #'ECPrivateKey'{version = _Version, privateKey = _PrivKey, + parameters = Params, publicKey = {0, PubKey}} -> + public_key:pkix_verify(DerEncodedCert, {#'ECPoint'{point = PubKey}, Params}) + end. + +%%%%%%%%%%%%%%%%%%%%%%%%% Implementation %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +get_key(Opts) -> + case proplists:get_value(key, Opts) of + undefined -> make_key(rsa, Opts); + rsa -> make_key(rsa, Opts); + dsa -> make_key(dsa, Opts); + ec -> make_key(ec, Opts); + Key -> + Password = proplists:get_value(password, Opts, no_passwd), + decode_key(Key, Password) + end. + +decode_key({Key, Pw}) -> + decode_key(Key, Pw); +decode_key(Key) -> + decode_key(Key, no_passwd). + + +decode_key(#'RSAPublicKey'{} = Key,_) -> + Key; +decode_key(#'RSAPrivateKey'{} = Key,_) -> + Key; +decode_key(#'DSAPrivateKey'{} = Key,_) -> + Key; +decode_key(#'ECPrivateKey'{} = Key,_) -> + Key; +decode_key(PemEntry = {_,_,_}, Pw) -> + public_key:pem_entry_decode(PemEntry, Pw); +decode_key(PemBin, Pw) -> + [KeyInfo] = public_key:pem_decode(PemBin), + decode_key(KeyInfo, Pw). + +encode_key(Key = #'RSAPrivateKey'{}) -> + {ok, Der} = 'OTP-PUB-KEY':encode('RSAPrivateKey', Key), + {'RSAPrivateKey', Der, not_encrypted}; +encode_key(Key = #'DSAPrivateKey'{}) -> + {ok, Der} = 'OTP-PUB-KEY':encode('DSAPrivateKey', Key), + {'DSAPrivateKey', Der, not_encrypted}; +encode_key(Key = #'ECPrivateKey'{}) -> + {ok, Der} = 'OTP-PUB-KEY':encode('ECPrivateKey', Key), + {'ECPrivateKey', Der, not_encrypted}. + +make_tbs(SubjectKey, Opts) -> + Version = list_to_atom("v"++integer_to_list(proplists:get_value(version, Opts, 3))), + + IssuerProp = proplists:get_value(issuer, Opts, true), + {Issuer, IssuerKey} = issuer(IssuerProp, Opts, SubjectKey), + + {Algo, Parameters} = sign_algorithm(IssuerKey, Opts), + + SignAlgo = #'SignatureAlgorithm'{algorithm = Algo, + parameters = Parameters}, + Subject = case IssuerProp of + true -> %% Is a Root Ca + Issuer; + _ -> + subject(proplists:get_value(subject, Opts),false) + end, + + {#'OTPTBSCertificate'{serialNumber = trunc(random:uniform()*100000000)*10000 + 1, + signature = SignAlgo, + issuer = Issuer, + validity = validity(Opts), + subject = Subject, + subjectPublicKeyInfo = publickey(SubjectKey), + version = Version, + extensions = extensions(Opts) + }, IssuerKey}. + +issuer(true, Opts, SubjectKey) -> + %% Self signed + {subject(proplists:get_value(subject, Opts), true), SubjectKey}; +issuer({Issuer, IssuerKey}, _Opts, _SubjectKey) when is_binary(Issuer) -> + {issuer_der(Issuer), decode_key(IssuerKey)}; +issuer({File, IssuerKey}, _Opts, _SubjectKey) when is_list(File) -> + {ok, [{cert, Cert, _}|_]} = pem_to_der(File), + {issuer_der(Cert), decode_key(IssuerKey)}. + +issuer_der(Issuer) -> + Decoded = public_key:pkix_decode_cert(Issuer, otp), + #'OTPCertificate'{tbsCertificate=Tbs} = Decoded, + #'OTPTBSCertificate'{subject=Subject} = Tbs, + Subject. + +subject(undefined, IsRootCA) -> + User = if IsRootCA -> "RootCA"; true -> os:getenv("USER", "test_user") end, + Opts = [{email, User ++ "@erlang.org"}, + {name, User}, + {city, "Stockholm"}, + {country, "SE"}, + {org, "erlang"}, + {org_unit, "testing dep"}], + subject(Opts); +subject(Opts, _) -> + subject(Opts). + +subject(SubjectOpts) when is_list(SubjectOpts) -> + Encode = fun(Opt) -> + {Type,Value} = subject_enc(Opt), + [#'AttributeTypeAndValue'{type=Type, value=Value}] + end, + {rdnSequence, [Encode(Opt) || Opt <- SubjectOpts]}. + +%% Fill in the blanks +subject_enc({name, Name}) -> {?'id-at-commonName', {printableString, Name}}; +subject_enc({email, Email}) -> {?'id-emailAddress', Email}; +subject_enc({city, City}) -> {?'id-at-localityName', {printableString, City}}; +subject_enc({state, State}) -> {?'id-at-stateOrProvinceName', {printableString, State}}; +subject_enc({org, Org}) -> {?'id-at-organizationName', {printableString, Org}}; +subject_enc({org_unit, OrgUnit}) -> {?'id-at-organizationalUnitName', {printableString, OrgUnit}}; +subject_enc({country, Country}) -> {?'id-at-countryName', Country}; +subject_enc({serial, Serial}) -> {?'id-at-serialNumber', Serial}; +subject_enc({title, Title}) -> {?'id-at-title', {printableString, Title}}; +subject_enc({dnQualifer, DnQ}) -> {?'id-at-dnQualifier', DnQ}; +subject_enc(Other) -> Other. + + +extensions(Opts) -> + case proplists:get_value(extensions, Opts, []) of + false -> + asn1_NOVALUE; + Exts -> + lists:flatten([extension(Ext) || Ext <- default_extensions(Exts)]) + end. + +default_extensions(Exts) -> + Def = [{key_usage,undefined}, + {subject_altname, undefined}, + {issuer_altname, undefined}, + {basic_constraints, default}, + {name_constraints, undefined}, + {policy_constraints, undefined}, + {ext_key_usage, undefined}, + {inhibit_any, undefined}, + {auth_key_id, undefined}, + {subject_key_id, undefined}, + {policy_mapping, undefined}], + Filter = fun({Key, _}, D) -> lists:keydelete(Key, 1, D) end, + Exts ++ lists:foldl(Filter, Def, Exts). + +extension({_, undefined}) -> []; +extension({basic_constraints, Data}) -> + case Data of + default -> + #'Extension'{extnID = ?'id-ce-basicConstraints', + extnValue = #'BasicConstraints'{cA=true}, + critical=true}; + false -> + []; + Len when is_integer(Len) -> + #'Extension'{extnID = ?'id-ce-basicConstraints', + extnValue = #'BasicConstraints'{cA=true, pathLenConstraint=Len}, + critical=true}; + _ -> + #'Extension'{extnID = ?'id-ce-basicConstraints', + extnValue = Data} + end; +extension({Id, Data, Critical}) -> + #'Extension'{extnID = Id, extnValue = Data, critical = Critical}. + + +publickey(#'RSAPrivateKey'{modulus=N, publicExponent=E}) -> + Public = #'RSAPublicKey'{modulus=N, publicExponent=E}, + Algo = #'PublicKeyAlgorithm'{algorithm= ?rsaEncryption, parameters='NULL'}, + #'OTPSubjectPublicKeyInfo'{algorithm = Algo, + subjectPublicKey = Public}; +publickey(#'DSAPrivateKey'{p=P, q=Q, g=G, y=Y}) -> + Algo = #'PublicKeyAlgorithm'{algorithm= ?'id-dsa', + parameters={params, #'Dss-Parms'{p=P, q=Q, g=G}}}, + #'OTPSubjectPublicKeyInfo'{algorithm = Algo, subjectPublicKey = Y}; +publickey(#'ECPrivateKey'{version = _Version, + privateKey = _PrivKey, + parameters = Params, + publicKey = {0, PubKey}}) -> + Algo = #'PublicKeyAlgorithm'{algorithm= ?'id-ecPublicKey', parameters=Params}, + #'OTPSubjectPublicKeyInfo'{algorithm = Algo, + subjectPublicKey = #'ECPoint'{point = PubKey}}. + +validity(Opts) -> + DefFrom0 = calendar:gregorian_days_to_date(calendar:date_to_gregorian_days(date())-1), + DefTo0 = calendar:gregorian_days_to_date(calendar:date_to_gregorian_days(date())+7), + {DefFrom, DefTo} = proplists:get_value(validity, Opts, {DefFrom0, DefTo0}), + Format = fun({Y,M,D}) -> lists:flatten(io_lib:format("~w~2..0w~2..0w000000Z",[Y,M,D])) end, + #'Validity'{notBefore={generalTime, Format(DefFrom)}, + notAfter ={generalTime, Format(DefTo)}}. + +sign_algorithm(#'RSAPrivateKey'{}, Opts) -> + Type = case proplists:get_value(digest, Opts, sha1) of + sha1 -> ?'sha1WithRSAEncryption'; + sha512 -> ?'sha512WithRSAEncryption'; + sha384 -> ?'sha384WithRSAEncryption'; + sha256 -> ?'sha256WithRSAEncryption'; + md5 -> ?'md5WithRSAEncryption'; + md2 -> ?'md2WithRSAEncryption' + end, + {Type, 'NULL'}; +sign_algorithm(#'DSAPrivateKey'{p=P, q=Q, g=G}, _Opts) -> + {?'id-dsa-with-sha1', {params,#'Dss-Parms'{p=P, q=Q, g=G}}}; +sign_algorithm(#'ECPrivateKey'{}, Opts) -> + Type = case proplists:get_value(digest, Opts, sha1) of + sha1 -> ?'ecdsa-with-SHA1'; + sha512 -> ?'ecdsa-with-SHA512'; + sha384 -> ?'ecdsa-with-SHA384'; + sha256 -> ?'ecdsa-with-SHA256' + end, + {Type, 'NULL'}. + +make_key(rsa, _Opts) -> + %% (OBS: for testing only) + gen_rsa2(64); +make_key(dsa, _Opts) -> + gen_dsa2(128, 20); %% Bytes i.e. {1024, 160} +make_key(ec, _Opts) -> + %% (OBS: for testing only) + gen_ec2(secp256k1). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% RSA key generation (OBS: for testing only) +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +-define(SMALL_PRIMES, [65537,97,89,83,79,73,71,67,61,59,53, + 47,43,41,37,31,29,23,19,17,13,11,7,5,3]). + +gen_rsa2(Size) -> + P = prime(Size), + Q = prime(Size), + N = P*Q, + Tot = (P - 1) * (Q - 1), + [E|_] = lists:dropwhile(fun(Candidate) -> (Tot rem Candidate) == 0 end, ?SMALL_PRIMES), + {D1,D2} = extended_gcd(E, Tot), + D = erlang:max(D1,D2), + case D < E of + true -> + gen_rsa2(Size); + false -> + {Co1,Co2} = extended_gcd(Q, P), + Co = erlang:max(Co1,Co2), + #'RSAPrivateKey'{version = 'two-prime', + modulus = N, + publicExponent = E, + privateExponent = D, + prime1 = P, + prime2 = Q, + exponent1 = D rem (P-1), + exponent2 = D rem (Q-1), + coefficient = Co + } + end. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% DSA key generation (OBS: for testing only) +%% See http://en.wikipedia.org/wiki/Digital_Signature_Algorithm +%% and the fips_186-3.pdf +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +gen_dsa2(LSize, NSize) -> + Q = prime(NSize), %% Choose N-bit prime Q + X0 = prime(LSize), + P0 = prime((LSize div 2) +1), + + %% Choose L-bit prime modulus P such that p-1 is a multiple of q. + case dsa_search(X0 div (2*Q*P0), P0, Q, 1000) of + error -> + gen_dsa2(LSize, NSize); + P -> + G = crypto:mod_pow(2, (P-1) div Q, P), % Choose G a number whose multiplicative order modulo p is q. + %% such that This may be done by setting g = h^(p-1)/q mod p, commonly h=2 is used. + + X = prime(20), %% Choose x by some random method, where 0 < x < q. + Y = crypto:mod_pow(G, X, P), %% Calculate y = g^x mod p. + + #'DSAPrivateKey'{version=0, p = P, q = Q, + g = crypto:bytes_to_integer(G), y = crypto:bytes_to_integer(Y), x = X} + end. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% EC key generation (OBS: for testing only) +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +gen_ec2(CurveId) -> + {PubKey, PrivKey} = crypto:generate_key(ecdh, CurveId), + + #'ECPrivateKey'{version = 1, + privateKey = binary_to_list(PrivKey), + parameters = {namedCurve, pubkey_cert_records:namedCurves(CurveId)}, + publicKey = {0, PubKey}}. + +%% See fips_186-3.pdf +dsa_search(T, P0, Q, Iter) when Iter > 0 -> + P = 2*T*Q*P0 + 1, + case is_prime(P, 50) of + true -> P; + false -> dsa_search(T+1, P0, Q, Iter-1) + end; +dsa_search(_,_,_,_) -> + error. + + +%%%%%%% Crypto Math %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +prime(ByteSize) -> + Rand = odd_rand(ByteSize), + prime_odd(Rand, 0). + +prime_odd(Rand, N) -> + case is_prime(Rand, 50) of + true -> + Rand; + false -> + prime_odd(Rand+2, N+1) + end. + +%% see http://en.wikipedia.org/wiki/Fermat_primality_test +is_prime(_, 0) -> true; +is_prime(Candidate, Test) -> + CoPrime = odd_rand(10000, Candidate), + Result = crypto:mod_pow(CoPrime, Candidate, Candidate) , + is_prime(CoPrime, crypto:bytes_to_integer(Result), Candidate, Test). + +is_prime(CoPrime, CoPrime, Candidate, Test) -> + is_prime(Candidate, Test-1); +is_prime(_,_,_,_) -> + false. + +odd_rand(Size) -> + Min = 1 bsl (Size*8-1), + Max = (1 bsl (Size*8))-1, + odd_rand(Min, Max). + +odd_rand(Min,Max) -> + Rand = crypto:rand_uniform(Min,Max), + case Rand rem 2 of + 0 -> + Rand + 1; + _ -> + Rand + end. + +extended_gcd(A, B) -> + case A rem B of + 0 -> + {0, 1}; + N -> + {X, Y} = extended_gcd(B, N), + {Y, X-Y*(A div B)} + end. + +pem_to_der(File) -> + {ok, PemBin} = file:read_file(File), + public_key:pem_decode(PemBin). + +der_to_pem(File, Entries) -> + PemBin = public_key:pem_encode(Entries), + file:write_file(File, PemBin). + diff --git a/lib/ftp/test/ftp.config b/lib/ftp/test/ftp.config new file mode 100644 index 0000000000..2600237da9 --- /dev/null +++ b/lib/ftp/test/ftp.config @@ -0,0 +1 @@ +[].
\ No newline at end of file diff --git a/lib/ftp/test/ftp.cover b/lib/ftp/test/ftp.cover new file mode 100644 index 0000000000..5b155991bc --- /dev/null +++ b/lib/ftp/test/ftp.cover @@ -0,0 +1,2 @@ +{incl_app,ftp,details}. + diff --git a/lib/ftp/test/ftp.spec b/lib/ftp/test/ftp.spec new file mode 100644 index 0000000000..faf1e532a8 --- /dev/null +++ b/lib/ftp/test/ftp.spec @@ -0,0 +1 @@ +{suites,"../ftp_test", all}. diff --git a/lib/inets/test/ftp_SUITE.erl b/lib/ftp/test/ftp_SUITE.erl index 3dfec01ba2..92d2c36a86 100644 --- a/lib/inets/test/ftp_SUITE.erl +++ b/lib/ftp/test/ftp_SUITE.erl @@ -18,16 +18,10 @@ %% %CopyrightEnd% %% %% - -%% -%% ct:run("../inets_test", ftp_SUITE). -%% - -module(ftp_SUITE). -include_lib("kernel/include/file.hrl"). -include_lib("common_test/include/ct.hrl"). --include("inets_test_lib.hrl"). %% Note: This directive should only be used in test suites. -compile(export_all). @@ -59,6 +53,9 @@ all() -> {group, ftp_active}, {group, ftps_passive}, {group, ftps_active}, + {group, ftp_sup}, + app, + appup, error_ehost, clean_shutdown ]. @@ -68,7 +65,8 @@ groups() -> {ftp_passive, [], ftp_tests()}, {ftp_active, [], ftp_tests()}, {ftps_passive, [], ftp_tests()}, - {ftps_active, [], ftp_tests()} + {ftps_active, [], ftp_tests()}, + {ftp_sup, [], ftp_sup_tests()} ]. ftp_tests()-> @@ -109,6 +107,12 @@ ftp_tests()-> unexpected_bang ]. +ftp_sup_tests() -> + [ + start_ftp, + ftp_worker + ]. + %%-------------------------------------------------------------------- %%% Config @@ -181,7 +185,8 @@ init_per_suite(Config) -> {ok,Data} -> TstDir = filename:join(proplists:get_value(priv_dir,Config), "test"), file:make_dir(TstDir), - make_cert_files(dsa, rsa, "server-", proplists:get_value(data_dir,Config)), + %% make_cert_files(dsa, rsa, "server-", proplists:get_value(data_dir,Config)), + ftp_test_lib:make_cert_files(proplists:get_value(data_dir,Config)), start_ftpd([{test_dir,TstDir}, {ftpd_data,Data} | Config]) @@ -204,17 +209,43 @@ init_per_group(Group, Config) when Group == ftps_active, _:_ -> {skip, "Crypto did not start"} end; - +init_per_group(ftp_sup, Config) -> + try ftp:start() of + ok -> + Config + catch + _:_ -> + {skip, "Ftp did not start"} + end; init_per_group(_Group, Config) -> Config. + +end_per_group(ftp_sup, Config) -> + ftp:stop(), + Config; end_per_group(_Group, Config) -> Config. %%-------------------------------------------------------------------- +init_per_testcase(T, Config0) when T =:= app; T =:= appup -> + Config0; init_per_testcase(Case, Config0) -> Group = proplists:get_value(name, proplists:get_value(tc_group_properties,Config0)), - TLS = [{tls,[{reuse_sessions,true}]}], + + %% Workaround for interoperability issues with vsftpd =< 3.0.2: + %% + %% vsftpd =< 3.0.2 does not support ECDHE ciphers and the ssl application + %% removed ciphers with RSA key exchange from its default cipher list. + %% To allow interoperability with old versions of vsftpd, cipher suites + %% with RSA key exchange are appended to the default cipher list. + All = ssl:cipher_suites(all, 'tlsv1.2'), + Default = ssl:cipher_suites(default, 'tlsv1.2'), + RSASuites = + ssl:filter_cipher_suites(All, [{key_exchange, fun(rsa) -> true; + (_) -> false end}]), + Suites = ssl:append_cipher_suites(RSASuites, Default), + TLS = [{tls,[{reuse_sessions,true},{ciphers, Suites}]}], ACTIVE = [{mode,active}], PASSIVE = [{mode,passive}], CaseOpts = case Case of @@ -229,6 +260,7 @@ init_per_testcase(Case, Config0) -> ftps_active -> ftp__open(Config0, TLS++ ACTIVE ++ ExtraOpts); ftp_passive -> ftp__open(Config0, PASSIVE ++ ExtraOpts); ftps_passive -> ftp__open(Config0, TLS++PASSIVE ++ ExtraOpts); + ftp_sup -> ftp_start_service(Config0, ACTIVE ++ ExtraOpts); undefined -> Config0 end, case Case of @@ -244,7 +276,7 @@ init_per_testcase(Case, Config0) -> Config end. - +end_per_testcase(T, _Config) when T =:= app; T =:= appup -> ok; end_per_testcase(user, _Config) -> ok; end_per_testcase(bad_user, _Config) -> ok; end_per_testcase(error_elogin, _Config) -> ok; @@ -261,11 +293,30 @@ end_per_testcase(_Case, Config) -> _:_ -> ok end end, - ftp__close(Config). + Group = proplists:get_value(name, proplists:get_value(tc_group_properties,Config)), + case Group of + ftp_sup -> + ftp_stop_service(Config); + _Else -> + ftp__close(Config) + end. %%-------------------------------------------------------------------- %% Test Cases -------------------------------------------------------- %%-------------------------------------------------------------------- +app() -> + [{doc, "Test that the ftp app file is ok"}]. +app(Config) when is_list(Config) -> + ok = ?t:app_test(ftp). + +%%-------------------------------------------------------------------- +appup() -> + [{doc, "Test that the ftp appup file is ok"}]. +appup(Config) when is_list(Config) -> + ok = ?t:appup_test(ftp). + +%%-------------------------------------------------------------------- + user() -> [ {doc, "Open an ftp connection to a host, and logon as anonymous ftp," " then logoff"}]. @@ -834,6 +885,43 @@ clean_shutdown(Config) -> end end. +%%------------------------------------------------------------------------- +start_ftp() -> + [{doc, "Start/stop of ftp service"}]. +start_ftp(Config) -> + Pid0 = proplists:get_value(ftp,Config), + Pids0 = [ServicePid || {_, ServicePid} <- ftp:services()], + true = lists:member(Pid0, Pids0), + {ok, [_|_]} = ftp:service_info(Pid0), + ftp:stop_service(Pid0), + ct:sleep(100), + Pids1 = [ServicePid || {_, ServicePid} <- ftp:services()], + false = lists:member(Pid0, Pids1), + + Host = proplists:get_value(ftpd_host,Config), + Port = proplists:get_value(ftpd_port,Config), + + {ok, Pid1} = ftp:start_standalone([{host, Host},{port, Port}]), + Pids2 = [ServicePid || {_, ServicePid} <- ftp:services()], + false = lists:member(Pid1, Pids2). + +%%------------------------------------------------------------------------- +ftp_worker() -> + [{doc, "Makes sure the ftp worker processes are added and removed " + "appropriatly to/from the supervison tree."}]. +ftp_worker(Config) -> + Pid = proplists:get_value(ftp,Config), + case supervisor:which_children(ftp_sup) of + [{_,_, worker, [ftp]}] -> + ftp:stop_service(Pid), + ct:sleep(5000), + [] = supervisor:which_children(ftp_sup), + ok; + Children -> + ct:fail("Unexpected children: ~p",[Children]) + end. + + %%%---------------------------------------------------------------- %%% Error codes not tested elsewhere @@ -867,22 +955,6 @@ error_ehost(_Config) -> %% Internal functions ----------------------------------------------- %%-------------------------------------------------------------------- -make_cert_files(Alg1, Alg2, Prefix, Dir) -> - CaInfo = {CaCert,_} = erl_make_certs:make_cert([{key,Alg1}]), - {Cert,CertKey} = erl_make_certs:make_cert([{key,Alg2},{issuer,CaInfo}]), - CaCertFile = filename:join(Dir, Prefix++"cacerts.pem"), - CertFile = filename:join(Dir, Prefix++"cert.pem"), - KeyFile = filename:join(Dir, Prefix++"key.pem"), - der_to_pem(CaCertFile, [{'Certificate', CaCert, not_encrypted}]), - der_to_pem(CertFile, [{'Certificate', Cert, not_encrypted}]), - der_to_pem(KeyFile, [CertKey]), - ok. - -der_to_pem(File, Entries) -> - PemBin = public_key:pem_encode(Entries), - file:write_file(File, PemBin). - -%%-------------------------------------------------------------------- chk_file(Path=[C|_], ExpectedContents, Config) when 0<C,C=<255 -> chk_file([Path], ExpectedContents, Config); @@ -1029,6 +1101,17 @@ ftp__close(Config) -> ok = ftp:close(proplists:get_value(ftp,Config)), Config. +ftp_start_service(Config, Options) -> + Host = proplists:get_value(ftpd_host,Config), + Port = proplists:get_value(ftpd_port,Config), + ct:log("Host=~p, Port=~p",[Host,Port]), + {ok,Pid} = ftp:start_service([{host, Host},{port,Port} | Options]), + [{ftp,Pid}|Config]. + +ftp_stop_service(Config) -> + ok = ftp:stop_service(proplists:get_value(ftp,Config)), + Config. + split(Cs) -> string:tokens(Cs, "\r\n"). find_diff(Bin1, Bin2) -> diff --git a/lib/inets/test/ftp_SUITE_data/ftpd_hosts.skel b/lib/ftp/test/ftp_SUITE_data/ftpd_hosts.skel index 75096ce687..75096ce687 100644 --- a/lib/inets/test/ftp_SUITE_data/ftpd_hosts.skel +++ b/lib/ftp/test/ftp_SUITE_data/ftpd_hosts.skel diff --git a/lib/ftp/test/ftp_SUITE_data/vsftpd.conf b/lib/ftp/test/ftp_SUITE_data/vsftpd.conf new file mode 100644 index 0000000000..4568fad147 --- /dev/null +++ b/lib/ftp/test/ftp_SUITE_data/vsftpd.conf @@ -0,0 +1,33 @@ + +### +### Some parameters are given in the vsftpd start command. +### +### Typical command-line paramters are such that has a file path +### component like cert files. +### + + +listen=YES +listen_port=9999 +run_as_launching_user=YES +ssl_enable=YES +ssl_ciphers=HIGH:!aNULL:!MD5 +allow_anon_ssl=YES + +background=YES + +write_enable=YES +anonymous_enable=YES +anon_upload_enable=YES +anon_mkdir_write_enable=YES +anon_other_write_enable=YES +anon_world_readable_only=NO + +### Shouldn't be necessary.... +require_ssl_reuse=NO + +### Logging +#vsftpd_log_file=/devel/otp/vsftpd.log +#xferlog_enable=YES +#xferlog_std_format=NO +#log_ftp_protocol=YES
\ No newline at end of file diff --git a/lib/ftp/test/ftp_bench.spec b/lib/ftp/test/ftp_bench.spec new file mode 100644 index 0000000000..4d1ecf8891 --- /dev/null +++ b/lib/ftp/test/ftp_bench.spec @@ -0,0 +1 @@ +{suites,"../ftp_test",[]}. diff --git a/lib/inets/test/ftp_format_SUITE.erl b/lib/ftp/test/ftp_format_SUITE.erl index 95d594a44b..95d594a44b 100644 --- a/lib/inets/test/ftp_format_SUITE.erl +++ b/lib/ftp/test/ftp_format_SUITE.erl diff --git a/lib/ftp/test/ftp_internal.hrl b/lib/ftp/test/ftp_internal.hrl new file mode 120000 index 0000000000..2ae5c46460 --- /dev/null +++ b/lib/ftp/test/ftp_internal.hrl @@ -0,0 +1 @@ +../src/ftp_internal.hrl
\ No newline at end of file diff --git a/lib/inets/test/ftp_property_test_SUITE.erl b/lib/ftp/test/ftp_property_test_SUITE.erl index b314882296..46ed6959a8 100644 --- a/lib/inets/test/ftp_property_test_SUITE.erl +++ b/lib/ftp/test/ftp_property_test_SUITE.erl @@ -41,9 +41,11 @@ all() -> [prop_ftp_case]. init_per_suite(Config) -> - inets:start(), + ftp:start(), ct_property_test:init_per_suite(Config). +end_per_suite(Config) -> + Config. %%%---- test case prop_ftp_case(Config) -> diff --git a/lib/ftp/test/ftp_test_lib.erl b/lib/ftp/test/ftp_test_lib.erl new file mode 100644 index 0000000000..f5fbc39037 --- /dev/null +++ b/lib/ftp/test/ftp_test_lib.erl @@ -0,0 +1,126 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2001-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% +-module(ftp_test_lib). + +-include_lib("public_key/include/public_key.hrl"). + +-export([make_cert_files/1]). + + +make_cert_files(Dir) -> + #{server_config := ServerConf, + client_config := _} = + public_key:pkix_test_data(#{server_chain => + #{root => [{key, hardcode_rsa_key(1)}], + intermediates => [[{key, hardcode_rsa_key(2)}]], + peer => [{key, hardcode_rsa_key(3)}]}, + client_chain => + #{root => [{key, hardcode_rsa_key(1)}], + intermediates => [[{key, hardcode_rsa_key(3)}]], + peer => [{key, hardcode_rsa_key(2)}]}}), + + CaCertFile = filename:join(Dir, "server-cacerts.pem"), + CertFile = filename:join(Dir, "server-cert.pem"), + KeyFile = filename:join(Dir, "server-key.pem"), + + CAs = proplists:get_value(cacerts, ServerConf), + Cert = proplists:get_value(cert, ServerConf), + Key = proplists:get_value(key, ServerConf), + der_to_pem(CertFile, [cert_entry(Cert)]), + der_to_pem(KeyFile, [key_entry(Key)]), + der_to_pem(CaCertFile, ca_entries(CAs)). + +cert_entry(Cert) -> + {'Certificate', Cert, not_encrypted}. + +key_entry({'RSAPrivateKey', DERKey}) -> + {'RSAPrivateKey', DERKey, not_encrypted}; +key_entry({'DSAPrivateKey', DERKey}) -> + {'DSAPrivateKey', DERKey, not_encrypted}; +key_entry({'ECPrivateKey', DERKey}) -> + {'ECPrivateKey', DERKey, not_encrypted}. + +ca_entries(CAs) -> + [{'Certificate', CACert, not_encrypted} || CACert <- CAs]. + +der_to_pem(File, Entries) -> + PemBin = public_key:pem_encode(Entries), + file:write_file(File, PemBin). + +hardcode_rsa_key(1) -> + #'RSAPrivateKey'{ + version = 'two-prime', + modulus = +23995666614853919027835084074500048897452890537492185072956789802729257783422306095699263934587064480357348855732149402060270996295002843755712064937715826848741191927820899197493902093529581182351132392364214171173881547273475904587683433713767834856230531387991145055273426806331200574039205571401702219159773947658558490957010003143162250693492642996408861265758000254664396313741422909188635443907373976005987612936763564996605457102336549804831742940035613780926178523017685712710473543251580072875247250504243621640157403744718833162626193206685233710319205099867303242759099560438381385658382486042995679707669, + publicExponent = 17, + privateExponent = +11292078406990079542510627799764728892919007311761028269626724613049062486316379339152594792746853873109340637991599718616598115903530750002688030558925094987642913848386305504703012749896273497577003478759630198199473669305165131570674557041773098755873191241407597673069847908861741446606684974777271632545629600685952292605647052193819136445675100211504432575554351515262198132231537860917084269870590492135731720141577986787033006338680118008484613510063003323516659048210893001173583018220214626635609151105287049126443102976056146630518124476470236027123782297108342869049542023328584384300970694412006494684657, +prime1 = +169371138592582642967021557955633494538845517070305333860805485424261447791289944610138334410987654265476540480228705481960508520379619587635662291973699651583489223555422528867090299996446070521801757353675026048850480903160224210802452555900007597342687137394192939372218903554801584969667104937092080815197, + prime2 = +141675062317286527042995673340952251894209529891636708844197799307963834958115010129693036021381525952081167155681637592199810112261679449166276939178032066869788822014115556349519329537177920752776047051833616197615329017439297361972726138285974555338480581117881706656603857310337984049152655480389797687577, + exponent1 = +119556097830058336212015217380447172615655659108450823901745048534772786676204666783627059584226579481512852103690850928442711896738555003036938088452023283470698275450886490965004917644550167427154181661417665446247398284583687678213495921811770068712485038160606780733330990744565824684470897602653233516609, + exponent2 = +41669135975672507953822256864985956439473391144599032012999352737636422046504414744027363535700448809435637398729893409470532385959317485048904982111185902020526124121798693043976273393287623750816484427009887116945685005129205106462566511260580751570141347387612266663707016855981760014456663376585234613993, + coefficient = +76837684977089699359024365285678488693966186052769523357232308621548155587515525857011429902602352279058920284048929101483304120686557782043616693940283344235057989514310975192908256494992960578961614059245280827077951132083993754797053182279229469590276271658395444955906108899267024101096069475145863928441, + otherPrimeInfos = asn1_NOVALUE}; + +hardcode_rsa_key(2) -> + #'RSAPrivateKey'{ + version = 'two-prime', + modulus = +21343679768589700771839799834197557895311746244621307033143551583788179817796325695589283169969489517156931770973490560582341832744966317712674900833543896521418422508485833901274928542544381247956820115082240721897193055368570146764204557110415281995205343662628196075590438954399631753508888358737971039058298703003743872818150364935790613286541190842600031570570099801682794056444451081563070538409720109449780410837763602317050353477918147758267825417201591905091231778937606362076129350476690460157227101296599527319242747999737801698427160817755293383890373574621116766934110792127739174475029121017282777887777, + publicExponent = 17, + privateExponent = +18832658619343853622211588088997845201745658451136447382185486691577805721584993260814073385267196632785528033211903435807948675951440868570007265441362261636545666919252206383477878125774454042314841278013741813438699754736973658909592256273895837054592950290554290654932740253882028017801960316533503857992358685308186680144968293076156011747178275038098868263178095174694099811498968993700538293188879611375604635940554394589807673542938082281934965292051746326331046224291377703201248790910007232374006151098976879987912446997911775904329728563222485791845480864283470332826504617837402078265424772379987120023773, + prime1 = +146807662748886761089048448970170315054939768171908279335181627815919052012991509112344782731265837727551849787333310044397991034789843793140419387740928103541736452627413492093463231242466386868459637115999163097726153692593711599245170083315894262154838974616739452594203727376460632750934355508361223110419, + prime2 = +145385325050081892763917667176962991350872697916072592966410309213561884732628046256782356731057378829876640317801978404203665761131810712267778698468684631707642938779964806354584156202882543264893826268426566901882487709510744074274965029453915224310656287149777603803201831202222853023280023478269485417083, + exponent1 = +51814469205489445090252393754177758254684624060673510353593515699736136004585238510239335081623236845018299924941168250963996835808180162284853901555621683602965806809675350150634081614988136541809283687999704622726877773856604093851236499993845033701707873394143336209718962603456693912094478414715725803677, + exponent2 = +51312467664734785681382706062457526359131540440966797517556579722433606376221663384746714140373192528191755406283051201483646739222992016094510128871300458249756331334105225772206172777487956446433115153562317730076172132768497908567634716277852432109643395464627389577600646306666889302334125933506877206029, + coefficient = +30504662229874176232343608562807118278893368758027179776313787938167236952567905398252901545019583024374163153775359371298239336609182249464886717948407152570850677549297935773605431024166978281486607154204888016179709037883348099374995148481968169438302456074511782717758301581202874062062542434218011141540, + otherPrimeInfos = asn1_NOVALUE}; + +hardcode_rsa_key(3) -> + #'RSAPrivateKey'{ + version = 'two-prime', + modulus = +25089040456112869869472694987833070928503703615633809313972554887193090845137746668197820419383804666271752525807484521370419854590682661809972833718476098189250708650325307850184923546875260207894844301992963978994451844985784504212035958130279304082438876764367292331581532569155681984449177635856426023931875082020262146075451989132180409962870105455517050416234175675478291534563995772675388370042873175344937421148321291640477650173765084699931690748536036544188863178325887393475703801759010864779559318631816411493486934507417755306337476945299570726975433250753415110141783026008347194577506976486290259135429, + publicExponent = 17, + privateExponent = +8854955455098659953931539407470495621824836570223697404931489960185796768872145882893348383311931058684147950284994536954265831032005645344696294253579799360912014817761873358888796545955974191021709753644575521998041827642041589721895044045980930852625485916835514940558187965584358347452650930302268008446431977397918214293502821599497633970075862760001650736520566952260001423171553461362588848929781360590057040212831994258783694027013289053834376791974167294527043946669963760259975273650548116897900664646809242902841107022557239712438496384819445301703021164043324282687280801738470244471443835900160721870265, + prime1 = +171641816401041100605063917111691927706183918906535463031548413586331728772311589438043965564336865070070922328258143588739626712299625805650832695450270566547004154065267940032684307994238248203186986569945677705100224518137694769557564475390859269797990555863306972197736879644001860925483629009305104925823, + prime2 +=146170909759497809922264016492088453282310383272504533061020897155289106805616042710009332510822455269704884883705830985184223718261139908416790475825625309815234508695722132706422885088219618698987115562577878897003573425367881351537506046253616435685549396767356003663417208105346307649599145759863108910523, + exponent1 = +60579464612132153154728441333538327425711971378777222246428851853999433684345266860486105493295364142377972586444050678378691780811632637288529186629507258781295583787741625893888579292084087601124818789392592131211843947578009918667375697196773859928702549128225990187436545756706539150170692591519448797349, + exponent2 = +137572620950115585809189662580789132500998007785886619351549079675566218169991569609420548245479957900898715184664311515467504676010484619686391036071176762179044243478326713135456833024206699951987873470661533079532774988581535389682358631768109586527575902839864474036157372334443583670210960715165278974609, + coefficient = +15068630434698373319269196003209754243798959461311186548759287649485250508074064775263867418602372588394608558985183294561315208336731894947137343239541687540387209051236354318837334154993136528453613256169847839789803932725339395739618592522865156272771578671216082079933457043120923342632744996962853951612, + otherPrimeInfos = asn1_NOVALUE}. diff --git a/lib/ftp/test/property_test/README b/lib/ftp/test/property_test/README new file mode 100644 index 0000000000..57602bf719 --- /dev/null +++ b/lib/ftp/test/property_test/README @@ -0,0 +1,12 @@ + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% %%% +%%% WARNING %%% +%%% %%% +%%% This is experimental code which may be changed or removed %%% +%%% anytime without any warning. %%% +%%% %%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +The test in this directory are written assuming that the user has a QuickCheck license. They are to be run manually. Some may be possible to be run with other tools, e.g. PropEr. + diff --git a/lib/ftp/test/property_test/ftp_simple_client_server.erl b/lib/ftp/test/property_test/ftp_simple_client_server.erl new file mode 100644 index 0000000000..1bc54128f6 --- /dev/null +++ b/lib/ftp/test/property_test/ftp_simple_client_server.erl @@ -0,0 +1,307 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2004-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% + +-module(ftp_simple_client_server). + +-compile(export_all). + +-ifndef(EQC). +-ifndef(PROPER). +-define(EQC,true). +%%-define(PROPER,true). +-endif. +-endif. + + +-ifdef(EQC). + +-include_lib("eqc/include/eqc.hrl"). +-include_lib("eqc/include/eqc_statem.hrl"). +-define(MOD_eqc, eqc). +-define(MOD_eqc_gen, eqc_gen). +-define(MOD_eqc_statem, eqc_statem). + +-else. +-ifdef(PROPER). + +-include_lib("proper/include/proper.hrl"). +-define(MOD_eqc, proper). +-define(MOD_eqc_gen, proper_gen). +-define(MOD_eqc_statem, proper_statem). + +-endif. +-endif. + +-record(state, { + initialized = false, + priv_dir, + data_dir, + servers = [], % [ {IP,Port,Userid,Pwd} ] + clients = [], % [ client_ref() ] + store = [] % [ {Name,Contents} ] + }). + +-define(fmt(F,A), io:format(F,A)). +%%-define(fmt(F,A), ok). + +-define(v(K,L), proplists:get_value(K,L)). + +%%%================================================================ +%%% +%%% Properties +%%% + +%% This function is for normal eqc calls: +prop_ftp() -> + {ok,PWD} = file:get_cwd(), + prop_ftp(filename:join([PWD,?MODULE_STRING++"_data"]), + filename:join([PWD,?MODULE_STRING,"_files"])). + +%% This function is for calls from common_test test cases: +prop_ftp(Config) -> + prop_ftp(filename:join([?v(property_dir,Config), ?MODULE_STRING++"_data"]), + ?v(priv_dir,Config) ). + + +prop_ftp(DataDir, PrivDir) -> + S0 = #state{data_dir = DataDir, + priv_dir = PrivDir}, + ?FORALL(Cmds, more_commands(10,commands(?MODULE,S0)), + aggregate(command_names(Cmds), + begin {_H,S,Result} = run_commands(?MODULE,Cmds), + % io:format('**** Result=~p~n',[Result]), + % io:format('**** S=~p~n',[S]), + % io:format('**** _H=~p~n',[_H]), + % io:format('**** Cmds=~p~n',[Cmds]), + [cmnd_stop_server(X) || X <- S#state.servers], + [ftp:stop_service(X) || {ok,X} <- S#state.clients], + Result==ok + end) + ). + +%%%================================================================ +%%% +%%% State model +%%% + +%% @doc Returns the state in which each test case starts. (Unless a different +%% initial state is supplied explicitly to, e.g. commands/2.) +-spec initial_state() ->?MOD_eqc_statem:symbolic_state(). +initial_state() -> + ?fmt("Initial_state()~n",[]), + #state{}. + +%% @doc Command generator, S is the current state +-spec command(S :: ?MOD_eqc_statem:symbolic_state()) -> ?MOD_eqc_gen:gen(eqc_statem:call()). + +command(#state{initialized=false, + priv_dir=PrivDir}) -> + {call,?MODULE,cmnd_init,[PrivDir]}; + +command(#state{servers=[], + priv_dir=PrivDir, + data_dir=DataDir}) -> + {call,?MODULE,cmnd_start_server,[PrivDir,DataDir]}; + +command(#state{servers=Ss=[_|_], + clients=[]}) -> + {call,?MODULE,cmnd_start_client,[oneof(Ss)]}; + +command(#state{servers=Ss=[_|_], + clients=Cs=[_|_], + store=Store=[_|_] + }) -> + frequency([ + { 5, {call,?MODULE,cmnd_start_client,[oneof(Ss)]}}, + { 5, {call,?MODULE,cmnd_stop_client,[oneof(Cs)]}}, + {10, {call,?MODULE,cmnd_put,[oneof(Cs),file_path(),file_contents()]}}, + {20, {call,?MODULE,cmnd_get,[oneof(Cs),oneof(Store)]}}, + {10, {call,?MODULE,cmnd_delete,[oneof(Cs),oneof(Store)]}} + ]); + +command(#state{servers=Ss=[_|_], + clients=Cs=[_|_], + store=[] + }) -> + frequency([ + {5, {call,?MODULE,cmnd_start_client,[oneof(Ss)]}}, + {5, {call,?MODULE,cmnd_stop_client,[oneof(Cs)]}}, + {10, {call,?MODULE,cmnd_put,[oneof(Cs),file_path(),file_contents()]}} + ]). + +%% @doc Precondition, checked before command is added to the command sequence. +-spec precondition(S :: ?MOD_eqc_statem:symbolic_state(), C :: ?MOD_eqc_statem:call()) -> boolean(). + +precondition(#state{clients=Cs}, {call, _, cmnd_put, [C,_,_]}) -> lists:member(C,Cs); + +precondition(#state{clients=Cs, store=Store}, + {call, _, cmnd_get, [C,X]}) -> lists:member(C,Cs) andalso lists:member(X,Store); + +precondition(#state{clients=Cs, store=Store}, + {call, _, cmnd_delete, [C,X]}) -> lists:member(C,Cs) andalso lists:member(X,Store); + +precondition(#state{servers=Ss}, {call, _, cmnd_start_client, _}) -> Ss =/= []; + +precondition(#state{clients=Cs}, {call, _, cmnd_stop_client, [C]}) -> lists:member(C,Cs); + +precondition(#state{initialized=IsInit}, {call, _, cmnd_init, _}) -> IsInit==false; + +precondition(_S, {call, _, _, _}) -> true. + + +%% @doc Postcondition, checked after command has been evaluated +%% Note: S is the state before next_state(S,_,C) +-spec postcondition(S :: ?MOD_eqc_statem:dynamic_state(), C :: ?MOD_eqc_statem:call(), + Res :: term()) -> boolean(). + +postcondition(_S, {call, _, cmnd_get, [_,{_Name,Expected}]}, {ok,Value}) -> + Value == Expected; + +postcondition(S, {call, _, cmnd_delete, [_,{Name,_Expected}]}, ok) -> + ?fmt("file:read_file(..) = ~p~n",[file:read_file(filename:join(S#state.priv_dir,Name))]), + {error,enoent} == file:read_file(filename:join(S#state.priv_dir,Name)); + +postcondition(S, {call, _, cmnd_put, [_,Name,Value]}, ok) -> + {ok,Bin} = file:read_file(filename:join(S#state.priv_dir,Name)), + Bin == unicode:characters_to_binary(Value); + +postcondition(_S, {call, _, cmnd_stop_client, _}, ok) -> true; + +postcondition(_S, {call, _, cmnd_start_client, _}, {ok,_}) -> true; + +postcondition(_S, {call, _, cmnd_init, _}, ok) -> true; + +postcondition(_S, {call, _, cmnd_start_server, _}, {ok,_}) -> true. + + +%% @doc Next state transformation, S is the current state. Returns next state. +-spec next_state(S :: ?MOD_eqc_statem:symbolic_state(), + V :: ?MOD_eqc_statem:var(), + C :: ?MOD_eqc_statem:call()) -> ?MOD_eqc_statem:symbolic_state(). + +next_state(S, _V, {call, _, cmnd_put, [_,Name,Val]}) -> + S#state{store = [{Name,Val} | lists:keydelete(Name,1,S#state.store)]}; + +next_state(S, _V, {call, _, cmnd_delete, [_,{Name,_Val}]}) -> + S#state{store = lists:keydelete(Name,1,S#state.store)}; + +next_state(S, V, {call, _, cmnd_start_client, _}) -> + S#state{clients = [V | S#state.clients]}; + +next_state(S, V, {call, _, cmnd_start_server, _}) -> + S#state{servers = [V | S#state.servers]}; + +next_state(S, _V, {call, _, cmnd_stop_client, [C]}) -> + S#state{clients = S#state.clients -- [C]}; + +next_state(S, _V, {call, _, cmnd_init, _}) -> + S#state{initialized=true}; + +next_state(S, _V, {call, _, _, _}) -> + S. + +%%%================================================================ +%%% +%%% Data model +%%% + +file_path() -> non_empty(list(alphanum_char())). +%%file_path() -> non_empty( list(oneof([alphanum_char(), utf8_char()])) ). + +%%file_contents() -> list(alphanum_char()). +file_contents() -> list(oneof([alphanum_char(), utf8_char()])). + +alphanum_char() -> oneof(lists:seq($a,$z) ++ lists:seq($A,$Z) ++ lists:seq($0,$9)). + +utf8_char() -> oneof("åäöÅÄÖ話话カタカナひらがな"). + +%%%================================================================ +%%% +%%% Commands doing something with the System Under Test +%%% + +cmnd_init(PrivDir) -> + ?fmt('Call cmnd_init(~p)~n',[PrivDir]), + os:cmd("killall vsftpd"), + clear_files(PrivDir), + ok. + +cmnd_start_server(PrivDir, DataDir) -> + ?fmt('Call cmnd_start_server(~p, ~p)~n',[PrivDir,DataDir]), + Cmnd = ["vsftpd ", filename:join(DataDir,"vsftpd.conf"), + " -oftpd_banner=erlang_otp_testing" + " -oanon_root=",PrivDir + ], + ?fmt("Cmnd=~s~n",[Cmnd]), + case os:cmd(Cmnd) of + [] -> + {ok,{"localhost",9999,"ftp","[email protected]"}}; + Other -> + {error,Other} + end. + +cmnd_stop_server({ok,{_Host,Port,_Usr,_Pwd}}) -> + os:cmd("kill `netstat -tpln | grep "++integer_to_list(Port)++" | awk '{print $7}' | awk -F/ '{print $1}'`"). + +cmnd_start_client({ok,{Host,Port,Usr,Pwd}}) -> + ?fmt('Call cmnd_start_client(~p)...',[{Host,Port,Usr,Pwd}]), + case ftp:start_service([{host,Host},{port,Port}]) of + {ok,Client} -> + ?fmt("~p...",[{ok,Client}]), + case ftp:user(Client, Usr, Pwd) of + ok -> + ?fmt("OK!~n",[]), + {ok,Client}; + Other -> + ?fmt("Other1=~p~n",[Other]), + ftp:stop_service(Client), Other + end; + Other -> + ?fmt("Other2=~p~n",[Other]), + Other + end. + +cmnd_stop_client({ok,Client}) -> + ?fmt('Call cmnd_stop_client(~p)~n',[Client]), + ftp:stop_service(Client). %% -> ok | Other + +cmnd_delete({ok,Client}, {Name,_ExpectedValue}) -> + ?fmt('Call cmnd_delete(~p, ~p)~n',[Client,Name]), + R=ftp:delete(Client, Name), + ?fmt("R=~p~n",[R]), + R. + +cmnd_put({ok,Client}, Name, Value) -> + ?fmt('Call cmnd_put(~p, ~p, ~p)...',[Client, Name, Value]), + R = ftp:send_bin(Client, unicode:characters_to_binary(Value), Name), % ok | {error,Error} + ?fmt('~p~n',[R]), + R. + +cmnd_get({ok,Client}, {Name,_ExpectedValue}) -> + ?fmt('Call cmnd_get(~p, ~p)~n',[Client,Name]), + case ftp:recv_bin(Client, Name) of + {ok,Bin} -> {ok, unicode:characters_to_list(Bin)}; + Other -> Other + end. + + +clear_files(Dir) -> + os:cmd(["rm -fr ",filename:join(Dir,"*")]). diff --git a/lib/inets/test/ftp_SUITE_data/vsftpd.conf b/lib/ftp/test/property_test/ftp_simple_client_server_data/vsftpd.conf index a5584f5916..fd48e2abf0 100644 --- a/lib/inets/test/ftp_SUITE_data/vsftpd.conf +++ b/lib/ftp/test/property_test/ftp_simple_client_server_data/vsftpd.conf @@ -10,8 +10,8 @@ listen=YES listen_port=9999 run_as_launching_user=YES -ssl_enable=YES -allow_anon_ssl=YES +ssl_enable=NO +#allow_anon_ssl=YES background=YES diff --git a/lib/ftp/vsn.mk b/lib/ftp/vsn.mk new file mode 100644 index 0000000000..3099144a6e --- /dev/null +++ b/lib/ftp/vsn.mk @@ -0,0 +1,24 @@ +#-*-makefile-*- ; force emacs to enter makefile-mode + +# %CopyrightBegin% +# +# Copyright Ericsson AB 2001-2018. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# %CopyrightEnd% + +APPLICATION = ftp +FTP_VSN = 1.0 +PRE_VSN = +APP_VSN = "$(APPLICATION)-$(FTP_VSN)$(PRE_VSN)" diff --git a/lib/hipe/cerl/Makefile b/lib/hipe/cerl/Makefile index 9f50d6bf91..b6116c4276 100644 --- a/lib/hipe/cerl/Makefile +++ b/lib/hipe/cerl/Makefile @@ -44,7 +44,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/hipe-$(VSN) # Target Specs # ---------------------------------------------------- MODULES = cerl_cconv cerl_closurean cerl_hipeify cerl_lib \ - cerl_messagean cerl_pmatch cerl_prettypr cerl_to_icode \ + cerl_pmatch cerl_prettypr cerl_to_icode \ cerl_typean erl_bif_types erl_types HRL_FILES= cerl_hipe_primops.hrl diff --git a/lib/hipe/cerl/cerl_messagean.erl b/lib/hipe/cerl/cerl_messagean.erl deleted file mode 100644 index c79e045bd0..0000000000 --- a/lib/hipe/cerl/cerl_messagean.erl +++ /dev/null @@ -1,1095 +0,0 @@ -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% @copyright 2002 Richard Carlsson -%% @author Richard Carlsson <[email protected]> -%% @doc Message analysis of Core Erlang programs. - -%% TODO: might need a "top" (`any') element for any-length value lists. - --module(cerl_messagean). - --export([annotate/1]). - --import(cerl, [alias_pat/1, alias_var/1, ann_c_var/2, ann_c_fun/3, - apply_args/1, apply_op/1, atom_val/1, bitstr_size/1, - bitstr_val/1, binary_segments/1, c_letrec/2, - ann_c_tuple/2, c_nil/0, call_args/1, call_module/1, - call_name/1, case_arg/1, case_clauses/1, catch_body/1, - clause_body/1, clause_guard/1, clause_pats/1, cons_hd/1, - cons_tl/1, fun_body/1, fun_vars/1, get_ann/1, int_val/1, - is_c_atom/1, is_c_int/1, let_arg/1, let_body/1, - let_vars/1, letrec_body/1, letrec_defs/1, module_defs/1, - module_defs/1, module_exports/1, pat_vars/1, - primop_args/1, primop_name/1, receive_action/1, - receive_clauses/1, receive_timeout/1, seq_arg/1, - seq_body/1, set_ann/2, try_arg/1, try_body/1, try_vars/1, - try_evars/1, try_handler/1, tuple_es/1, type/1, - values_es/1]). - --import(cerl_trees, [get_label/1]). - --define(DEF_LIMIT, 4). - -%% -export([test/1, test1/1, ttest/1]). - -%% ttest(F) -> -%% {T, _} = cerl_trees:label(user_default:read(F)), -%% {Time0, _} = erlang:statistics(runtime), -%% analyze(T), -%% {Time1, _} = erlang:statistics(runtime), -%% Time1 - Time0. - -%% test(F) -> -%% {T, _} = cerl_trees:label(user_default:read(F)), -%% {Time0, _} = erlang:statistics(runtime), -%% {Esc, _Vars} = analyze(T), -%% {Time1, _} = erlang:statistics(runtime), -%% io:fwrite("messages: ~p.\n", [Esc]), -%% Set = sets:from_list(Esc), -%% H = fun (Node, Ctxt, Cont) -> -%% Doc = case get_ann(Node) of -%% [{label, L} | _] -> -%% B = sets:is_element(L, Set), -%% bf(Node, Ctxt, Cont, B); -%% _ -> -%% bf(Node, Ctxt, Cont, false) -%% end, -%% case type(Node) of -%% cons -> color(Doc); -%% tuple -> color(Doc); -%% _ -> Doc -%% end -%% end, -%% {ok, FD} = file:open("out.html",[write]), -%% Txt = cerl_prettypr:format(T, [{hook, H},{user,false}]), -%% io:put_chars(FD, "<pre>\n"), -%% io:put_chars(FD, html(Txt)), -%% io:put_chars(FD, "</pre>\n"), -%% file:close(FD), -%% {ok, Time1 - Time0}. - -%% test1(F) -> -%% {T, _} = cerl_trees:label(user_default:read(F)), -%% {Time0, _} = erlang:statistics(runtime), -%% {T1, Esc, Vars} = annotate(T), -%% {Time1, _} = erlang:statistics(runtime), -%% io:fwrite("messages: ~p.\n", [Esc]), -%% %%% io:fwrite("vars: ~p.\n", [[X || X <- dict:to_list(Vars)]]), -%% T2 = hhl_transform:transform(T1, Vars), -%% Set = sets:from_list(Esc), -%% H = fun (Node, Ctxt, Cont) -> -%% case get_ann(Node) of -%% [{label, L} | _] -> -%% B = sets:is_element(L, Set), -%% bf(Node, Ctxt, Cont, B); -%% _ -> -%% bf(Node, Ctxt, Cont, false) -%% end -%% end, -%% {ok, FD} = file:open("out.html",[write]), -%% Txt = cerl_prettypr:format(T2, [{hook, H},{user,false}]), -%% io:put_chars(FD, "<pre>\n"), -%% io:put_chars(FD, html(Txt)), -%% io:put_chars(FD, "</pre>\n"), -%% file:close(FD), -%% {ok, Time1 - Time0}. - -%% html(Cs) -> -%% html(Cs, []). - -%% html([$#, $< | Cs], As) -> -%% html_1(Cs, [$< | As]); -%% html([$< | Cs], As) -> -%% html(Cs, ";tl&" ++ As); -%% html([$> | Cs], As) -> -%% html(Cs, ";tg&" ++ As); -%% html([$& | Cs], As) -> -%% html(Cs, ";pma&" ++ As); -%% html([C | Cs], As) -> -%% html(Cs, [C | As]); -%% html([], As) -> -%% lists:reverse(As). - -%% html_1([$> | Cs], As) -> -%% html(Cs, [$> | As]); -%% html_1([C | Cs], As) -> -%% html_1(Cs, [C | As]). - -%% bf(Node, Ctxt, Cont, B) -> -%% B0 = cerl_prettypr:get_ctxt_user(Ctxt), -%% if B /= B0 -> -%% Ctxt1 = cerl_prettypr:set_ctxt_user(Ctxt, B), -%% Doc = Cont(Node, Ctxt1), -%% case B of -%% true -> -%% Start = "<b>", -%% End = "</b>"; -%% false -> -%% Start = "</b>", -%% End = "<b>" -%% end, -%% markup(Doc, Start, End); -%% true -> -%% Cont(Node, Ctxt) -%% end. - -%% color(Doc) -> -%% % Doc. -%% markup(Doc, "<font color=blue>", "</font>"). - -%% markup(Doc, Start, End) -> -%% prettypr:beside( -%% prettypr:null_text([$# | Start]), -%% prettypr:beside(Doc, -%% prettypr:null_text([$# | End]))). - - -%% ===================================================================== -%% annotate(Tree) -> {Tree1, Escapes, Vars} -%% -%% Tree = cerl:cerl() -%% -%% Analyzes `Tree' (see `analyze') and appends a term 'escapes', to -%% the annotation list of each constructor expression node and of -%% `Tree', corresponding to the escape information derived by the -%% analysis. Any previous such annotations are removed from `Tree'. -%% `Tree1' is the modified tree; for details on `OutList', -%% `Outputs' , `Dependencies', `Escapes' and `Parents', see -%% `analyze'. -%% -%% Note: `Tree' must be annotated with labels in order to use this -%% function; see `analyze' for details. - --type label() :: integer() | 'external' | 'top'. --type ordset(X) :: [X]. % XXX: TAKE ME OUT - --spec annotate(cerl:cerl()) -> {cerl:cerl(), ordset(label()), dict:dict()}. - -annotate(Tree) -> - {Esc0, Vars} = analyze(Tree), - Esc = sets:from_list(Esc0), - F = fun (T) -> - case type(T) of - literal -> T; -%%% var -> -%%% L = get_label(T), -%%% T1 = ann_escape(T, L, Esc), -%%% X = dict:fetch(L, Vars), -%%% set_ann(T1, append_ann({s,X}, get_ann(T1))); - _ -> - L = get_label(T), - ann_escape(T, L, Esc) - end - end, - {cerl_trees:map(F, Tree), Esc0, Vars}. - -ann_escape(T, L, Esc) -> - case sets:is_element(L, Esc) of - true -> - set_ann(T, append_ann(escapes, get_ann(T))); - false -> - T - end. - -append_ann(Tag, [X | Xs]) -> - if tuple_size(X) >= 1, element(1, X) =:= Tag -> - append_ann(Tag, Xs); - true -> - [X | append_ann(Tag, Xs)] - end; -append_ann(Tag, []) -> - [Tag]. - - -%% ===================================================================== -%% analyze(Tree) -> Escapes -%% -%% Tree = cerl:cerl() -%% Escapes = ordset(Label) -%% Label = integer() | external | top -%% -%% Analyzes a module or an expression represented by `Tree'. -%% -%% `Escapes' is the set of labels of constructor expressions in -%% `Tree' such that the created values may be accessed from outside -%% `Tree'. -%% -%% Note: `Tree' must be annotated with labels (as done by the -%% function `cerl_trees:label/1') in order to use this function. -%% The label annotation `{label, L}' (where L should be an integer) -%% must be the first element of the annotation list of each node in -%% the tree. Instances of variables bound in `Tree' which denote -%% the same variable must have the same label; apart from this, -%% labels should be unique. Constant literals do not need to be -%% labeled. - --record(state, {vars, out, dep, work, funs, k}). - -%% Note: We assume that all remote calls and primops return a single -%% value. - -%% The analysis determines which objects (identified by the -%% corresponding "cons-point" labels in the code) are likely to be -%% passed in a message. (If so, we say that they "escape".) It is always -%% safe to assume either case, because the send operation will assure -%% that things are copied if necessary. This analysis tries to -%% anticipate that copying will be done. -%% -%% Rules: -%% 1) An object passed as message argument (or part of such an -%% argument) to a known send-operation, will probably be a message. -%% 2) A received value is always a message (safe). -%% 3) The external function can return any object (unsafe). -%% 4) A function called from the external function can receive any -%% object (unsafe) as argument. -%% 5) Unknown functions/operations can return any object (unsafe). - -%% We wrap the given syntax tree T in a fun-expression labeled `top', -%% which is initially in the set of escaped labels. `top' will be -%% visited at least once. -%% -%% We create a separate function labeled `external', defined as: -%% "'external'/1 = fun () -> Any", which will represent any and all -%% functions outside T, and which returns the 'unsafe' value. - -analyze(Tree) -> - analyze(Tree, ?DEF_LIMIT). - -analyze(Tree, Limit) -> - {_, _, Esc, Dep, _Par} = cerl_closurean:analyze(Tree), -%%% io:fwrite("dependencies: ~w.\n", [dict:to_list(Dep)]), - analyze(Tree, Limit, Dep, Esc). - -analyze(Tree, Limit, Dep0, Esc0) -> - %% Note that we use different name spaces for variable labels and - %% function/call site labels, so we can reuse some names here. We - %% assume that the labeling of Tree only uses integers, not atoms. - Any = ann_c_var([{label, any}], 'Any'), - External = ann_c_var([{label, external}], {external, 1}), - ExtFun = ann_c_fun([{label, external}], [], Any), -%%% io:fwrite("external fun:\n~s.\n", -%%% [cerl_prettypr:format(ExtFun, [noann, {paper, 80}])]), - Top = ann_c_var([{label, top}], {top, 0}), - TopFun = ann_c_fun([{label, top}], [], Tree), - - %% The "start fun" just makes the initialisation easier. It is not - %% itself in the call graph. - StartFun = ann_c_fun([{label, start}], [], - c_letrec([{External, ExtFun}, {Top, TopFun}], - c_nil())), -%%% io:fwrite("start fun:\n~s.\n", -%%% [cerl_prettypr:format(StartFun, [{paper, 80}])]), - - %% Initialise the Any and Escape variables. Gather a database of all - %% fun-expressions in Tree and initialise their outputs and parameter - %% variables. All escaping functions can receive any values as - %% inputs. Bind all module- and letrec-defined variables to their - %% corresponding labels. - Esc = sets:from_list(Esc0), - Unsafe = unsafe(), - Empty = empty(), - Funs0 = dict:new(), - Vars0 = dict:store(escape, empty(), - dict:store(any, Unsafe, dict:new())), - Out0 = dict:new(), - F = fun (T, S = {Fs, Vs, Os}) -> - case type(T) of - 'fun' -> - L = get_label(T), - As = fun_vars(T), - X = case sets:is_element(L, Esc) of - true -> Unsafe; - false -> Empty - end, - {dict:store(L, T, Fs), - bind_vars_single(As, X, Vs), - dict:store(L, none, Os)}; - letrec -> - {Fs, bind_defs(letrec_defs(T), Vs), Os}; - module -> - {Fs, bind_defs(module_defs(T), Vs), Os}; - _ -> - S - end - end, - {Funs, Vars, Out} = cerl_trees:fold(F, {Funs0, Vars0, Out0}, StartFun), - - %% Add the dependency for the loop in 'external': - Dep = add_dep(loop, external, Dep0), - - %% Enter the fixpoint iteration at the StartFun. - St = loop(StartFun, start, #state{vars = Vars, - out = Out, - dep = Dep, - work = init_work(), - funs = Funs, - k = Limit}), - Ms = labels(dict:fetch(escape, St#state.vars)), - {Ms, St#state.vars}. - -loop(T, L, St0) -> -%%% io:fwrite("analyzing: ~w.\n",[L]), -%%% io:fwrite("work: ~w.\n", [St0#state.work]), - Xs0 = dict:fetch(L, St0#state.out), - {Xs1, St1} = visit(fun_body(T), L, St0), - Xs = limit(Xs1, St1#state.k), - {W, M} = case equal(Xs0, Xs) of - true -> - {St1#state.work, St1#state.out}; - false -> -%%% io:fwrite("out (~w) changed: ~w <- ~w.\n", -%%% [L, Xs, Xs0]), - M1 = dict:store(L, Xs, St1#state.out), - case dict:find(L, St1#state.dep) of - {ok, S} -> - {add_work(set__to_list(S), St1#state.work), - M1}; - error -> - {St1#state.work, M1} - end - end, - St2 = St1#state{out = M}, - case take_work(W) of - {ok, L1, W1} -> - T1 = dict:fetch(L1, St2#state.funs), - loop(T1, L1, St2#state{work = W1}); - none -> - St2 - end. - -visit(T, L, St) -> -%%% io:fwrite("visiting: ~w.\n",[type(T)]), - case type(T) of - literal -> - %% This is (or should be) a constant, even if it's compound, - %% so it's bugger all whether it is sent or not. - case cerl:concrete(T) of - [] -> {[empty()], St}; - X when is_atom(X) -> {[empty()], St}; - X when is_integer(X) -> {[empty()], St}; - X when is_float(X) -> {[empty()], St}; - _ -> - exit({not_literal, T}) - end; - var -> - %% If a variable is not already in the store here, it must - %% be free in the program. - L1 = get_label(T), - Vars = St#state.vars, - case dict:find(L1, Vars) of - {ok, X} -> - {[X], St}; - error -> -%%% io:fwrite("free var: ~w.\n",[L1]), - X = unsafe(), - St1 = St#state{vars = dict:store(L1, X, Vars)}, - {[X], St1} - end; - 'fun' -> - %% Must revisit the fun also, because its environment might - %% have changed. (We don't keep track of such dependencies.) - L1 = get_label(T), - St1 = St#state{work = add_work([L1], St#state.work)}, - %% Currently, lambda expressions can only be locally - %% allocated, and therefore we have to force copying by - %% treating them as "unsafe" for now. - {[unsafe()], St1}; - %% {[singleton(L1)], St1}; - values -> - visit_list(values_es(T), L, St); - cons -> - {[X1, X2], St1} = visit_list([cons_hd(T), cons_tl(T)], L, St), - L1 = get_label(T), - X = make_cons(L1, X1, X2), - %% Also store the values of the elements. - Hd = get_hd(X), - Tl = get_tl(X), - St2 = St1#state{vars = dict:store(L1, [Hd, Tl], St1#state.vars)}, - {[X], St2}; - tuple -> - {Xs, St1} = visit_list(tuple_es(T), L, St), - L1 = get_label(T), - %% Also store the values of the elements. - St2 = St1#state{vars = dict:store(L1, Xs, St1#state.vars)}, - {[struct(L1, Xs)], St2}; - 'let' -> - {Xs, St1} = visit(let_arg(T), L, St), - Vars = bind_vars(let_vars(T), Xs, St1#state.vars), - visit(let_body(T), L, St1#state{vars = Vars}); - seq -> - {_, St1} = visit(seq_arg(T), L, St), - visit(seq_body(T), L, St1); - apply -> - {_F, St1} = visit(apply_op(T), L, St), - {As, St2} = visit_list(apply_args(T), L, St1), - L1 = get_label(T), - Ls = get_deps(L1, St#state.dep), - Out = St2#state.out, - Xs1 = join_list([dict:fetch(X, Out) || X <- Ls]), - {Xs1, call_site(Ls, As, St2)}; - call -> - M = call_module(T), - F = call_name(T), - As = call_args(T), - {_, St1} = visit(M, L, St), - {_, St2} = visit(F, L, St1), - {Xs, St3} = visit_list(As, L, St2), - L1 = get_label(T), - remote_call(M, F, Xs, As, L1, St3); - primop -> - As = primop_args(T), - {Xs, St1} = visit_list(As, L, St), - F = atom_val(primop_name(T)), - primop_call(F, length(Xs), Xs, As, St1); - 'case' -> - {Xs, St1} = visit(case_arg(T), L, St), - visit_clauses(Xs, case_clauses(T), L, St1); - 'receive' -> - %% The received value is of course a message, so it - %% is 'empty()', not 'unsafe()'. - X = empty(), - {Xs1, St1} = visit_clauses([X], receive_clauses(T), L, St), - {_, St2} = visit(receive_timeout(T), L, St1), - {Xs2, St3} = visit(receive_action(T), L, St2), - {join(Xs1, Xs2), St3}; - 'try' -> - {Xs1, St1} = visit(try_arg(T), L, St), - X = unsafe(), - Vars = bind_vars(try_vars(T), Xs1, St1#state.vars), - {Xs2, St2} = visit(try_body(T), L, St1#state{vars = Vars}), - EVars = bind_vars(try_evars(T), [X, X, X], St2#state.vars), - {Xs3, St3} = visit(try_handler(T), L, St2#state{vars = EVars}), - {join(Xs2, Xs3), St3}; - 'catch' -> - %% If we catch an exception, we can get unsafe data. - {Xs, St1} = visit(catch_body(T), L, St), - {join([unsafe()], Xs), St1}; - binary -> - %% Binaries are heap objects, but we don't have special - %% shared-heap allocation operators for them at the moment. - %% They must therefore be treated as unsafe. - {_, St1} = visit_list(binary_segments(T), L, St), - {[unsafe()], St1}; - bitstr -> - %% The other fields are constant literals. - {_, St1} = visit(bitstr_val(T), L, St), - {_, St2} = visit(bitstr_size(T), L, St1), - {none, St2}; - letrec -> - %% All the bound funs should be revisited, because the - %% environment might have changed. - Ls = [get_label(F) || {_, F} <- letrec_defs(T)], - St1 = St#state{work = add_work(Ls, St#state.work)}, - visit(letrec_body(T), L, St1); - module -> - %% We regard a module as a tuple of function variables in - %% the body of a `letrec'. - visit(c_letrec(module_defs(T), - ann_c_tuple([{label, get_label(T)}], - module_exports(T))), - L, St) - end. - -visit_clause(T, Xs, L, St) -> - Vars = bind_pats(clause_pats(T), Xs, St#state.vars), - {_, St1} = visit(clause_guard(T), L, St#state{vars = Vars}), - visit(clause_body(T), L, St1). - -%% We assume correct value-list typing. - -visit_list([T | Ts], L, St) -> - {Xs, St1} = visit(T, L, St), - {Xs1, St2} = visit_list(Ts, L, St1), - X = case Xs of - [X1] -> X1; - _ -> empty() - end, - {[X | Xs1], St2}; -visit_list([], _L, St) -> - {[], St}. - -visit_clauses(Xs, [T | Ts], L, St) -> - {Xs1, St1} = visit_clause(T, Xs, L, St), - {Xs2, St2} = visit_clauses(Xs, Ts, L, St1), - {join(Xs1, Xs2), St2}; -visit_clauses(_, [], _L, St) -> - {none, St}. - -bind_defs([{V, F} | Ds], Vars) -> - bind_defs(Ds, dict:store(get_label(V), singleton(get_label(F)), Vars)); -bind_defs([], Vars) -> - Vars. - -bind_pats(Ps, none, Vars) -> - bind_pats_single(Ps, empty(), Vars); -bind_pats(Ps, Xs, Vars) -> - if length(Xs) =:= length(Ps) -> - bind_pats_list(Ps, Xs, Vars); - true -> - bind_pats_single(Ps, empty(), Vars) - end. - -%% The lists might not be of the same length. - -bind_pats_list([P | Ps], [X | Xs], Vars) -> - bind_pats_list(Ps, Xs, bind_pat_vars(P, X, Vars)); -bind_pats_list(Ps, [], Vars) -> - bind_pats_single(Ps, empty(), Vars); -bind_pats_list([], _, Vars) -> - Vars. - -bind_pats_single([P | Ps], X, Vars) -> - bind_pats_single(Ps, X, bind_pat_vars(P, X, Vars)); -bind_pats_single([], _X, Vars) -> - Vars. - -bind_pat_vars(P, X, Vars) -> - case type(P) of - var -> - dict:store(get_label(P), X, Vars); - literal -> - Vars; - cons -> - bind_pats_list([cons_hd(P), cons_tl(P)], - [get_hd(X), get_tl(X)], Vars); - tuple -> - case elements(X) of - none -> - bind_vars_single(pat_vars(P), X, Vars); - Xs -> - bind_pats_list(tuple_es(P), Xs, Vars) - end; - binary -> - %% See the handling of binary-expressions. - bind_pats_single(binary_segments(P), unsafe(), Vars); - bitstr -> - %% See the handling of binary-expressions. - bind_pats_single([bitstr_val(P), bitstr_size(P)], - unsafe(), Vars); - alias -> - P1 = alias_pat(P), - Vars1 = bind_pat_vars(P1, X, Vars), - dict:store(get_label(alias_var(P)), X, Vars1) - end. - -%%% %% This is the "exact" version of list representation, which simply -%%% %% mimics the actual cons, head and tail operations. -%%% make_cons(L, X1, X2) -> -%%% struct(L1, [X1, X2]). -%%% get_hd(X) -> -%%% case elements(X) of -%%% none -> X; -%%% [X1 | _] -> X1; -%%% _ -> empty() -%%% end. -%%% get_tl(X) -> -%%% case elements(X) of -%%% none -> X; -%%% [_, X2 | _] -> X2; -%%% _ -> empty() -%%% end. - -%% This version does not unnecessarily confuse spine labels with element -%% labels, and is safe. However, it loses precision if cons cells are -%% used for other things than proper lists. - -make_cons(L, X1, X2) -> - %% join subtypes and cons locations - join_single(struct(L, [X1]), X2). - -get_hd(X) -> - case elements(X) of - none -> X; - [X1 | _] -> X1; % First element represents list subtype. - _ -> empty() - end. - -get_tl(X) -> X. % Tail of X has same type as X. - -bind_vars(Vs, none, Vars) -> - bind_vars_single(Vs, empty(), Vars); -bind_vars(Vs, Xs, Vars) -> - if length(Vs) =:= length(Xs) -> - bind_vars_list(Vs, Xs, Vars); - true -> - bind_vars_single(Vs, empty(), Vars) - end. - -bind_vars_list([V | Vs], [X | Xs], Vars) -> - bind_vars_list(Vs, Xs, dict:store(get_label(V), X, Vars)); -bind_vars_list([], [], Vars) -> - Vars. - -bind_vars_single([V | Vs], X, Vars) -> - bind_vars_single(Vs, X, dict:store(get_label(V), X, Vars)); -bind_vars_single([], _X, Vars) -> - Vars. - -%% This handles a call site, updating parameter variables with respect -%% to the actual parameters. The 'external' function is handled -%% specially, since it can get an arbitrary number of arguments. For our -%% purposes here, calls to the external function can be ignored. - -call_site(Ls, Xs, St) -> -%%% io:fwrite("call site: ~w -> ~w (~w).\n", [L, Ls, Xs]), - {W, V} = call_site(Ls, Xs, St#state.work, St#state.vars, - St#state.funs, St#state.k), - St#state{work = W, vars = V}. - -call_site([external | Ls], Xs, W, V, Fs, Limit) -> - call_site(Ls, Xs, W, V, Fs, Limit); -call_site([L | Ls], Xs, W, V, Fs, Limit) -> - Vs = fun_vars(dict:fetch(L, Fs)), - case bind_args(Vs, Xs, V, Limit) of - {V1, true} -> - call_site(Ls, Xs, add_work([L], W), V1, Fs, Limit); - {V1, false} -> - call_site(Ls, Xs, W, V1, Fs, Limit) - end; -call_site([], _, W, V, _, _) -> - {W, V}. - -add_dep(Source, Target, Deps) -> - case dict:find(Source, Deps) of - {ok, X} -> - case set__is_member(Target, X) of - true -> - Deps; - false -> -%%% io:fwrite("new dep: ~w <- ~w.\n", [Target, Source]), - dict:store(Source, set__add(Target, X), Deps) - end; - error -> -%%% io:fwrite("new dep: ~w <- ~w.\n", [Target, Source]), - dict:store(Source, set__singleton(Target), Deps) - end. - -%% If the arity does not match the call, nothing is done here. - -bind_args(Vs, Xs, Vars, Limit) -> - if length(Vs) =:= length(Xs) -> - bind_args(Vs, Xs, Vars, Limit, false); - true -> - {Vars, false} - end. - -bind_args([V | Vs], [X | Xs], Vars, Limit, Ch) -> - L = get_label(V), - {Vars1, Ch1} = bind_arg(L, X, Vars, Limit, Ch), - bind_args(Vs, Xs, Vars1, Limit, Ch1); -bind_args([], [], Vars, _Limit, Ch) -> - {Vars, Ch}. - -%% bind_arg(L, X, Vars, Limit) -> -%% bind_arg(L, X, Vars, Limit, false). - -bind_arg(L, X, Vars, Limit, Ch) -> - X0 = dict:fetch(L, Vars), - X1 = limit_single(join_single(X, X0), Limit), - case equal_single(X0, X1) of - true -> - {Vars, Ch}; - false -> -%%% io:fwrite("arg (~w) changed: ~w <- ~w + ~w.\n", -%%% [L, X1, X0, X]), - {dict:store(L, X1, Vars), true} - end. - -%% This handles escapes from things like primops and remote calls. - -escape(Xs, Ns, St) -> - escape(Xs, Ns, 1, St). - -escape([_ | Xs], Ns=[N1 | _], N, St) when is_integer(N1), N1 > N -> - escape(Xs, Ns, N + 1, St); -escape([X | Xs], [N | Ns], N, St) -> - Vars = St#state.vars, - X0 = dict:fetch(escape, Vars), - X1 = join_single(X, X0), - case equal_single(X0, X1) of - true -> - escape(Xs, Ns, N + 1, St); - false -> -%%% io:fwrite("escape changed: ~w <- ~w + ~w.\n", [X1, X0, X]), - Vars1 = dict:store(escape, X1, Vars), - escape(Xs, Ns, N + 1, St#state{vars = Vars1}) - end; -escape(Xs, [_ | Ns], N, St) -> - escape(Xs, Ns, N + 1, St); -escape(_, _, _, St) -> - St. - -%% Handle primop calls: (At present, we assume that all unknown calls -%% yield exactly one value. This might have to be changed.) - -primop_call(F, A, Xs, _As, St0) -> - %% St1 = case is_escape_op(F, A) of - %% [] -> St0; - %% Ns -> escape(Xs, Ns, St0) - %% end, - St1 = St0, - case is_imm_op(F, A) of - true -> - {[empty()], St1}; - false -> - call_unknown(Xs, St1) - end. - -%% Handle remote-calls: (At present, we assume that all unknown calls -%% yield exactly one value. This might have to be changed.) - -remote_call(M, F, Xs, As, L, St) -> - case is_c_atom(M) andalso is_c_atom(F) of - true -> - remote_call_1(atom_val(M), atom_val(F), length(Xs), - Xs, As, L, St); - false -> - %% Unknown function - call_unknown(Xs, St) - end. - -%% When calling an unknown function, we assume that the result does -%% *not* contain any of the constructors in its arguments (but it could -%% return locally allocated data that we don't know about). Note that -%% even a "pure" function can still cons up new data. - -call_unknown(_Xs, St) -> - {[unsafe()], St}. - -%% We need to handle some important standard functions in order to get -%% decent precision. -%% TODO: foldl, map, mapfoldl - -remote_call_1(erlang, hd, 1, [X], _As, _L, St) -> - {[get_hd(X)], St}; -remote_call_1(erlang, tl, 1, [X], _As, _L, St) -> - {[get_tl(X)], St}; -remote_call_1(erlang, element, 2, [_,X], [N|_], _L, St) -> - case elements(X) of - none -> {[X], St}; - Xs -> - case is_c_int(N) of - true -> - N1 = int_val(N), - if is_integer(N1), 1 =< N1, N1 =< length(Xs) -> - {[nth(N1, Xs)], St}; - true -> - {none, St} - end; - false -> - %% Even if we don't know which element is selected, - %% we know that the top level is never part of the - %% returned value. - {[join_single_list(Xs)], St} - end - end; -remote_call_1(erlang, setelement, 3, [_,X, Y], [N|_], L, St) -> - %% The constructor gets the label of the call operation. - case elements(X) of - none -> {[join_single(singleton(L), join_single(X, Y))], St}; - Xs -> - case is_c_int(N) of - true -> - N1 = int_val(N), - if is_integer(N1), 1 =< N1, N1 =< length(Xs) -> - Xs1 = set_nth(N1, Y, Xs), - {[struct(L, Xs1)], St}; - true -> - {none, St} - end; - false -> - %% Even if we don't know which element is selected, - %% we know that the top level is never part of the - %% returned value (a new tuple is always created). - Xs1 = [join_single(Y, X1) || X1 <- Xs], - {[struct(L, Xs1)], St} - end - end; -remote_call_1(erlang, '++', 2, [X1,X2], _As, _L, St) -> - %% Note: this is unsafe for non-proper lists! (See make_cons/3). - %% No safe version is implemented. - {[join_single(X1, X2)], St}; -remote_call_1(erlang, '--', 2, [X1,_X2], _As, _L, St) -> - {[X1], St}; -remote_call_1(lists, append, 2, Xs, As, L, St) -> - remote_call_1(erlang, '++', 2, Xs, As, L, St); -remote_call_1(lists, subtract, 2, Xs, As, L, St) -> - remote_call_1(erlang, '--', 2, Xs, As, L, St); -remote_call_1(M, F, A, Xs, _As, _L, St0) -> - St1 = case is_escape_op(M, F, A) of - [] -> St0; - Ns -> escape(Xs, Ns, St0) - end, - case is_imm_op(M, F, A) of - true -> - {[empty()], St1}; - false -> - call_unknown(Xs, St1) - end. - -%% 1-based n:th-element list selector and update function. - -nth(1, [X | _Xs]) -> X; -nth(N, [_X | Xs]) when N > 1 -> nth(N - 1, Xs). - -set_nth(1, Y, [_X | Xs]) -> [Y | Xs]; -set_nth(N, Y, [X | Xs]) when N > 1 -> [X | set_nth(N - 1, Y, Xs)]. - -%% Domain: none | [V], where V = {S, none} | {S, [V]}, S = set(integer()). - -join(none, Xs2) -> Xs2; -join(Xs1, none) -> Xs1; -join(Xs1, Xs2) -> - if length(Xs1) =:= length(Xs2) -> - join_1(Xs1, Xs2); - true -> - none - end. - -join_1([X1 | Xs1], [X2 | Xs2]) -> - [join_single(X1, X2) | join_1(Xs1, Xs2)]; -join_1([], []) -> - []. - -join_list([Xs | Xss]) -> - join(Xs, join_list(Xss)); -join_list([]) -> - none. - -empty() -> {set__new(), []}. - -singleton(X) -> {set__singleton(X), []}. - -struct(X, Xs) -> {set__singleton(X), Xs}. - -elements({_, Xs}) -> Xs. - -unsafe() -> {set__singleton(unsafe), none}. - -equal(none, none) -> true; -equal(none, _) -> false; -equal(_, none) -> false; -equal(X1, X2) -> equal_1(X1, X2). - -equal_1([X1 | Xs1], [X2 | Xs2]) -> - equal_single(X1, X2) andalso equal_1(Xs1, Xs2); -equal_1([], []) -> true; -equal_1(_, _) -> false. - -equal_single({S1, none}, {S2, none}) -> - set__equal(S1, S2); -equal_single({_, none}, _) -> - false; -equal_single(_, {_, none}) -> - false; -equal_single({S1, Vs1}, {S2, Vs2}) -> - set__equal(S1, S2) andalso equal_single_lists(Vs1, Vs2). - -equal_single_lists([X1 | Xs1], [X2 | Xs2]) -> - equal_single(X1, X2) andalso equal_single_lists(Xs1, Xs2); -equal_single_lists([], []) -> - true; -equal_single_lists(_, _) -> - false. - -join_single({S, none}, V) -> - {set__union(S, labels(V)), none}; -join_single(V, {S, none}) -> - {set__union(S, labels(V)), none}; -join_single({S1, Vs1}, {S2, Vs2}) -> - {set__union(S1, S2), join_single_lists(Vs1, Vs2)}. - -join_single_list([V | Vs]) -> - join_single(V, join_single_list(Vs)); -join_single_list([]) -> - empty(). - -%% If one list has more elements that the other, and N is the length of -%% the longer list, then the result has N elements. - -join_single_lists([V1], [V2]) -> - [join_single(V1, V2)]; -join_single_lists([V1 | Vs1], [V2 | Vs2]) -> - [join_single(V1, V2) | join_single_lists(Vs1, Vs2)]; -join_single_lists([], Vs) -> Vs; -join_single_lists(Vs, []) -> Vs. - -collapse(V) -> - {labels(V), none}. - -%% collapse_list([]) -> -%% empty(); -%% collapse_list(Vs) -> -%% {labels_list(Vs), none}. - -labels({S, none}) -> S; -labels({S, []}) -> S; -labels({S, Vs}) -> set__union(S, labels_list(Vs)). - -labels_list([V]) -> - labels(V); -labels_list([V | Vs]) -> - set__union(labels(V), labels_list(Vs)). - -limit(none, _K) -> none; -limit(X, K) -> limit_list(X, K). - -limit_list([X | Xs], K) -> - [limit_single(X, K) | limit_list(Xs, K)]; -limit_list([], _) -> - []. - -limit_single({_, none} = V, _K) -> - V; -limit_single({_, []} = V, _K) -> - V; -limit_single({S, Vs}, K) when K > 0 -> - {S, limit_list(Vs, K - 1)}; -limit_single(V, _K) -> - collapse(V). - -%% Set abstraction for label sets in the domain. - -%% set__is_empty([]) -> true; -%% set__is_empty(_) -> false. - -set__new() -> []. - -set__singleton(X) -> [X]. - -set__to_list(S) -> S. - -%% set__from_list(S) -> ordsets:from_list(S). - -set__union(X, Y) -> ordsets:union(X, Y). - -set__add(X, S) -> ordsets:add_element(X, S). - -set__is_member(X, S) -> ordsets:is_element(X, S). - -%% set__subtract(X, Y) -> ordsets:subtract(X, Y). - -set__equal(X, Y) -> X =:= Y. - -%% A simple but efficient functional queue. - -queue__new() -> {[], []}. - -queue__put(X, {In, Out}) -> {[X | In], Out}. - -queue__get({In, [X | Out]}) -> {ok, X, {In, Out}}; -queue__get({[], _}) -> empty; -queue__get({In, _}) -> - [X | In1] = lists:reverse(In), - {ok, X, {[], In1}}. - -%% The work list - a queue without repeated elements. - -init_work() -> - {queue__new(), sets:new()}. - -add_work(Ls, {Q, Set}) -> - add_work(Ls, Q, Set). - -%% Note that the elements are enqueued in order. - -add_work([L | Ls], Q, Set) -> - case sets:is_element(L, Set) of - true -> - add_work(Ls, Q, Set); - false -> - add_work(Ls, queue__put(L, Q), sets:add_element(L, Set)) - end; -add_work([], Q, Set) -> - {Q, Set}. - -take_work({Queue0, Set0}) -> - case queue__get(Queue0) of - {ok, L, Queue1} -> - Set1 = sets:del_element(L, Set0), - {ok, L, {Queue1, Set1}}; - empty -> - none - end. - -get_deps(L, Dep) -> - case dict:find(L, Dep) of - {ok, Ls} -> Ls; - error -> [] - end. - -%% Escape operators may let their arguments escape. For this analysis, -%% only send-operations are considered as causing escapement, and only -%% in specific arguments. - -%% is_escape_op(_F, _A) -> []. - --spec is_escape_op(atom(), atom(), arity()) -> [arity()]. - -is_escape_op(erlang, '!', 2) -> [2]; -is_escape_op(erlang, send, 2) -> [2]; -is_escape_op(erlang, spawn, 1) -> [1]; -is_escape_op(erlang, spawn, 3) -> [3]; -is_escape_op(erlang, spawn, 4) -> [4]; -is_escape_op(erlang, spawn_link, 3) -> [3]; -is_escape_op(erlang, spawn_link, 4) -> [4]; -is_escape_op(_M, _F, _A) -> []. - -%% "Immediate" operators will never return heap allocated data. This is -%% of course true for operators that never return, like 'exit/1'. (Note -%% that floats are always heap allocated objects, and that most integer -%% arithmetic can return a bignum on the heap.) - --spec is_imm_op(atom(), arity()) -> boolean(). - -is_imm_op(match_fail, 1) -> true; -is_imm_op(_, _) -> false. - --spec is_imm_op(atom(), atom(), arity()) -> boolean(). - -is_imm_op(erlang, self, 0) -> true; -is_imm_op(erlang, '=:=', 2) -> true; -is_imm_op(erlang, '==', 2) -> true; -is_imm_op(erlang, '=/=', 2) -> true; -is_imm_op(erlang, '/=', 2) -> true; -is_imm_op(erlang, '<', 2) -> true; -is_imm_op(erlang, '=<', 2) -> true; -is_imm_op(erlang, '>', 2) -> true; -is_imm_op(erlang, '>=', 2) -> true; -is_imm_op(erlang, 'and', 2) -> true; -is_imm_op(erlang, 'or', 2) -> true; -is_imm_op(erlang, 'xor', 2) -> true; -is_imm_op(erlang, 'not', 1) -> true; -is_imm_op(erlang, is_alive, 0) -> true; -is_imm_op(erlang, is_atom, 1) -> true; -is_imm_op(erlang, is_binary, 1) -> true; -is_imm_op(erlang, is_builtin, 3) -> true; -is_imm_op(erlang, is_float, 1) -> true; -is_imm_op(erlang, is_function, 1) -> true; -is_imm_op(erlang, is_integer, 1) -> true; -is_imm_op(erlang, is_list, 1) -> true; -is_imm_op(erlang, is_number, 1) -> true; -is_imm_op(erlang, is_pid, 1) -> true; -is_imm_op(erlang, is_port, 1) -> true; -is_imm_op(erlang, is_process_alive, 1) -> true; -is_imm_op(erlang, is_reference, 1) -> true; -is_imm_op(erlang, is_tuple, 1) -> true; -is_imm_op(erlang, length, 1) -> true; % never a bignum -is_imm_op(erlang, list_to_atom, 1) -> true; -is_imm_op(erlang, node, 0) -> true; -is_imm_op(erlang, node, 1) -> true; -is_imm_op(erlang, throw, 1) -> true; -is_imm_op(erlang, exit, 1) -> true; -is_imm_op(erlang, error, 1) -> true; -is_imm_op(erlang, error, 2) -> true; -is_imm_op(_M, _F, _A) -> false. diff --git a/lib/hipe/cerl/erl_bif_types.erl b/lib/hipe/cerl/erl_bif_types.erl index bfffb8db41..fe6ab0659c 100644 --- a/lib/hipe/cerl/erl_bif_types.erl +++ b/lib/hipe/cerl/erl_bif_types.erl @@ -770,6 +770,9 @@ type(erlang, length, 1, Xs, Opaques) -> %% Guard bif, needs to be here. type(erlang, map_size, 1, Xs, Opaques) -> type(maps, size, 1, Xs, Opaques); +%% Guard bif, needs to be here. +type(erlang, map_get, 2, Xs, Opaques) -> + type(maps, get, 2, Xs, Opaques); type(erlang, make_fun, 3, Xs, Opaques) -> strict(erlang, make_fun, 3, Xs, fun ([_, _, Arity]) -> @@ -2391,6 +2394,9 @@ arg_types(erlang, length, 1) -> %% Guard bif, needs to be here. arg_types(erlang, map_size, 1) -> [t_map()]; +%% Guard bif, needs to be here. +arg_types(erlang, map_get, 2) -> + [t_map(), t_any()]; arg_types(erlang, make_fun, 3) -> [t_atom(), t_atom(), t_arity()]; arg_types(erlang, make_tuple, 2) -> diff --git a/lib/hipe/cerl/erl_types.erl b/lib/hipe/cerl/erl_types.erl index 8a609ef911..a91da97f93 100644 --- a/lib/hipe/cerl/erl_types.erl +++ b/lib/hipe/cerl/erl_types.erl @@ -217,7 +217,7 @@ ]). %%-define(DO_ERL_TYPES_TEST, true). --compile({no_auto_import,[min/2,max/2]}). +-compile({no_auto_import,[min/2,max/2,map_get/2]}). -ifdef(DO_ERL_TYPES_TEST). -export([test/0]). diff --git a/lib/hipe/doc/src/hipe_app.xml b/lib/hipe/doc/src/hipe_app.xml index aaeb06193d..fc42ecd97d 100644 --- a/lib/hipe/doc/src/hipe_app.xml +++ b/lib/hipe/doc/src/hipe_app.xml @@ -99,6 +99,41 @@ each mode.</p> </item> + <tag>Optimization for <c>receive</c> with unique references</tag> + <item> + <p> + The BEAM compiler can do an optimization when a receive + statement is only waiting for messages containing a reference + created before the receive. All messages that existed in the + queue when the reference was created will be bypassed, as they + cannot possibly contain the reference. HiPE currently has an + optimization similar this, but it is not guaranteed to + bypass all messages. In the worst case scenario it, cannot + bypass any messages at all. + </p> + <p> + An example of this is when <c>gen_server:call()</c> waits for + the reply message. + </p> + </item> + + </taglist> + </section> + <section> + <title>Stability Issues</title> + <taglist> + <tag>Not yielding in <c>receive</c> statements</tag> + <item> + <p>HiPE will not yield in <c>receive</c> statements where + appropriate. If a process have lots of signals in its signal + queue and execute a HiPE compiled <c>receive</c> statement, + the scheduler thread performing the execution may be stuck + in the <c>receive</c> statement for a very long time. This + can in turn cause various severe issues such as for example + prevent the runtime system from being able to release + memory. + </p> + </item> </taglist> </section> <section> diff --git a/lib/hipe/main/hipe.app.src b/lib/hipe/main/hipe.app.src index 1138d72dd2..7350e873aa 100644 --- a/lib/hipe/main/hipe.app.src +++ b/lib/hipe/main/hipe.app.src @@ -26,7 +26,6 @@ cerl_closurean, cerl_hipeify, cerl_lib, - cerl_messagean, cerl_pmatch, cerl_prettypr, cerl_to_icode, diff --git a/lib/hipe/main/hipe.erl b/lib/hipe/main/hipe.erl index acb9b7b062..97814fe217 100644 --- a/lib/hipe/main/hipe.erl +++ b/lib/hipe/main/hipe.erl @@ -669,8 +669,8 @@ run_compiler_1(Name, DisasmFun, IcodeFun, Options) -> {Icode, WholeModule} = IcodeFun(Code, Opts), CompRes = compile_finish(Icode, WholeModule, Opts), compiler_return(CompRes, Parent) - catch error:Error -> - print_crash_message(Name, Error), + catch error:Error:StackTrace -> + print_crash_message(Name, Error, StackTrace), exit(Error) end end), @@ -757,8 +757,8 @@ finalize(OrigList, Mod, Exports, WholeModule, Opts) -> TargetArch = get(hipe_target_arch), {ok, {TargetArch,Bin}} catch - error:Error -> - {error,Error,erlang:get_stacktrace()} + error:Error:StackTrace -> + {error,Error,StackTrace} end end. @@ -843,16 +843,16 @@ finalize_fun_sequential({MFA, Icode}, Opts, Servers) -> {llvm_binary, Binary} -> {MFA, Binary} catch - error:Error -> + error:Error:StackTrace -> ?when_option(verbose, Opts, ?debug_untagged_msg("\n", [])), - print_crash_message(MFA, Error), + print_crash_message(MFA, Error, StackTrace), exit(Error) end. -print_crash_message(What, Error) -> +print_crash_message(What, Error, StackTrace) -> StackFun = fun(_,_,_) -> false end, FormatFun = fun (Term, _) -> io_lib:format("~p", [Term]) end, - StackTrace = lib:format_stacktrace(1, erlang:get_stacktrace(), + StackTrace = lib:format_stacktrace(1, StackTrace, StackFun, FormatFun), WhatS = case What of {M,F,A} -> io_lib:format("~w:~w/~w", [M,F,A]); diff --git a/lib/ic/c_src/oe_ei_encode_atom.c b/lib/ic/c_src/oe_ei_encode_atom.c index 758586d1d4..99a9fe26f0 100644 --- a/lib/ic/c_src/oe_ei_encode_atom.c +++ b/lib/ic/c_src/oe_ei_encode_atom.c @@ -20,28 +20,37 @@ */ #include <ic.h> +#include <string.h> + + +#define DIRTY_ATOM_ENC_MAX(LATIN1_CHARS) ((LATIN1_CHARS)*2 + 3) + int oe_ei_encode_atom(CORBA_Environment *ev, const char *p) { int size = ev->_iout; + size_t len = strlen(p); + + if (DIRTY_ATOM_ENC_MAX(len) >= ev->_outbufsz) { + + ei_encode_atom_len(0,&size,p,len); + + if (size >= ev->_outbufsz) { + char *buf = ev->_outbuf; + int bufsz = ev->_outbufsz + ev->_memchunk; + + while (size >= bufsz) + bufsz += ev->_memchunk; + + if ((buf = realloc(buf, bufsz)) == NULL) { + CORBA_exc_set(ev, CORBA_SYSTEM_EXCEPTION, NO_MEMORY, "End of heap memory while encoding"); + return -1; /* OUT OF MEMORY */ + } - ei_encode_atom(0,&size,p); - - if (size >= ev->_outbufsz) { - char *buf = ev->_outbuf; - int bufsz = ev->_outbufsz + ev->_memchunk; - - while (size >= bufsz) - bufsz += ev->_memchunk; - - if ((buf = realloc(buf, bufsz)) == NULL) { - CORBA_exc_set(ev, CORBA_SYSTEM_EXCEPTION, NO_MEMORY, "End of heap memory while encoding"); - return -1; /* OUT OF MEMORY */ - } - - ev->_outbuf = buf; - ev->_outbufsz = bufsz; + ev->_outbuf = buf; + ev->_outbufsz = bufsz; + } } - return ei_encode_atom(ev->_outbuf,&ev->_iout,p); + return ei_encode_atom_len(ev->_outbuf,&ev->_iout,p,len); } diff --git a/lib/ic/doc/src/notes.xml b/lib/ic/doc/src/notes.xml index fc68ec386c..38cc77ca98 100644 --- a/lib/ic/doc/src/notes.xml +++ b/lib/ic/doc/src/notes.xml @@ -31,7 +31,22 @@ <file>notes.xml</file> </header> - <section><title>IC 4.4.3</title> + <section><title>IC 4.4.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Optimize encoding/decoding for pure 7-bit ascii atoms.</p> + <p> + Own Id: OTP-15023 Aux Id: ERIERL-150 </p> + </item> + </list> + </section> + +</section> + +<section><title>IC 4.4.3</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/ic/vsn.mk b/lib/ic/vsn.mk index b9f1ef7f20..d35d1dce1e 100644 --- a/lib/ic/vsn.mk +++ b/lib/ic/vsn.mk @@ -1 +1 @@ -IC_VSN = 4.4.3 +IC_VSN = 4.4.4 diff --git a/lib/inets/doc/src/Makefile b/lib/inets/doc/src/Makefile index cbfa5c9e30..90c1258d4a 100644 --- a/lib/inets/doc/src/Makefile +++ b/lib/inets/doc/src/Makefile @@ -43,13 +43,10 @@ XML_CHAPTER_FILES = \ inets_services.xml \ http_client.xml \ http_server.xml \ - ftp_client.xml \ notes.xml XML_REF3_FILES = \ inets.xml \ - ftp.xml \ - tftp.xml \ http_uri.xml\ httpc.xml\ httpd.xml \ diff --git a/lib/inets/doc/src/httpc.xml b/lib/inets/doc/src/httpc.xml index 14662f257c..521ad6a015 100644 --- a/lib/inets/doc/src/httpc.xml +++ b/lib/inets/doc/src/httpc.xml @@ -312,8 +312,7 @@ <v>Body = string() | binary()</v> <v>Profile = profile() | pid()</v> <d>When started <c>stand_alone</c> only the pid can be used.</d> - <v>Reason = {connect_failed, term()} | - {send_failed, term()} | term()</v> + <v>Reason = term()</v> </type> <desc> @@ -442,17 +441,22 @@ <tag><c><![CDATA[socket_opts]]></c></tag> <item> - <p>Socket options to be used for this and subsequent - requests.</p> + <p>Socket options to be used for this request.</p> <p>Overrides any value set by function <seealso marker="#set_options-1">set_options</seealso>.</p> <p>The validity of the options is <em>not</em> checked by the HTTP client they are assumed to be correct and passed on to ssl application and inet driver, which may reject - them if they are not correct. Note that the current - implementation assumes the requests to the same host, port - combination will use the same socket options. + them if they are not correct. </p> + <note> + <p> + Persistent connections are not supported when setting the + <c>socket_opts</c> option. When <c>socket_opts</c> is not + set the current implementation assumes the requests to the + same host, port combination will use the same socket options. + </p> + </note> <p>By default the socket options set by function <seealso marker="#set_options-1">set_options/[1,2]</seealso> @@ -625,8 +629,11 @@ to complete. The HTTP/1.1 specification suggests a limit of two persistent connections per server, which is the default value of option <c>max_sessions</c>.</p> + <p> + The current implementation assumes the requests to the same host, port + combination will use the same socket options. + </p> </note> - <marker id="get_options"></marker> </desc> </func> diff --git a/lib/inets/doc/src/inets.xml b/lib/inets/doc/src/inets.xml index 137381cbe9..eb4e51584f 100644 --- a/lib/inets/doc/src/inets.xml +++ b/lib/inets/doc/src/inets.xml @@ -188,10 +188,9 @@ <section> <title>SEE ALSO</title> - <p><seealso marker="ftp">ftp(3)</seealso>, - <seealso marker="httpc">httpc(3)</seealso>, - <seealso marker="httpd">httpd(3)</seealso>, - <seealso marker="tftp">tftp(3)</seealso></p> + <p><seealso marker="httpc">httpc(3)</seealso>, + <seealso marker="httpd">httpd(3)</seealso> + </p> </section> </erlref> diff --git a/lib/inets/doc/src/introduction.xml b/lib/inets/doc/src/introduction.xml index 1af2ef5dae..faf911f188 100644 --- a/lib/inets/doc/src/introduction.xml +++ b/lib/inets/doc/src/introduction.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1997</year><year>2016</year> + <year>1997</year><year>2018</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -22,12 +22,12 @@ </legalnotice> <title>Introduction</title> - <prepared>Ingela Anderton Andin</prepared> + <prepared>Péter Dimitrov</prepared> <responsible></responsible> <docno></docno> <approved></approved> <checked></checked> - <date>2004-09-28</date> + <date>2018-02-28</date> <rev>A</rev> <file>introduction.xml</file> </header> @@ -37,8 +37,6 @@ <p><c>Inets</c> is a container for Internet clients and servers including the following:</p> <list type="bulleted"> - <item>An FTP client</item> - <item>A TFTP client and server</item> <item>An <term id="HTTP"></term> client and server</item> </list> <p>The HTTP client and server are HTTP 1.1 compliant as @@ -50,7 +48,7 @@ <title>Prerequisites</title> <p>It is assumed that the reader is familiar with the Erlang programming language, concepts of OTP, and has a basic - understanding of the FTP, TFTP, and HTTP protocols.</p> + understanding of and HTTP protocol.</p> </section> </chapter> diff --git a/lib/inets/doc/src/notes.xml b/lib/inets/doc/src/notes.xml index 0417e07de8..10dd26322c 100644 --- a/lib/inets/doc/src/notes.xml +++ b/lib/inets/doc/src/notes.xml @@ -33,7 +33,22 @@ <file>notes.xml</file> </header> - <section><title>Inets 6.5</title> + <section><title>Inets 6.5.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix broken options handling in httpc (ERL-441).</p> + <p> + Own Id: OTP-15007</p> + </item> + </list> + </section> + +</section> + +<section><title>Inets 6.5</title> <section><title>Fixed Bugs and Malfunctions</title> <list> @@ -1766,7 +1781,7 @@ <list> <item> <p>[ftpc] Add a config option to specify a - <seealso marker="ftp#dtimeout">data connect timeout</seealso>. + <seealso marker="ftp:ftp#dtimeout">data connect timeout</seealso>. That is how long the ftp client will wait for the server to connect to the data socket. If this timeout occurs, an error will be returned to the caller and the ftp client process will be @@ -2649,10 +2664,10 @@ <item> <p>It is now also possible to start a standalone FTP client process using the re-introduced - <seealso marker="ftp#open">ftp:open</seealso> + <seealso marker="ftp:ftp#open">ftp:open</seealso> function. </p> <p>This is an alternative to starting the client using the - <seealso marker="ftp#service_start">inets service framework</seealso>. </p> + <seealso marker="ftp:ftp#service_start">inets service framework</seealso>. </p> <p>The old <c>ftp:open/1</c>, undocumented, function, caused the client to be hooken into the inets service supervision framework. This is <em>no</em> longer the @@ -2665,10 +2680,10 @@ flag), and only used IPv4 if this did not work. This has now been <em>changed</em>. </p> <p>A new option, - <seealso marker="ftp#ipfamily">ipfamily</seealso>, + <seealso marker="ftp:ftp#ipfamily">ipfamily</seealso>, has been introduced, with the default value <c>inet</c> (IPv4). </p> - <p>See <seealso marker="ftp#open">ftp:open</seealso> + <p>See <seealso marker="ftp:ftp#open">ftp:open</seealso> for more info.</p> <p>*** POTENTIAL INCOMPATIBILITY ***</p> </item> @@ -2702,9 +2717,9 @@ <item> <p>[ftpc] - The - <seealso marker="ftp#ls2">ls/2</seealso> function (LIST command) + <seealso marker="ftp:ftp#ls2">ls/2</seealso> function (LIST command) and the - <seealso marker="ftp#nlist2">nlist/2</seealso> function + <seealso marker="ftp:ftp#nlist2">nlist/2</seealso> function (NLST command) with wildcards did not work properly. </p> diff --git a/lib/inets/doc/src/part.xml b/lib/inets/doc/src/part.xml index f777481b5c..b9c8ed674c 100644 --- a/lib/inets/doc/src/part.xml +++ b/lib/inets/doc/src/part.xml @@ -4,7 +4,7 @@ <part xmlns:xi="http://www.w3.org/2001/XInclude"> <header> <copyright> - <year>2004</year><year>2016</year> + <year>2004</year><year>2018</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -23,9 +23,9 @@ </legalnotice> <title>Inets User's Guide</title> - <prepared>Ingela Anderton Andin</prepared> + <prepared>Péter Dimitrov</prepared> <docno></docno> - <date>2002-09-17</date> + <date>2018-02-28</date> <rev>A</rev> <file>part.sgml</file> </header> @@ -33,8 +33,6 @@ <p>The <c>Inets</c> application provides a set of Internet-related services as follows:</p> <list type="bulleted"> - <item>An FTP client</item> - <item>A TFTP client and server</item> <item>An <term id="HTTP"></term> client and server</item> </list> <p>The HTTP client and server are HTTP 1.1 compliant as @@ -43,7 +41,6 @@ </description> <xi:include href="introduction.xml"/> <xi:include href="inets_services.xml"/> - <xi:include href="ftp_client.xml"/> <xi:include href="http_client.xml"/> <xi:include href="http_server.xml"/> </part> diff --git a/lib/inets/doc/src/ref_man.xml b/lib/inets/doc/src/ref_man.xml index 27021ea09a..58c1a651f9 100644 --- a/lib/inets/doc/src/ref_man.xml +++ b/lib/inets/doc/src/ref_man.xml @@ -4,7 +4,7 @@ <application xmlns:xi="http://www.w3.org/2001/XInclude"> <header> <copyright> - <year>1997</year><year>2015</year> + <year>1997</year><year>2018</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -23,20 +23,16 @@ </legalnotice> <title>Inets Reference Manual</title> - <prepared>Joakim Grebenö</prepared> + <prepared>Péter Dimitrov</prepared> <docno></docno> - <date>1997-07-16</date> + <date>2018-02-28</date> <rev>2.1</rev> <file>ref_man.xml</file> </header> <description> - <p><c>Inets</c> is a container for Internet clients and - servers. An FTP client, an HTTP client and server, and - a TFTP client and server are incorporated in <c>Inets</c>.</p> + <p><c>Inets</c> is a container for an HTTP client and server.</p> </description> <xi:include href="inets.xml"/> - <xi:include href="ftp.xml"/> - <xi:include href="tftp.xml"/> <xi:include href="httpc.xml"/> <xi:include href="httpd.xml"/> <xi:include href="httpd_custom_api.xml"/> diff --git a/lib/inets/src/ftp/ftp_sup.erl b/lib/inets/src/ftp/ftp_sup.erl deleted file mode 100644 index 21dcfb6ab2..0000000000 --- a/lib/inets/src/ftp/ftp_sup.erl +++ /dev/null @@ -1,60 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2004-2016. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% -%% -%%---------------------------------------------------------------------- -%% Purpose: The top supervisor for the ftp hangs under inets_sup. -%%---------------------------------------------------------------------- --module(ftp_sup). - --behaviour(supervisor). - -%% API --export([start_link/0]). --export([start_child/1]). - -%% Supervisor callback --export([init/1]). - -%%%========================================================================= -%%% API -%%%========================================================================= -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - -start_child(Args) -> - supervisor:start_child(?MODULE, Args). - -%%%========================================================================= -%%% Supervisor callback -%%%========================================================================= -init(_) -> - RestartStrategy = simple_one_for_one, - MaxR = 0, - MaxT = 3600, - - Name = undefined, % As simple_one_for_one is used. - StartFunc = {ftp, start_link, []}, - Restart = temporary, % E.g. should not be restarted - Shutdown = 4000, - Modules = [ftp], - Type = worker, - - ChildSpec = {Name, StartFunc, Restart, Shutdown, Type, Modules}, - {ok, {{RestartStrategy, MaxR, MaxT}, [ChildSpec]}}. diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl index 9b09832eb8..c9763507d1 100644 --- a/lib/inets/src/http_client/httpc_handler.erl +++ b/lib/inets/src/http_client/httpc_handler.erl @@ -48,19 +48,17 @@ queue_timer :: reference() | 'undefined' }). --type session_failed() :: {'connect_failed',term()} | {'send_failed',term()}. - -record(state, { request :: request() | 'undefined', - session :: session() | session_failed() | 'undefined', + session :: session() | 'undefined', status_line, % {Version, StatusCode, ReasonPharse} headers :: http_response_h() | 'undefined', body :: binary() | 'undefined', mfa, % {Module, Function, Args} pipeline = queue:new() :: queue:queue(), keep_alive = queue:new() :: queue:queue(), - status, % undefined | new | pipeline | keep_alive | close | {ssl_tunnel, Request} + status :: undefined | new | pipeline | keep_alive | close | {ssl_tunnel, request()}, canceled = [], % [RequestId] max_header_size = nolimit :: nolimit | integer(), max_body_size = nolimit :: nolimit | integer(), @@ -255,8 +253,8 @@ handle_call(Request, From, State) -> Result -> Result catch - _:Reason -> - {stop, {shutdown, Reason} , State} + Class:Reason:ST -> + {stop, {shutdown, {{Class, Reason}, ST}}, State} end. @@ -271,8 +269,8 @@ handle_cast(Msg, State) -> Result -> Result catch - _:Reason -> - {stop, {shutdown, Reason} , State} + Class:Reason:ST -> + {stop, {shutdown, {{Class, Reason}, ST}}, State} end. %%-------------------------------------------------------------------- @@ -286,8 +284,8 @@ handle_info(Info, State) -> Result -> Result catch - _:Reason -> - {stop, {shutdown, Reason} , State} + Class:Reason:ST -> + {stop, {shutdown, {{Class, Reason}, ST}}, State} end. %%-------------------------------------------------------------------- @@ -295,23 +293,6 @@ handle_info(Info, State) -> %% Description: Shutdown the httpc_handler %%-------------------------------------------------------------------- -%% Init error there is no socket to be closed. -terminate(normal, - #state{request = Request, - session = {send_failed, _} = Reason} = State) -> - maybe_send_answer(Request, - httpc_response:error(Request, Reason), - State), - ok; - -terminate(normal, - #state{request = Request, - session = {connect_failed, _} = Reason} = State) -> - maybe_send_answer(Request, - httpc_response:error(Request, Reason), - State), - ok; - terminate(normal, #state{session = undefined}) -> ok; @@ -588,11 +569,11 @@ do_handle_info({Proto, _Socket, Data}, activate_once(Session), {noreply, State#state{mfa = NewMFA}} catch - _:Reason -> + Class:Reason:ST -> ClientReason = {could_not_parse_as_http, Data}, ClientErrMsg = httpc_response:error(Request, ClientReason), NewState = answer_request(Request, ClientErrMsg, State), - {stop, {shutdown, Reason}, NewState} + {stop, {shutdown, {{Class, Reason}, ST}}, NewState} end; do_handle_info({Proto, Socket, Data}, @@ -858,7 +839,7 @@ connect_and_send_first_request(Address, Request, #state{options = Options0} = St self() ! {init_error, error_sending, httpc_response:error(Request, Reason)}, {ok, State#state{request = Request, - session = #session{socket = Socket}}} + session = Session}} end; {error, Reason} -> self() ! {init_error, error_connecting, @@ -1058,15 +1039,15 @@ handle_response(#state{status = new} = State) -> ?hcrd("handle response - status = new", []), handle_response(try_to_enable_pipeline_or_keep_alive(State)); -handle_response(#state{request = Request, - status = Status, - session = Session, - status_line = StatusLine, - headers = Headers, - body = Body, - options = Options, - profile_name = ProfileName} = State) - when Status =/= new -> +handle_response(#state{status = Status0} = State0) when Status0 =/= new -> + State = handle_server_closing(State0), + #state{request = Request, + session = Session, + status_line = StatusLine, + headers = Headers, + body = Body, + options = Options, + profile_name = ProfileName} = State, handle_cookies(Headers, Request, Options, ProfileName), case httpc_response:result({StatusLine, Headers, Body}, Request) of %% 100-continue @@ -1330,6 +1311,14 @@ try_to_enable_pipeline_or_keep_alive( State#state{status = close} end. +handle_server_closing(State = #state{status = close}) -> State; +handle_server_closing(State = #state{headers = undefined}) -> State; +handle_server_closing(State = #state{headers = Headers}) -> + case httpc_response:is_server_closing(Headers) of + true -> State#state{status = close}; + false -> State + end. + answer_request(#request{id = RequestId, from = From} = Request, Msg, #state{session = Session, timers = Timers, diff --git a/lib/inets/src/http_client/httpc_manager.erl b/lib/inets/src/http_client/httpc_manager.erl index 7b8d7875de..c3404dbb37 100644 --- a/lib/inets/src/http_client/httpc_manager.erl +++ b/lib/inets/src/http_client/httpc_manager.erl @@ -750,8 +750,26 @@ handle_request(#request{settings = start_handler(NewRequest#request{headers = NewHeaders}, State), {reply, {ok, NewRequest#request.id}, State}; -handle_request(Request, State = #state{options = Options}) -> +%% Simple socket options handling (ERL-441). +%% +%% TODO: Refactor httpc to enable sending socket options in requests +%% using persistent connections. This workaround opens a new +%% connection for each request with non-empty socket_opts. +handle_request(Request0 = #request{socket_opts = SocketOpts}, + State0 = #state{options = Options0}) + when is_list(SocketOpts) andalso length(SocketOpts) > 0 -> + Request = handle_cookies(generate_request_id(Request0), State0), + Options = convert_options(SocketOpts, Options0), + State = State0#state{options = Options}, + Headers = + (Request#request.headers)#http_request_h{connection + = "close"}, + %% Reset socket_opts to avoid setopts failure. + start_handler(Request#request{headers = Headers, socket_opts = []}, State), + %% Do not change the state + {reply, {ok, Request#request.id}, State0}; +handle_request(Request, State = #state{options = Options}) -> NewRequest = handle_cookies(generate_request_id(Request), State), SessionType = session_type(Options), case select_session(Request#request.method, @@ -775,6 +793,18 @@ handle_request(Request, State = #state{options = Options}) -> {reply, {ok, NewRequest#request.id}, State}. +%% Convert Request options to State options +convert_options([], Options) -> + Options; +convert_options([{ipfamily, Value}|T], Options) -> + convert_options(T, Options#options{ipfamily = Value}); +convert_options([{ip, Value}|T], Options) -> + convert_options(T, Options#options{ip = Value}); +convert_options([{port, Value}|T], Options) -> + convert_options(T, Options#options{port = Value}); +convert_options([Option|T], Options = #options{socket_opts = SocketOpts}) -> + convert_options(T, Options#options{socket_opts = SocketOpts ++ [Option]}). + start_handler(#request{id = Id, from = From} = Request, #state{profile_name = ProfileName, diff --git a/lib/inets/src/http_client/httpc_request.erl b/lib/inets/src/http_client/httpc_request.erl index 89872a3831..641b6559de 100644 --- a/lib/inets/src/http_client/httpc_request.erl +++ b/lib/inets/src/http_client/httpc_request.erl @@ -190,35 +190,11 @@ is_client_closing(Headers) -> %%%======================================================================== post_data(Method, Headers, {ContentType, Body}, HeadersAsIs) when (Method =:= post) - orelse (Method =:= put) - orelse (Method =:= patch) - orelse (Method =:= delete) -> - - NewBody = case Headers#http_request_h.expect of - "100-continue" -> - ""; - _ -> - Body - end, - - NewHeaders = case HeadersAsIs of - [] -> - Headers#http_request_h{ - 'content-type' = ContentType, - 'content-length' = case body_length(Body) of - undefined -> - % on upload streaming the caller must give a - % value to the Content-Length header - % (or use chunked Transfer-Encoding) - Headers#http_request_h.'content-length'; - Len when is_list(Len) -> - Len - end - }; - _ -> - HeadersAsIs - end, - + orelse (Method =:= put) + orelse (Method =:= patch) + orelse (Method =:= delete) -> + NewBody = update_body(Headers, Body), + NewHeaders = update_headers(Headers, ContentType, Body, HeadersAsIs), {NewHeaders, NewBody}; post_data(_, Headers, _, []) -> @@ -226,14 +202,39 @@ post_data(_, Headers, _, []) -> post_data(_, _, _, HeadersAsIs = [_|_]) -> {HeadersAsIs, ""}. +update_body(Headers, Body) -> + case Headers#http_request_h.expect of + "100-continue" -> + ""; + _ -> + Body + end. + +update_headers(Headers, ContentType, Body, []) -> + case Body of + [] -> + Headers#http_request_h{'content-length' = "0"}; + <<>> -> + Headers#http_request_h{'content-length' = "0"}; + {Fun, _Acc} when is_function(Fun, 1) -> + %% A client MUST NOT generate a 100-continue expectation in a request + %% that does not include a message body. This implies that either the + %% Content-Length or the Transfer-Encoding header MUST be present. + %% DO NOT send content-type when Body is empty. + Headers#http_request_h{'content-type' = ContentType}; + _ -> + Headers#http_request_h{ + 'content-length' = body_length(Body), + 'content-type' = ContentType} + end; +update_headers(_, _, _, HeadersAsIs) -> + HeadersAsIs. + body_length(Body) when is_binary(Body) -> integer_to_list(size(Body)); body_length(Body) when is_list(Body) -> - integer_to_list(length(Body)); - -body_length({DataFun, _Acc}) when is_function(DataFun, 1) -> - undefined. + integer_to_list(length(Body)). method(Method) -> http_util:to_upper(atom_to_list(Method)). diff --git a/lib/inets/src/http_client/httpc_response.erl b/lib/inets/src/http_client/httpc_response.erl index 58ab9144df..92dc9b0e02 100644 --- a/lib/inets/src/http_client/httpc_response.erl +++ b/lib/inets/src/http_client/httpc_response.erl @@ -83,7 +83,6 @@ whole_body(Body, Length) -> %% result(Response, Request) -> %% Response - {StatusLine, Headers, Body} %% Request - #request{} -%% Session - #tcp_session{} %% %% Description: Checks the status code ... %%------------------------------------------------------------------------- diff --git a/lib/inets/src/http_server/httpd.erl b/lib/inets/src/http_server/httpd.erl index 540e68e749..1eaa1c930a 100644 --- a/lib/inets/src/http_server/httpd.erl +++ b/lib/inets/src/http_server/httpd.erl @@ -36,7 +36,13 @@ ]). %% API --export([parse_query/1, reload_config/2, info/1, info/2, info/3]). +-export([ + parse_query/1, + reload_config/2, + info/1, + info/2, + info/3 + ]). %%%======================================================================== %%% API @@ -49,13 +55,24 @@ parse_query(String) -> reload_config(Config = [Value| _], Mode) when is_tuple(Value) -> do_reload_config(Config, Mode); reload_config(ConfigFile, Mode) -> - case httpd_conf:load(ConfigFile) of - {ok, ConfigList} -> - do_reload_config(ConfigList, Mode); - Error -> - Error + try file:consult(ConfigFile) of + {ok, [PropList]} -> + %% Erlang terms format + do_reload_config(PropList, Mode); + {error, _ } -> + %% Apache format + case httpd_conf:load(ConfigFile) of + {ok, ConfigList} -> + do_reload_config(ConfigList, Mode); + Error -> + Error + end + catch + exit:_ -> + throw({error, {could_not_consult_proplist_file, ConfigFile}}) end. + info(Pid) when is_pid(Pid) -> info(Pid, []). diff --git a/lib/inets/src/http_server/mod_alias.erl b/lib/inets/src/http_server/mod_alias.erl index 0333076546..8f0b92710e 100644 --- a/lib/inets/src/http_server/mod_alias.erl +++ b/lib/inets/src/http_server/mod_alias.erl @@ -163,28 +163,24 @@ longest_match([], _RequestURI, _LongestNo, LongestAlias) -> real_script_name(_ConfigDB, _RequestURI, []) -> not_a_script; - -real_script_name(ConfigDB, RequestURI, [{MP,Replacement} | Rest]) - when element(1, MP) =:= re_pattern -> - case re:run(RequestURI, MP, [{capture, none}]) of - match -> - ActualName = - re:replace(RequestURI, MP, Replacement, [{return,list}]), - httpd_util:split_script_path(default_index(ConfigDB, ActualName)); - nomatch -> - real_script_name(ConfigDB, RequestURI, Rest) - end; - real_script_name(ConfigDB, RequestURI, [{FakeName,RealName} | Rest]) -> case re:run(RequestURI, "^" ++ FakeName, [{capture, none}]) of match -> - ActualName = + ActualName0 = re:replace(RequestURI, "^" ++ FakeName, RealName, [{return,list}]), + ActualName = abs_script_path(ConfigDB, ActualName0), httpd_util:split_script_path(default_index(ConfigDB, ActualName)); nomatch -> real_script_name(ConfigDB, RequestURI, Rest) end. +%% ERL-574: relative path in script_alias property results in malformed url +abs_script_path(ConfigDB, [$.|_] = RelPath) -> + Root = httpd_util:lookup(ConfigDB, server_root), + Root ++ "/" ++ RelPath; +abs_script_path(_, RelPath) -> + RelPath. + %% default_index default_index(ConfigDB, Path) -> diff --git a/lib/inets/src/inets_app/Makefile b/lib/inets/src/inets_app/Makefile index eb0098dbee..fad2fefe2f 100644 --- a/lib/inets/src/inets_app/Makefile +++ b/lib/inets/src/inets_app/Makefile @@ -48,7 +48,9 @@ MODULES = \ inets_app \ inets_sup \ inets_trace \ - inets_lib + inets_lib \ + inets_ftp_wrapper \ + inets_tftp_wrapper INTERNAL_HRL_FILES = inets_internal.hrl EXTERNAL_HRL_FILES = ../../include/httpd.hrl \ diff --git a/lib/inets/src/inets_app/inets.app.src b/lib/inets/src/inets_app/inets.app.src index eb4be932ac..dfe773e7fe 100644 --- a/lib/inets/src/inets_app/inets.app.src +++ b/lib/inets/src/inets_app/inets.app.src @@ -30,10 +30,7 @@ inets_lib, %% FTP - ftp, - ftp_progress, - ftp_response, - ftp_sup, + inets_ftp_wrapper, %% HTTP client: httpc, @@ -101,13 +98,7 @@ mod_trace, %% TFTP - tftp, - tftp_binary, - tftp_engine, - tftp_file, - tftp_lib, - tftp_logger, - tftp_sup + inets_tftp_wrapper ]}, {registered,[inets_sup, httpc_manager]}, %% If the "new" ssl is used then 'crypto' must be started before inets. diff --git a/lib/inets/src/inets_app/inets.erl b/lib/inets/src/inets_app/inets.erl index 2d380012d7..450adf1a02 100644 --- a/lib/inets/src/inets_app/inets.erl +++ b/lib/inets/src/inets_app/inets.erl @@ -465,13 +465,19 @@ call_service(Service, Call, Args) -> exit:{noproc, _} -> {error, inets_not_started} end. - + +%% Obsolete! Kept for backward compatiblity! +%% TFTP application has been moved out from inets service_module(tftpd) -> - tftp; + inets_tftp_wrapper; service_module(tftpc) -> - tftp; + inets_tftp_wrapper; +service_module(tftp) -> + inets_tftp_wrapper; +%% Obsolete! Kept for backward compatiblity! +%% FTP application has been moved out from inets service_module(ftpc) -> - ftp; + inets_ftp_wrapper; service_module(Service) -> Service. diff --git a/lib/ssh/src/ssh_server_key.erl b/lib/inets/src/inets_app/inets_ftp_wrapper.erl index 2ce0c7e3fe..e350a490f7 100644 --- a/lib/ssh/src/ssh_server_key.erl +++ b/lib/inets/src/inets_app/inets_ftp_wrapper.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2011-2016. All Rights Reserved. +%% Copyright Ericsson AB 2006-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -17,18 +17,32 @@ %% %% %CopyrightEnd% %% +-module(inets_ftp_wrapper). --module(ssh_server_key). --include_lib("public_key/include/public_key.hrl"). --include("ssh.hrl"). +-export([start_standalone/1, + start_service/1, + stop_service/1, + services/0, + service_info/1]). --type ssh_algorithm() :: string(). --callback host_key(Algorithm :: ssh_algorithm(), Options :: list()) -> - {ok, [{public_key(), Attributes::list()}]} | public_key() - | {error, string()}. +start_standalone(Options) -> + ftp:start_standalone(Options). --callback is_auth_key(Key :: public_key(), User :: string(), - Algorithm :: ssh_algorithm(), Options :: list()) -> - boolean(). + +start_service(Options) -> + application:ensure_started(ftp), + ftp:start_service(Options). + + +stop_service(Pid) -> + ftp:stop_service(Pid). + + +services() -> + []. + + +service_info(_) -> + []. diff --git a/lib/inets/src/inets_app/inets_sup.erl b/lib/inets/src/inets_app/inets_sup.erl index d8ae7eff26..22c928f9f9 100644 --- a/lib/inets/src/inets_app/inets_sup.erl +++ b/lib/inets/src/inets_app/inets_sup.erl @@ -61,19 +61,7 @@ children() -> Services = get_services(), HttpdServices = [Service || Service <- Services, is_httpd(Service)], HttpcServices = [Service || Service <- Services, is_httpc(Service)], - TftpdServices = [Service || Service <- Services, is_tftpd(Service)], - [ftp_child_spec(), httpc_child_spec(HttpcServices), - httpd_child_spec(HttpdServices), tftpd_child_spec(TftpdServices)]. - -ftp_child_spec() -> - Name = ftp_sup, - StartFunc = {ftp_sup, start_link, []}, - Restart = permanent, - Shutdown = infinity, - Modules = [ftp_sup], - Type = supervisor, - {Name, StartFunc, Restart, Shutdown, Type, Modules}. - + [httpc_child_spec(HttpcServices), httpd_child_spec(HttpdServices)]. httpc_child_spec(HttpcServices0) -> HttpcServices = default_profile(HttpcServices0, []), @@ -94,15 +82,6 @@ httpd_child_spec(HttpdServices) -> Type = supervisor, {Name, StartFunc, Restart, Shutdown, Type, Modules}. -tftpd_child_spec(TftpServices) -> - Name = tftp_sup, - StartFunc = {tftp_sup, start_link, [TftpServices]}, - Restart = permanent, - Shutdown = infinity, - Modules = [tftp_sup], - Type = supervisor, - {Name, StartFunc, Restart, Shutdown, Type, Modules}. - is_httpd({httpd, _}) -> true; is_httpd({httpd, _, _}) -> @@ -115,11 +94,6 @@ is_httpc({httpc, _}) -> is_httpc(_) -> false. -is_tftpd({tftpd, _}) -> - true; -is_tftpd(_) -> - false. - default_profile([], Acc) -> [{httpc, {default, only_session_cookies}} | Acc]; default_profile([{httpc, {default, _}} | _] = Profiles, Acc) -> diff --git a/lib/ssh/src/ssh_client_key.erl b/lib/inets/src/inets_app/inets_tftp_wrapper.erl index 5296ac2a02..1e5deb234b 100644 --- a/lib/ssh/src/ssh_client_key.erl +++ b/lib/inets/src/inets_app/inets_tftp_wrapper.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2011-2016. All Rights Reserved. +%% Copyright Ericsson AB 2006-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -17,19 +17,32 @@ %% %% %CopyrightEnd% %% +-module(inets_tftp_wrapper). --module(ssh_client_key). --include_lib("public_key/include/public_key.hrl"). --include("ssh.hrl"). +-export([start_standalone/1, + start_service/1, + stop_service/1, + services/0, + service_info/1]). --callback is_host_key(Key :: public_key(), Host :: string(), - Algorithm :: 'ssh-rsa'| 'ssh-dsa'| atom(), Options :: proplists:proplist()) -> - boolean(). --callback user_key(Algorithm :: 'ssh-rsa'| 'ssh-dsa'| atom(), Options :: list()) -> - {ok, PrivateKey :: term()} | {error, string()}. +start_standalone(Options) -> + tftp:start_standalone(Options). --callback add_host_key(Host :: string(), PublicKey :: term(), Options :: list()) -> - ok | {error, Error::term()}. +start_service(Options) -> + application:ensure_started(tftp), + tftp:start_service(Options). + + +stop_service(Pid) -> + tftp:stop_service(Pid). + + +services() -> + []. + + +service_info(_) -> + []. diff --git a/lib/inets/src/subdirs.mk b/lib/inets/src/subdirs.mk index 9f2a0079f2..e9f4de959c 100644 --- a/lib/inets/src/subdirs.mk +++ b/lib/inets/src/subdirs.mk @@ -1,3 +1,3 @@ #-*-makefile-*- ; force emacs to enter makefile-mode -SUB_DIRECTORIES = inets_app http_lib http_client http_server ftp tftp +SUB_DIRECTORIES = inets_app http_lib http_client http_server diff --git a/lib/inets/test/Makefile b/lib/inets/test/Makefile index 99a7e6a9db..0e33b72095 100644 --- a/lib/inets/test/Makefile +++ b/lib/inets/test/Makefile @@ -44,8 +44,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN) INCLUDES = -I. \ -I$(ERL_TOP)/lib/inets/src/inets_app \ -I$(ERL_TOP)/lib/inets/src/http_lib \ - -I$(ERL_TOP)/lib/inets/src/http_client \ - -I$(ERL_TOP)/lib/inets/src/ftp + -I$(ERL_TOP)/lib/inets/src/http_client CP = cp @@ -68,34 +67,6 @@ INETS_FLAGS = -Dinets_data_dir='"$(INETS_DATA_DIR)"' \ ### ### test suite debug flags ### -ifeq ($(FTP_DEBUG_CLIENT),) - FTP_DEBUG_CLIENT = y -endif - -ifeq ($(FTP_DEBUG_CLIENT),) - FTP_FLAGS += -Dftp_debug_client -endif - -ifeq ($(FTP_TRACE_CLIENT),) - FTP_DEBUG_CLIENT = y -endif - -ifeq ($(FTP_TRACE_CLIENT),y) - FTP_FLAGS += -Dftp_trace_client -endif - -ifneq ($(FTP_DEBUG),) - FTP_DEBUG = s -endif - -ifeq ($(FTP_DEBUG),l) - FTP_FLAGS += -Dftp_log -endif - -ifeq ($(FTP_DEBUG),d) - FTP_FLAGS += -Dftp_debug -Dftp_log -endif - ifeq ($(INETS_DEBUG),) INETS_DEBUG = d endif @@ -151,8 +122,6 @@ MODULES = \ inets_test_lib \ erl_make_certs \ make_certs \ - ftp_SUITE \ - ftp_format_SUITE \ http_format_SUITE \ httpc_SUITE \ httpc_cookie_SUITE \ @@ -169,8 +138,6 @@ MODULES = \ httpd_test_lib \ inets_sup_SUITE \ inets_SUITE \ - tftp_test_lib \ - tftp_SUITE \ uri_SUITE \ inets_socketwrap_SUITE @@ -179,10 +146,8 @@ EBIN = . HRL_FILES = inets_test_lib.hrl \ inets_internal.hrl \ - ftp_internal.hrl \ httpc_internal.hrl \ - http_internal.hrl \ - tftp_test_lib.hrl + http_internal.hrl ERL_FILES = $(MODULES:%=%.erl) @@ -197,18 +162,15 @@ INETS_FILES = inets.config $(INETS_SPECS) # SUB_SUITES = \ # inets_sup_suite \ # inets_httpd_suite \ -# inets_httpc_suite \ -# inets_ftp_suite \ -# inets_tftp_suite +# inets_httpc_suite INETS_DATADIRS = inets_SUITE_data inets_socketwrap_SUITE_data HTTPD_DATADIRS = httpd_test_data httpd_SUITE_data httpd_basic_SUITE_data old_httpd_SUITE_data httpd_bench_SUITE_data HTTPC_DATADIRS = httpc_SUITE_data httpc_proxy_SUITE_data -FTP_DATADIRS = ftp_SUITE_data -DATADIRS = $(INETS_DATADIRS) $(HTTPD_DATADIRS) $(HTTPC_DATADIRS) $(FTP_DATADIRS) +DATADIRS = $(INETS_DATADIRS) $(HTTPD_DATADIRS) $(HTTPC_DATADIRS) EMAKEFILE = Emakefile MAKE_EMAKE = $(wildcard $(ERL_TOP)/make/make_emakefile) @@ -238,7 +200,6 @@ RELTESTSYSBINDIR = $(RELTESTSYSALLDATADIR)/bin # ---------------------------------------------------- ERL_COMPILE_FLAGS += \ $(INCLUDES) \ - $(FTP_FLAGS) \ $(INETS_FLAGS) # ---------------------------------------------------- @@ -334,11 +295,3 @@ info: @echo "INETS_PRIV_DIR = $(INETS_PRIV_DIR)" @echo "INETS_ROOT = $(INETS_ROOT)" @echo "INETS_FLAGS = $(INETS_FLAGS)" - @echo "FTP_FLAGS = $(FTP_FLAGS)" - -tftp: - erlc $(ERL_COMPILE_FLAGS) tftp_test_lib.erl tftp_SUITE.erl && erl -pa ../../inets/ebin -s tftp_SUITE t -s erlang halt - -tftp_work: - echo "tftp_test_lib:t([{tftp_SUITE, all}])." - erlc $(ERL_COMPILE_FLAGS) tftp_test_lib.erl tftp_SUITE.erl && erl -pa ../../inets/ebin diff --git a/lib/inets/test/ftp_internal.hrl b/lib/inets/test/ftp_internal.hrl deleted file mode 120000 index af57081f14..0000000000 --- a/lib/inets/test/ftp_internal.hrl +++ /dev/null @@ -1 +0,0 @@ -../src/ftp/ftp_internal.hrl
\ No newline at end of file diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index 38705372c9..47c7ffd190 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -53,6 +53,7 @@ suite() -> all() -> [ {group, http}, + {group, http_ipv6}, {group, sim_http}, {group, http_internal}, {group, http_unix_socket}, @@ -64,10 +65,11 @@ all() -> groups() -> [ {http, [], real_requests()}, + {http_ipv6, [], [request_options]}, %% process_leak_on_keepalive is depending on stream_fun_server_close %% and it shall be the last test case in the suite otherwise cookie %% will fail. - {sim_http, [], only_simulated() ++ [process_leak_on_keepalive]}, + {sim_http, [], only_simulated() ++ server_closing_connection() ++ [process_leak_on_keepalive]}, {http_internal, [], real_requests_esi()}, {http_unix_socket, [], simulated_unix_socket()}, {https, [], real_requests()}, @@ -151,9 +153,16 @@ only_simulated() -> relaxed, multipart_chunks, get_space, + delete_no_body, stream_fun_server_close ]. +server_closing_connection() -> + [ + server_closing_connection_on_first_response, + server_closing_connection_on_second_response + ]. + misc() -> [ server_does_not_exist, @@ -207,6 +216,16 @@ init_per_group(http_unix_socket = Group, Config0) -> Port = server_start(Group, server_config(Group, Config)), [{port, Port} | Config] end; +init_per_group(http_ipv6 = Group, Config0) -> + case is_ipv6_supported() of + true -> + start_apps(Group), + Config = proplists:delete(port, Config0), + Port = server_start(Group, server_config(Group, Config)), + [{port, Port} | Config]; + false -> + {skip, "Host does not support IPv6"} + end; init_per_group(Group, Config0) -> start_apps(Group), Config = proplists:delete(port, Config0), @@ -233,7 +252,7 @@ init_per_testcase(pipeline, Config) -> init_per_testcase(persistent_connection, Config) -> inets:start(httpc, [{profile, persistent}]), httpc:set_options([{keep_alive_timeout, 50000}, - {max_keep_alive_length, 3}], persistent_connection), + {max_keep_alive_length, 3}], persistent), Config; init_per_testcase(wait_for_whole_response, Config) -> @@ -252,10 +271,38 @@ end_per_testcase(pipeline, _Config) -> inets:stop(httpc, pipeline); end_per_testcase(persistent_connection, _Config) -> inets:stop(httpc, persistent); +end_per_testcase(Case, Config) + when Case == server_closing_connection_on_first_response; + Case == server_closing_connection_on_second_response -> + %% Test case uses at most one session. Ensure no leftover + %% sessions left behind. + {_, Status} = proplists:lookup(tc_status, Config), + ShallCleanup = case Status of + ok -> true; + {failed, _} -> true; + {skipped, _} -> false + end, + if ShallCleanup =:= true -> + httpc:request(url(group_name(Config), "/just_close.html", Config)), + ok; + true -> + ct:pal("Not cleaning up because test case status was ~p", [Status]), + ok + end; end_per_testcase(_Case, _Config) -> ok. +is_ipv6_supported() -> + case gen_udp:open(0, [inet6]) of + {ok, Socket} -> + gen_udp:close(Socket), + true; + _ -> + false + end. + + %%-------------------------------------------------------------------- %% Test Cases -------------------------------------------------------- %%-------------------------------------------------------------------- @@ -1275,6 +1322,53 @@ stream_fun_server_close(Config) when is_list(Config) -> end. %%-------------------------------------------------------------------- +server_closing_connection_on_first_response() -> + [{doc, "Client receives \"Connection: close\" on first response." + "A client that receives a \"close\" connection option MUST cease sending" + "requests on that connection and close the connection after reading" + "the response message containing the \"close\""}]. +server_closing_connection_on_first_response(Config) when is_list(Config) -> + ReqSrvSendOctFun = + fun(V, U, S) -> + {ok, {{V, S, _}, Headers0, []}} = + httpc:request(get, {U, []}, [{version, V}], []), + {_, SendOctStr} = + proplists:lookup("x-socket-stat-send-oct", Headers0), + list_to_integer(SendOctStr) + end, + V = "HTTP/1.1", + Url0 = url(group_name(Config), "/http_1_1_send_oct.html", Config), + Url1 = url(group_name(Config), "/http_1_1_send_oct_and_connection_close.html", Config), + %% Test case assumes at most one reusable past session. + _ = ReqSrvSendOctFun(V, Url1, 204), + 0 = ReqSrvSendOctFun(V, Url0, 204), + ok. + +%%-------------------------------------------------------------------- +server_closing_connection_on_second_response() -> + [{doc, "Client receives \"Connection: close\" on second response." + "A client that receives a \"close\" connection option MUST cease sending" + "requests on that connection and close the connection after reading" + "the response message containing the \"close\""}]. +server_closing_connection_on_second_response(Config) when is_list(Config) -> + ReqSrvSendOctFun = + fun(V, U, S) -> + {ok, {{V, S, _}, Headers0, []}} = + httpc:request(get, {U, []}, [{version, V}], []), + {_, SendOctStr} = + proplists:lookup("x-socket-stat-send-oct", Headers0), + list_to_integer(SendOctStr) + end, + V = "HTTP/1.1", + Url0 = url(group_name(Config), "/http_1_1_send_oct.html", Config), + Url1 = url(group_name(Config), "/http_1_1_send_oct_and_connection_close.html", Config), + %% Test case assumes no reusable past sessions. + SendOct0 = 0 = ReqSrvSendOctFun(V, Url0, 204), + case ReqSrvSendOctFun(V, Url1, 204) of SendOct1 when SendOct1 > SendOct0 -> ok end, + 0 = ReqSrvSendOctFun(V, Url0, 204), + ok. + +%%-------------------------------------------------------------------- slow_connection() -> [{doc, "Test that a request on a slow keep-alive connection won't crash the httpc_manager"}]. slow_connection(Config) when is_list(Config) -> @@ -1305,6 +1399,26 @@ unix_domain_socket(Config) when is_list(Config) -> {ok, {{_,200,_}, [_ | _], _}} = httpc:request(get, {URL, []}, [], []). +%%------------------------------------------------------------------------- +delete_no_body(doc) -> + ["Test that a DELETE request without Body does not send a Content-Type header - Solves ERL-536"]; +delete_no_body(Config) when is_list(Config) -> + URL = url(group_name(Config), "/delete_no_body.html", Config), + %% Simulated server replies 500 if 'Content-Type' header is present + {ok, {{_,200,_}, _, _}} = + httpc:request(delete, {URL, []}, [], []), + {ok, {{_,500,_}, _, _}} = + httpc:request(delete, {URL, [], "text/plain", "TEST"}, [], []). + +%%-------------------------------------------------------------------- +request_options() -> + [{doc, "Test http get request with socket options against local server (IPv6)"}]. +request_options(Config) when is_list(Config) -> + Request = {url(group_name(Config), "/dummy.html", Config), []}, + {ok, {{_,200,_}, [_ | _], _ = [_ | _]}} = httpc:request(get, Request, [], + [{socket_opts,[{ipfamily, inet6}]}]), + {error,{failed_connect,_ }} = httpc:request(get, Request, [], []). + %%-------------------------------------------------------------------- @@ -1394,6 +1508,9 @@ url(http, End, Config) -> Port = proplists:get_value(port, Config), {ok,Host} = inet:gethostname(), ?URL_START ++ Host ++ ":" ++ integer_to_list(Port) ++ End; +url(http_ipv6, End, Config) -> + Port = proplists:get_value(port, Config), + ?URL_START ++ "[::1]" ++ ":" ++ integer_to_list(Port) ++ End; url(https, End, Config) -> Port = proplists:get_value(port, Config), {ok,Host} = inet:gethostname(), @@ -1438,7 +1555,11 @@ server_start(http_unix_socket, Config) -> {_Pid, Port} = http_test_lib:dummy_server(unix_socket, Inet, [{content_cb, ?MODULE}, {unix_socket, Socket}]), Port; - +server_start(http_ipv6, HttpdConfig) -> + {ok, Pid} = inets:start(httpd, HttpdConfig), + Serv = inets:services_info(), + {value, {_, _, Info}} = lists:keysearch(Pid, 2, Serv), + proplists:get_value(port, Info); server_start(_, HttpdConfig) -> {ok, Pid} = inets:start(httpd, HttpdConfig), Serv = inets:services_info(), @@ -1457,6 +1578,17 @@ server_config(http, Config) -> {mime_type, "text/plain"}, {script_alias, {"/cgi-bin/", filename:join(ServerRoot, "cgi-bin") ++ "/"}} ]; +server_config(http_ipv6, Config) -> + ServerRoot = proplists:get_value(server_root, Config), + [{port, 0}, + {server_name,"httpc_test"}, + {server_root, ServerRoot}, + {document_root, proplists:get_value(doc_root, Config)}, + {bind_address, {0,0,0,0,0,0,0,1}}, + {ipfamily, inet6}, + {mime_type, "text/plain"}, + {script_alias, {"/cgi-bin/", filename:join(ServerRoot, "cgi-bin") ++ "/"}} + ]; server_config(http_internal, Config) -> ServerRoot = proplists:get_value(server_root, Config), [{port, 0}, @@ -1811,6 +1943,13 @@ auth_header([{"authorization", Value} | _]) -> auth_header([_ | Tail]) -> auth_header(Tail). +content_type_header([]) -> + not_found; +content_type_header([{"content-type", Value}|_]) -> + {ok, string:strip(Value)}; +content_type_header([_|T]) -> + content_type_header(T). + handle_auth("Basic " ++ UserInfo, Challange, DefaultResponse) -> case string:tokens(base64:decode_to_string(UserInfo), ":") of ["alladin", "sesame"] = Auth -> @@ -2232,10 +2371,40 @@ handle_uri("GET","/v1/kv/foo",_,_,_,_) -> "Content-Length: 24\r\n" ++ "Content-Type: application/json\r\n\r\n" ++ "[{\"Value\": \"aGVsbG8=\"}]\n"; - +handle_uri(_,"/http_1_1_send_oct.html",_,_,Socket,_) -> + "HTTP/1.1 204 No Content\r\n" ++ + "X-Socket-Stat-Send-Oct: " ++ integer_to_list(get_stat(Socket, send_oct)) ++ "\r\n" ++ + "\r\n"; +handle_uri(_,"/http_1_1_send_oct_and_connection_close.html",_,_,Socket,_) -> + "HTTP/1.1 204 No Content\r\n" ++ + "X-Socket-Stat-Send-Oct: " ++ integer_to_list(get_stat(Socket, send_oct)) ++ "\r\n" ++ + "Connection: close\r\n" ++ + "\r\n"; +handle_uri(_,"/delete_no_body.html", _,Headers,_, DefaultResponse) -> + Error = "HTTP/1.1 500 Internal Server Error\r\n" ++ + "Content-Length:0\r\n\r\n", + case content_type_header(Headers) of + {ok, _} -> + Error; + not_found -> + DefaultResponse + end; handle_uri(_,_,_,_,_,DefaultResponse) -> DefaultResponse. +get_stat(S, Opt) -> + case getstat(S, [Opt]) of + {ok, [{Opt, V}]} when is_integer(V) -> + V; + {error, _} = E -> + E + end. + +getstat(#sslsocket{} = S, Opts) -> + ssl:getstat(S, Opts); +getstat(S, Opts) -> + inet:getstat(S, Opts). + url_start(#sslsocket{}) -> {ok,Host} = inet:gethostname(), ?TLS_URL_START ++ Host ++ ":"; diff --git a/lib/inets/test/httpd_SUITE.erl b/lib/inets/test/httpd_SUITE.erl index 9a85c51d24..5020b5a802 100644 --- a/lib/inets/test/httpd_SUITE.erl +++ b/lib/inets/test/httpd_SUITE.erl @@ -75,6 +75,7 @@ all() -> {group, http_mime_types}, {group, http_logging}, {group, http_post}, + {group, http_rel_path_script_alias}, mime_types_format ]. @@ -112,7 +113,8 @@ groups() -> non_disturbing_0_9, disturbing_1_1, disturbing_1_0, - disturbing_0_9 + disturbing_0_9, + reload_config_file ]}, {post, [], [chunked_post, chunked_chunked_encoded_post]}, {basic_auth, [], [basic_auth_1_1, basic_auth_1_0, basic_auth_0_9]}, @@ -131,7 +133,8 @@ groups() -> trace, range, if_modified_since, mod_esi_chunk_timeout, esi_put, esi_post] ++ http_head() ++ http_get() ++ load()}, {http_1_0, [], [host, cgi, trace] ++ http_head() ++ http_get() ++ load()}, - {http_0_9, [], http_head() ++ http_get() ++ load()} + {http_0_9, [], http_head() ++ http_get() ++ load()}, + {http_rel_path_script_alias, [], [cgi]} ]. basic_groups ()-> @@ -168,6 +171,7 @@ init_per_suite(Config) -> ServerRoot = filename:join(PrivDir, "server_root"), inets_test_lib:del_dirs(ServerRoot), DocRoot = filename:join(ServerRoot, "htdocs"), + setup_tmp_dir(PrivDir), setup_server_dirs(ServerRoot, DocRoot, DataDir), {ok, Hostname0} = inet:gethostname(), Inet = @@ -268,6 +272,9 @@ init_per_group(http_logging, Config) -> ServerRoot = proplists:get_value(server_root, Config1), Path = ServerRoot ++ "/httpd_log_transfer", [{transfer_log, Path} | Config1]; +init_per_group(http_rel_path_script_alias = Group, Config) -> + ok = start_apps(Group), + init_httpd(Group, [{type, ip_comm},{http_version, "HTTP/1.1"}| Config]); init_per_group(_, Config) -> Config. @@ -1536,6 +1543,45 @@ non_disturbing(Config) when is_list(Config)-> end, inets_test_lib:close(Type, Socket), [{server_name, "httpd_non_disturbing_" ++ Version}] = httpd:info(Server, [server_name]). +%%------------------------------------------------------------------------- +reload_config_file(Config) when is_list(Config) -> + ServerRoot = proplists:get_value(server_root, Config), + HttpdConf = filename:join(get_tmp_dir(Config), "inets_httpd_server.conf"), + ServerConfig = + "[\n" ++ + "{bind_address, \"localhost\"}," ++ + "{port,0}," ++ + "{server_name,\"httpd_test\"}," ++ + "{server_root,\"" ++ ServerRoot ++ "\"}," ++ + "{document_root,\"" ++ proplists:get_value(doc_root, Config) ++ "\"}" ++ + "].", + ok = file:write_file(HttpdConf, ServerConfig), + {ok, Server} = inets:start(httpd, [{proplist_file, HttpdConf}]), + Port = proplists:get_value(port, httpd:info(Server)), + NewConfig = + "[\n" ++ + "{bind_address, \"localhost\"}," ++ + "{port," ++ integer_to_list(Port) ++ "}," ++ + "{server_name,\"httpd_test_new\"}," ++ + "{server_root,\"" ++ ServerRoot ++ "\"}," ++ + "{document_root,\"" ++ proplists:get_value(doc_root, Config) ++ "\"}" ++ + "].", + NewConfigApache = + "BindAddress localhost\n" ++ + "Port " ++ integer_to_list(Port) ++ "\n" ++ + "ServerName httpd_test_new_apache\n" ++ + "ServerRoot " ++ ServerRoot ++ "\n" ++ + "DocumentRoot " ++ proplists:get_value(doc_root, Config) ++ "\n", + + %% Test Erlang term format + ok = file:write_file(HttpdConf, NewConfig), + ok = httpd:reload_config(HttpdConf, non_disturbing), + "httpd_test_new" = proplists:get_value(server_name, httpd:info(Server)), + + %% Test Apache format + ok = file:write_file(HttpdConf, NewConfigApache), + ok = httpd:reload_config(HttpdConf, non_disturbing), + "httpd_test_new_apache" = proplists:get_value(server_name, httpd:info(Server)). %%------------------------------------------------------------------------- mime_types_format(Config) when is_list(Config) -> @@ -1647,6 +1693,7 @@ mime_types_format(Config) when is_list(Config) -> {"cpt","application/mac-compactpro"}, {"hqx","application/mac-binhex40"}]} = httpd_conf:load_mime_types(MimeTypes). + %%-------------------------------------------------------------------- %% Internal functions ----------------------------------- %%-------------------------------------------------------------------- @@ -1728,7 +1775,15 @@ setup_server_dirs(ServerRoot, DocRoot, DataDir) -> {ok, FileInfo1} = file:read_file_info(EnvCGI), ok = file:write_file_info(EnvCGI, FileInfo1#file_info{mode = 8#00755}). - + +setup_tmp_dir(PrivDir) -> + TmpDir = filename:join(PrivDir, "tmp"), + ok = file:make_dir(TmpDir). + +get_tmp_dir(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + filename:join(PrivDir, "tmp"). + start_apps(Group) when Group == https_basic; Group == https_limit; Group == https_custom; @@ -1753,7 +1808,8 @@ start_apps(Group) when Group == http_basic; Group == http_logging; Group == http_reload; Group == http_post; - Group == http_mime_types-> + Group == http_mime_types; + Group == http_rel_path_script_alias -> inets_test_lib:start_apps([inets]). server_start(_, HttpdConfig) -> @@ -1878,7 +1934,27 @@ server_config(http, Config) -> {erl_script_alias, {"/cgi-bin/erl", [httpd_example, io]}}, {eval_script_alias, {"/eval", [httpd_example, io]}} ]; - +server_config(http_rel_path_script_alias, Config) -> + ServerRoot = proplists:get_value(server_root, Config), + [{port, 0}, + {socket_type, {ip_comm, [{nodelay, true}]}}, + {server_name,"httpd_test"}, + {server_root, ServerRoot}, + {document_root, proplists:get_value(doc_root, Config)}, + {bind_address, any}, + {ipfamily, proplists:get_value(ipfamily, Config)}, + {max_header_size, 256}, + {max_header_action, close}, + {directory_index, ["index.html", "welcome.html"]}, + {mime_types, [{"html","text/html"},{"htm","text/html"}, {"shtml","text/html"}, + {"gif", "image/gif"}]}, + {alias, {"/icons/", filename:join(ServerRoot,"icons") ++ "/"}}, + {alias, {"/pics/", filename:join(ServerRoot,"icons") ++ "/"}}, + {script_alias, {"/cgi-bin/", "./cgi-bin/"}}, + {script_alias, {"/htbin/", "./cgi-bin/"}}, + {erl_script_alias, {"/cgi-bin/erl", [httpd_example, io]}}, + {eval_script_alias, {"/eval", [httpd_example, io]}} + ]; server_config(https, Config) -> PrivDir = proplists:get_value(priv_dir, Config), [{socket_type, {essl, diff --git a/lib/inets/test/inets_SUITE.erl b/lib/inets/test/inets_SUITE.erl index 1abd96a228..07ce594a1d 100644 --- a/lib/inets/test/inets_SUITE.erl +++ b/lib/inets/test/inets_SUITE.erl @@ -41,9 +41,7 @@ groups() -> [{services_test, [], [start_inets, start_httpc, - start_httpd, - start_ftpc, - start_tftpd + start_httpd ]}, {app_test, [], [app, appup]}]. @@ -298,79 +296,6 @@ start_httpd(Config) when is_list(Config) -> %%------------------------------------------------------------------------- -start_ftpc(doc) -> - [{doc, "Start/stop of ftpc service"}]; -start_ftpc(Config0) when is_list(Config0) -> - process_flag(trap_exit, true), - ok = inets:start(), - case ftp_SUITE:init_per_suite(Config0) of - {skip, _} = Skip -> - Skip; - Config -> - FtpdHost = proplists:get_value(ftpd_host,Config), - {ok, Pid0} = inets:start(ftpc, [{host, FtpdHost}]), - Pids0 = [ServicePid || {_, ServicePid} <- - inets:services()], - true = lists:member(Pid0, Pids0), - [_|_] = inets:services_info(), - inets:stop(ftpc, Pid0), - ct:sleep(100), - Pids1 = [ServicePid || {_, ServicePid} <- - inets:services()], - false = lists:member(Pid0, Pids1), - {ok, Pid1} = - inets:start(ftpc, [{host, FtpdHost}], stand_alone), - Pids2 = [ServicePid || {_, ServicePid} <- - inets:services()], - false = lists:member(Pid1, Pids2), - ok = inets:stop(stand_alone, Pid1), - receive - {'EXIT', Pid1, shutdown} -> - ok - after 100 -> - ct:fail(stand_alone_not_shutdown) - end, - ok = inets:stop(), - catch ftp_SUITE:end_per_SUITE(Config) - end. - -%%------------------------------------------------------------------------- - -start_tftpd() -> - [{doc, "Start/stop of tfpd service"}]. -start_tftpd(Config) when is_list(Config) -> - process_flag(trap_exit, true), - ok = inets:start(), - {ok, Pid0} = inets:start(tftpd, [{host, "localhost"}, {port, 0}]), - Pids0 = [ServicePid || {_, ServicePid} <- inets:services()], - true = lists:member(Pid0, Pids0), - [_|_] = inets:services_info(), - inets:stop(tftpd, Pid0), - ct:sleep(100), - Pids1 = [ServicePid || {_, ServicePid} <- inets:services()], - false = lists:member(Pid0, Pids1), - {ok, Pid1} = - inets:start(tftpd, [{host, "localhost"}, {port, 0}], stand_alone), - Pids2 = [ServicePid || {_, ServicePid} <- inets:services()], - false = lists:member(Pid1, Pids2), - ok = inets:stop(stand_alone, Pid1), - receive - {'EXIT', Pid1, shutdown} -> - ok - after 100 -> - ct:fail(stand_alone_not_shutdown) - end, - ok = inets:stop(), - application:load(inets), - application:set_env(inets, services, [{tftpd,[{host, "localhost"}, - {port, 0}]}]), - ok = inets:start(), - (?NUM_DEFAULT_SERVICES + 1) = length(inets:services()), - application:unset_env(inets, services), - ok = inets:stop(). - -%%------------------------------------------------------------------------- - httpd_reload() -> [{doc, "Reload httpd configuration without restarting service"}]. httpd_reload(Config) when is_list(Config) -> diff --git a/lib/inets/test/inets_socketwrap_SUITE.erl b/lib/inets/test/inets_socketwrap_SUITE.erl index 7ea7e08ed1..fc87c595a9 100644 --- a/lib/inets/test/inets_socketwrap_SUITE.erl +++ b/lib/inets/test/inets_socketwrap_SUITE.erl @@ -30,7 +30,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [start_httpd_fd, start_tftpd_fd]. + [start_httpd_fd]. init_per_suite(Config) -> case os:type() of @@ -90,37 +90,7 @@ start_httpd_fd(Config) when is_list(Config) -> ct:fail(open_port_failed) end end. -%%------------------------------------------------------------------------- -start_tftpd_fd() -> - [{doc, "Start/stop of tfpd service with socket wrapper"}]. -start_tftpd_fd(Config) when is_list(Config) -> - DataDir = proplists:get_value(data_dir, Config), - case setup_node_info(node()) of - {skip, _} = Skip -> - Skip; - {Node, NodeArg} -> - InetPort = inets_test_lib:inet_port(node()), - ct:pal("Node: ~p~n", [Node]), - Wrapper = filename:join(DataDir, "setuid_socket_wrap"), - Cmd = Wrapper ++ - " -s -tftpd_69,0:" ++ integer_to_list(InetPort) - ++ " -p " ++ os:find_executable("erl") ++ - " -- " ++ NodeArg, - ct:pal("cmd: ~p~n", [Cmd]), - case open_port({spawn, Cmd}, [stderr_to_stdout]) of - Port when is_port(Port) -> - wait_node_up(Node, 10), - ct:pal("~p", [rpc:call(Node, init, get_argument, [tftpd_69])]), - ok = rpc:call(Node, inets, start, []), - {ok, Pid} = rpc:call(Node, inets, start, - [tftpd,[{host, "localhost"}]]), - {ok, Info} = rpc:call(Node, tftp, info, [Pid]), - {value,{port, InetPort}} = lists:keysearch(port, 1, Info), - rpc:call(Node, erlang, halt, []); - _ -> - ct:fail(open_port_failed) - end - end. + %%------------------------------------------------------------------------- %% Internal functions %%------------------------------------------------------------------------- diff --git a/lib/inets/test/inets_sup_SUITE.erl b/lib/inets/test/inets_sup_SUITE.erl index 1e664337e6..727e91e987 100644 --- a/lib/inets/test/inets_sup_SUITE.erl +++ b/lib/inets/test/inets_sup_SUITE.erl @@ -32,8 +32,7 @@ suite() -> ]. all() -> - [default_tree, ftpc_worker, tftpd_worker, - httpd_config, httpd_subtree, httpd_subtree_profile, + [default_tree, httpd_config, httpd_subtree, httpd_subtree_profile, httpc_subtree]. groups() -> @@ -147,15 +146,11 @@ default_tree() -> "in the default case."}]. default_tree(Config) when is_list(Config) -> TopSupChildren = supervisor:which_children(inets_sup), - 4 = length(TopSupChildren), + 2 = length(TopSupChildren), {value, {httpd_sup, _, supervisor,[httpd_sup]}} = lists:keysearch(httpd_sup, 1, TopSupChildren), {value, {httpc_sup, _,supervisor,[httpc_sup]}} = lists:keysearch(httpc_sup, 1, TopSupChildren), - {value, {ftp_sup,_,supervisor,[ftp_sup]}} = - lists:keysearch(ftp_sup, 1, TopSupChildren), - {value, {tftp_sup,_,supervisor,[tftp_sup]}} = - lists:keysearch(tftp_sup, 1, TopSupChildren), HttpcSupChildren = supervisor:which_children(httpc_sup), {value, {httpc_profile_sup,_, supervisor, [httpc_profile_sup]}} = @@ -163,8 +158,6 @@ default_tree(Config) when is_list(Config) -> {value, {httpc_handler_sup,_, supervisor, [httpc_handler_sup]}} = lists:keysearch(httpc_handler_sup, 1, HttpcSupChildren), - [] = supervisor:which_children(ftp_sup), - [] = supervisor:which_children(httpd_sup), %% Default profile @@ -172,48 +165,7 @@ default_tree(Config) when is_list(Config) -> = supervisor:which_children(httpc_profile_sup), [] = supervisor:which_children(httpc_handler_sup), - - [] = supervisor:which_children(tftp_sup), - - ok. -ftpc_worker() -> - [{doc, "Makes sure the ftp worker processes are added and removed " - "appropriatly to/from the supervison tree."}]. -ftpc_worker(Config0) when is_list(Config0) -> - [] = supervisor:which_children(ftp_sup), - case ftp_SUITE:init_per_suite(Config0) of - {skip, _} = Skip -> - Skip; - Config -> - FtpdHost = proplists:get_value(ftpd_host,Config), - {ok, Pid} = inets:start(ftpc, [{host, FtpdHost}]), - case supervisor:which_children(ftp_sup) of - [{_,_, worker, [ftp]}] -> - inets:stop(ftpc, Pid), - ct:sleep(5000), - [] = supervisor:which_children(ftp_sup), - catch ftp_SUITE:end_per_SUITE(Config), - ok; - Children -> - catch ftp_SUITE:end_per_SUITE(Config), - exit({unexpected_children, Children}) - end - end. - -tftpd_worker() -> - [{doc, "Makes sure the tftp sub tree is correct."}]. -tftpd_worker(Config) when is_list(Config) -> - [] = supervisor:which_children(tftp_sup), - {ok, Pid0} = inets:start(tftpd, [{host, inets_test_lib:hostname()}, - {port, 0}]), - {ok, _Pid1} = inets:start(tftpd, [{host, inets_test_lib:hostname()}, - {port, 0}], stand_alone), - - [{_,Pid0, worker, _}] = supervisor:which_children(tftp_sup), - inets:stop(tftpd, Pid0), - ct:sleep(5000), - [] = supervisor:which_children(tftp_sup), ok. httpd_config() -> diff --git a/lib/inets/vsn.mk b/lib/inets/vsn.mk index 1fad9afe33..3a489357ff 100644 --- a/lib/inets/vsn.mk +++ b/lib/inets/vsn.mk @@ -19,6 +19,6 @@ # %CopyrightEnd% APPLICATION = inets -INETS_VSN = 6.5 +INETS_VSN = 6.5.1 PRE_VSN = APP_VSN = "$(APPLICATION)-$(INETS_VSN)$(PRE_VSN)" diff --git a/lib/kernel/doc/src/Makefile b/lib/kernel/doc/src/Makefile index 0759f362d4..82869d7b15 100644 --- a/lib/kernel/doc/src/Makefile +++ b/lib/kernel/doc/src/Makefile @@ -56,6 +56,11 @@ XML_REF3_FILES = application.xml \ inet.xml \ inet_res.xml \ init_stub.xml \ + logger.xml \ + logger_std_h.xml \ + logger_disk_log_h.xml \ + logger_filters.xml \ + logger_formatter.xml \ net_adm.xml \ net_kernel.xml \ os.xml \ @@ -70,11 +75,17 @@ XML_REF4_FILES = app.xml config.xml XML_REF6_FILES = kernel_app.xml -XML_PART_FILES = -XML_CHAPTER_FILES = notes.xml +XML_PART_FILES = part.xml +XML_CHAPTER_FILES = \ + notes.xml \ + introduction_chapter.xml \ + logger_chapter.xml BOOK_FILES = book.xml +IMAGE_FILES = \ + logger_arch.png + XML_FILES = \ $(BOOK_FILES) $(XML_CHAPTER_FILES) \ $(XML_PART_FILES) $(XML_REF3_FILES) $(XML_REF4_FILES)\ @@ -111,7 +122,7 @@ SPECS_FLAGS = -I../../include # ---------------------------------------------------- # Targets # ---------------------------------------------------- -$(HTMLDIR)/%.gif: %.gif +$(HTMLDIR)/%: % $(INSTALL_DATA) $< $@ docs: man pdf html @@ -120,11 +131,12 @@ $(TOP_PDF_FILE): $(XML_FILES) pdf: $(TOP_PDF_FILE) -html: gifs $(HTML_REF_MAN_FILE) +html: images $(HTML_REF_MAN_FILE) man: $(MAN3_FILES) $(MAN4_FILES) $(MAN6_FILES) -gifs: $(GIF_FILES:%=$(HTMLDIR)/%) +images: $(IMAGE_FILES:%=$(HTMLDIR)/%) + debug opt: clean clean_docs: @@ -137,16 +149,16 @@ clean clean_docs: rm -f errs core *~ $(SPECDIR)/specs_erl_prim_loader_stub.xml: - escript $(SPECS_EXTRACTOR) $(SPECS_FLAGS) \ + $(gen_verbose)escript $(SPECS_EXTRACTOR) $(SPECS_FLAGS) \ -o$(dir $@) -module erl_prim_loader_stub $(SPECDIR)/specs_erlang_stub.xml: - escript $(SPECS_EXTRACTOR) $(SPECS_FLAGS) \ + $(gen_verbose)escript $(SPECS_EXTRACTOR) $(SPECS_FLAGS) \ -o$(dir $@) -module erlang_stub $(SPECDIR)/specs_init_stub.xml: - escript $(SPECS_EXTRACTOR) $(SPECS_FLAGS) \ + $(gen_verbose)escript $(SPECS_EXTRACTOR) $(SPECS_FLAGS) \ -o$(dir $@) -module init_stub $(SPECDIR)/specs_zlib_stub.xml: - escript $(SPECS_EXTRACTOR) $(SPECS_FLAGS) \ + $(gen_verbose)escript $(SPECS_EXTRACTOR) $(SPECS_FLAGS) \ -o$(dir $@) -module zlib_stub # ---------------------------------------------------- diff --git a/lib/kernel/doc/src/book.xml b/lib/kernel/doc/src/book.xml index 81a87d126d..0b69b547e7 100644 --- a/lib/kernel/doc/src/book.xml +++ b/lib/kernel/doc/src/book.xml @@ -34,6 +34,9 @@ <preamble> <contents level="2"></contents> </preamble> + <parts lift="yes"> + <xi:include href="part.xml"/> + </parts> <applications> <xi:include href="ref_man.xml"/> </applications> diff --git a/lib/kernel/doc/src/error_logger.xml b/lib/kernel/doc/src/error_logger.xml index 91bf57cb91..cb6165c73e 100644 --- a/lib/kernel/doc/src/error_logger.xml +++ b/lib/kernel/doc/src/error_logger.xml @@ -31,6 +31,16 @@ <module>error_logger</module> <modulesummary>Erlang error logger.</modulesummary> <description> + + <note> + <p>In OTP-21, a new API for logging was added to Erlang/OTP. The + old <c>error_logger</c> module can still be used by legacy + code, but new code should use the new API instead.</p> + <p>See <seealso marker="logger"><c>logger(3)</c></seealso> and + the <seealso marker="logger_chapter">Logging</seealso> chapter + in the user's guide for more information.</p> + </note> + <p>The Erlang <em>error logger</em> is an event manager (see <seealso marker="doc/design_principles:des_princ">OTP Design Principles</seealso> and <seealso marker="stdlib:gen_event"><c>gen_event(3)</c></seealso>), @@ -171,14 +181,17 @@ ok</pre> <func> <name name="get_format_depth" arity="0"/> <fsummary>Get the value of the Kernel application variable - <c>error_logger_format_depth</c>.</fsummary> + <c>logger_format_depth</c>.</fsummary> <desc> <p>Returns <c>max(10, Depth)</c>, where <c>Depth</c> is the value of - <seealso marker="kernel:kernel_app#error_logger_format_depth"> - error_logger_format_depth</seealso> + <seealso marker="kernel_app#logger_format_depth"> + logger_format_depth</seealso> in the Kernel application, if Depth is an integer. Otherwise, <c>unlimited</c> is returned.</p> + <p>For backwards compatibility, the value + of <c>error_logger_format_depth</c> is used + if <c>logger_format_depth</c> is not set.</p> </desc> </func> <func> diff --git a/lib/kernel/doc/src/heart.xml b/lib/kernel/doc/src/heart.xml index 5b5b71e521..46c7ce60b6 100644 --- a/lib/kernel/doc/src/heart.xml +++ b/lib/kernel/doc/src/heart.xml @@ -59,8 +59,9 @@ <pre> % <input>erl -heart -env HEART_BEAT_TIMEOUT 30 ...</input></pre> <p>The value (in seconds) must be in the range 10 < X <= 65535.</p> - <p>Notice that if the system clock is adjusted with - more than <c>HEART_BEAT_TIMEOUT</c> seconds, <c>heart</c> + <p>When running on OSs lacking support for monotonic time, + <c>heart</c> is susceptible to system clock adjustments of more than + <c>HEART_BEAT_TIMEOUT</c> seconds. When this happens, <c>heart</c> times out and tries to reboot the system. This can occur, for example, if the system clock is adjusted automatically by use of the Network Time Protocol (NTP).</p> diff --git a/lib/kernel/doc/src/introduction_chapter.xml b/lib/kernel/doc/src/introduction_chapter.xml new file mode 100644 index 0000000000..6e6990ddda --- /dev/null +++ b/lib/kernel/doc/src/introduction_chapter.xml @@ -0,0 +1,64 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>2017</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>Introduction</title> + <prepared></prepared> + <responsible></responsible> + <docno></docno> + <approved></approved> + <checked></checked> + <date></date> + <rev></rev> + <file>introduction.xml</file> + </header> + + <section> + <title>Scope</title> + <p>The Kernel application has all the code necessary to run + the Erlang runtime system: file servers, code servers, + and so on.</p> + <p>The Kernel application is the first application started. It is + mandatory in the sense that the minimal system based on + Erlang/OTP consists of Kernel and STDLIB. Kernel + contains the following functional areas:</p> + <list type="bulleted"> + <item>Start, stop, supervision, configuration, and distribution of applications</item> + <item>Code loading</item> + <item>Logging</item> + <item>Error logging</item> + <item>Global name service</item> + <item>Supervision of Erlang/OTP</item> + <item>Communication with sockets</item> + <item>Operating system interface</item> + </list> + </section> + + <section> + <title>Prerequisites</title> + <p>It is assumed that the reader is familiar with the Erlang programming + language.</p> + </section> +</chapter> + + diff --git a/lib/kernel/doc/src/kernel_app.xml b/lib/kernel/doc/src/kernel_app.xml index 0762cebc94..554d675383 100644 --- a/lib/kernel/doc/src/kernel_app.xml +++ b/lib/kernel/doc/src/kernel_app.xml @@ -51,10 +51,13 @@ </description> <section> - <title>Error Logger Event Handlers</title> - <p>Two standard error logger event handlers are defined in - the Kernel application. These are described in - <seealso marker="error_logger"><c>error_logger(3)</c></seealso>.</p> + <title>Logger Handlers</title> + <p>Two standard logger handlers are defined in + the Kernel application. These are described in the + <seealso marker="logger_chapter">Kernel User's Guide</seealso>, + and in <seealso marker="logger_std_h"><c>logger_std_h(3)</c></seealso> + and <seealso marker="logger_disk_log_h"><c>logger_disk_log_h(3)</c> + </seealso>.</p> </section> <section> @@ -113,6 +116,7 @@ </section> <section> + <marker id="configuration"/> <title>Configuration</title> <p>The following configuration parameters are defined for the Kernel application. For more information about configuration parameters, @@ -176,34 +180,105 @@ <p>Permissions are described in <seealso marker="application#permit/2"><c>application:permit/2</c></seealso>.</p> </item> - <tag><c>error_logger = Value</c></tag> + <tag><c>logger_dest = Value</c></tag> <item> <p><c>Value</c> is one of:</p> <taglist> <tag><c>tty</c></tag> - <item><p>Installs the standard event handler, which prints error - reports to <c>stdio</c>. This is the default option.</p></item> + <item><p>Installs the standard handler, <seealso marker="logger_std_h"> + <c>logger_std_h(3)</c></seealso>, with <c>type</c> set + to <c>standard_io</c>. This is the default + option.</p></item> <tag><c>{file, FileName}</c></tag> - <item><p>Installs the standard event handler, which prints error - reports to file <c>FileName</c>, where <c>FileName</c> + <item><p>Installs the standard handler, <seealso marker="logger_std_h"> + <c>logger_std_h(3)</c></seealso>, with <c>type</c> set + to <c>{file, FileName}</c>, where <c>FileName</c> is a string. The file is opened with encoding UTF-8.</p></item> + <tag><c>{disk_log, FileName}</c></tag> + <item><p>Installs the disk_log handler, <seealso marker="logger_disk_log_h"> + <c>logger_disk_log_h(3)</c></seealso>, with <c>file</c> set + to <c>FileName</c> (a string), and possibly other disk_log + parameters set by the environment variables + <c>logger_disk_log_type</c>, <c>logger_disk_log_maxfiles</c> and + <c>logger_disk_log_maxbytes</c>, + see <seealso marker="#disk_log_vars">below</seealso>. The + file is opened with encoding UTF-8.</p></item> <tag><c>false</c></tag> <item> - <p>No standard event handler is installed, but - the initial, primitive event handler is kept, printing + <p>No standard handler is installed, but + the initial, primitive handler is kept, printing raw event messages to <c>tty</c>.</p> </item> <tag><c>silent</c></tag> <item> - <p>Error logging is turned off.</p> + <p>No standard handler is started, and the initial, + primitive handler is removed.</p> </item> </taglist> </item> - <tag><c>error_logger_format_depth = Depth</c></tag> + <tag><c>logger_level = Level</c></tag> + <item> + <p><c>Value = emergency | alert | critical | error | warning | + notice | info | debug</c></p> + <p>This parameter specifies which log levels to log. The + specified level, and all levels that are more severe, will + be logged.</p> + <p>This configuration parameter is used both for the global + logger level, and for the standard handler started by + the Kernel application (see <c>logger_dest</c> variable above).</p> + <p>The default value is <c>info</c></p> + </item> + <tag><marker id="disk_log_vars"/> + <c>logger_disk_log_type = halt | wrap</c></tag> + <item/> + <tag><c>logger_disk_log_maxfiles = integer()</c></tag> + <item/> + <tag><c>logger_disk_log_maxbytes = integer()</c></tag> + <item> + <p>If <c>logger_dest</c> is set to {disk_log,File}, then these + parameters specify the configuration to use when opening the + disk log file. They specify the type of disk log, the + maximum number of files (if the type is wrap) and the + maximum size of each file, respectively.</p> + <p>The default values are:</p> + <code> +logger_disk_log_type = wrap +logger_disk_log_maxfiles = 10 +logger_disk_log_maxbytes = 1048576</code> + </item> + <tag><marker id="logger_sasl_compatible"/> + <c>logger_sasl_compatible = boolean()</c></tag> + <item> + <p>If this parameter is set to true, then the logger handler + started by kernel will not log any progress-, crash-, or + supervisor reports. If the SASL application is starated, + these log events will be sent to a second handler instance + named sasl_h, according to values of the SASL environment + variables <c>sasl_error_logger</c> + and <c>sasl_errlog_type</c>, see + <seealso marker="sasl:sasl_app#configuration">SASL(6) + </seealso></p> + <p>The default value is <c>false</c></p> + <p>See chapter <seealso marker="logger_chapter#compatibility">Backwards + compatibility with error_logger</seealso> for more + information about handling of the so called SASL reports.</p> + </item> + <tag><marker id="logger_log_progress"/> + <c>logger_log_progress = boolean()</c></tag> + <item> + <p>If <c>logger_sasl_compatible = false</c>, + then <c>logger_log_progress</c> specifies if progress + reports from <c>supervisor</c> + and <c>application_controller</c> shall be logged or + not.</p> + <p>If <c>logger_sasl_compatible = false</c>, + then <c>logger_log_progress</c> is ignored.</p> + </item> + <tag><marker id="logger_format_depth"/> + <c>logger_format_depth = Depth</c></tag> <item> - <marker id="error_logger_format_depth"></marker> <p>Can be used to limit the size of the - formatted output from the error logger event handlers.</p> + formatted output from the logger handlers.</p> <note><p>This configuration parameter was introduced in OTP 18.1 and is experimental. Based on user feedback, it @@ -214,16 +289,16 @@ useless.</p></note> <p><c>Depth</c> is a positive integer representing the maximum - depth to which terms are printed by the error logger event + depth to which terms are printed by the logger handlers included in OTP. This - configuration parameter is used by the two event handlers - defined by the Kernel application and the two event - handlers in the SASL application. - (If you have implemented your own error handlers, this configuration - parameter has no effect on them.)</p> + configuration parameter is used by the default formatter, + <seealso marker="logger_formatter"><c>logger_formatter(3)</c></seealso>, + unless the formatter's <c>depth</c> parameter is explicitly set. + (If you have implemented your own formatter, this configuration + parameter has no effect on that.)</p> <p><c>Depth</c> is used as follows: Format strings - passed to the event handlers are rewritten. + received by the formatter are rewritten. The format controls <c>~p</c> and <c>~w</c> are replaced with <c>~P</c> and <c>~W</c>, respectively, and <c>Depth</c> is used as the depth parameter. For details, see @@ -234,7 +309,20 @@ <c>30</c>. We recommend to test crashing various processes in your application, examine the logs from the crashes, and then increase or decrease the value.</p></note> - </item> + </item> + <tag><c>logger_max_size = integer() | unlimited</c></tag> + <item> + <p>This parameter specifies the maximum size (bytes) each + log event can have when printed by the standard logger + handler. If the resulting string after formatting an event + is bigger than this, it will be truncated before printed + to the handler's destination.</p> + </item> + <tag><c>logger_utc = boolean()</c></tag> + <item> + <p>If set to <c>true</c>, the default formatter will display + all dates in Universal Coordinated Time.</p> + </item> <tag><c>global_groups = [GroupTuple]</c></tag> <item> <marker id="global_groups"></marker> @@ -497,6 +585,26 @@ MaxT = TickTime + TickTime / 4</code> </section> <section> + <title>Deprecated Configuration Parameters</title> + <p>In OTP-21, a new API for logging was added to Erlang/OTP. The + old <c>error_logger</c> event manager, and event handlers + running on this manager, will still work, but they are not used + by default.</p> + <p>The following application environment variables can still be + set, but they will only be used if the corresponding new logger + variables are not set.</p> + <taglist> + <tag><c>error_logger</c></tag> + <item>Replaced by <c>logger_dest</c></item> + <tag><c>error_logger_format_depth</c></tag> + <item>Replaced by <c>logger_format_depth</c></item> + </taglist> + <p>See <seealso marker="logger_chapter#compatibility">Backwards + compatibility with error_logger</seealso> for more + information.</p> + </section> + + <section> <title>See Also</title> <p><seealso marker="app"><c>app(4)</c></seealso>, <seealso marker="application"><c>application(3)</c></seealso>, diff --git a/lib/kernel/doc/src/logger.xml b/lib/kernel/doc/src/logger.xml new file mode 100644 index 0000000000..66e6e5c689 --- /dev/null +++ b/lib/kernel/doc/src/logger.xml @@ -0,0 +1,478 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>2017</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>logger</title> + <prepared></prepared> + <responsible></responsible> + <docno></docno> + <approved></approved> + <checked></checked> + <date></date> + <rev>A</rev> + <file>logger.xml</file> + </header> + <module>logger</module> + <modulesummary>API module for the logger application.</modulesummary> + + <description> + + </description> + + <datatypes> + <datatype> + <name name="level"/> + <desc> + <p>The severity level for the message to be logged.</p> + </desc> + </datatype> + <datatype> + <name name="log"/> + <desc> + <p></p> + </desc> + </datatype> + <datatype> + <name name="report"/> + <desc> + <p></p> + </desc> + </datatype> + <datatype> + <name name="msg_fun"/> + <desc> + <p></p> + </desc> + </datatype> + <datatype> + <name name="metadata"/> + <desc> + <p>Metadata associated with the message to be logged.</p> + </desc> + </datatype> + <datatype> + <name name="config"/> + <desc> + <p></p> + </desc> + </datatype> + <datatype> + <name name="handler_id"/> + <desc> + <p></p> + </desc> + </datatype> + <datatype> + <name name="filter_id"/> + <desc> + <p></p> + </desc> + </datatype> + <datatype> + <name name="filter"/> + <desc> + <p></p> + </desc> + </datatype> + <datatype> + <name name="filter_return"/> + <desc> + <p></p> + </desc> + </datatype> + </datatypes> + + <section> + <title>Macros</title> + <p>The following macros are defined:</p> + + <list> + <item><c>?LOG_EMERGENCY(StringOrReport[,Metadata])</c></item> + <item><c>?LOG_EMERGENCY(FunOrFormat,Args[,Metadata])</c></item> + <item><c>?LOG_ALERT(StringOrReport[,Metadata])</c></item> + <item><c>?LOG_ALERT(FunOrFormat,Args[,Metadata])</c></item> + <item><c>?LOG_CRITICAL(StringOrReport[,Metadata])</c></item> + <item><c>?LOG_CRITICAL(FunOrFormat,Args[,Metadata])</c></item> + <item><c>?LOG_ERROR(StringOrReport[,Metadata])</c></item> + <item><c>?LOG_ERROR(FunOrFormat,Args[,Metadata])</c></item> + <item><c>?LOG_WARNING(StringOrReport[,Metadata])</c></item> + <item><c>?LOG_WARNING(FunOrFormat,Args[,Metadata])</c></item> + <item><c>?LOG_NOTICE(StringOrReport[,Metadata])</c></item> + <item><c>?LOG_NOTICE(FunOrFormat,Args[,Metadata])</c></item> + <item><c>?LOG_INFO(StringOrReport[,Metadata])</c></item> + <item><c>?LOG_INFO(FunOrFormat,Args[,Metadata])</c></item> + <item><c>?LOG_DEBUG(StringOrReport[,Metadata])</c></item> + <item><c>?LOG_DEBUG(FunOrFormat,Args[,Metadata])</c></item> + </list> + + <p>All macros expand to a call to logger, where <c>Level</c> is + taken from the macro name, and the following metadata is added, + or merged with the given <c>Metadata</c>:</p> + + <code> +#{mfa=>{?MODULE,?FUNCTION_NAME,?FUNCTION_ARITY}, + file=>?FILE, + line=>?LINE} + </code> + + <p>The call is wrapped in a case statement and will be evaluated + only if <c>Level</c> is equal to or below the configured log + level.</p> + </section> + + <funcs> + <func> + <name>emergency(StringOrReport[,Metadata])</name> + <name>emergency(Format,Args[,Metadata])</name> + <name>emergency(Fun,FunArgs[,Metadata])</name> + <fsummary>Logs the given message as level <c>emergency</c>.</fsummary> + <desc> + <p>Equivalent to + <seealso marker="#log-2"><c>log(emergency,...)</c></seealso>.</p> + </desc> + </func> + + <func> + <name>alert(StringOrReport[,Metadata])</name> + <name>alert(Format,Args[,Metadata])</name> + <name>alert(Fun,FunArgs[,Metadata])</name> + <fsummary>Logs the given message as level <c>alert</c>.</fsummary> + <desc> + <p>Equivalent to + <seealso marker="#log-2"><c>log(alert,...)</c></seealso>.</p> + </desc> + </func> + + <func> + <name>critical(StringOrReport[,Metadata])</name> + <name>critical(Format,Args[,Metadata])</name> + <name>critical(Fun,FunArgs[,Metadata])</name> + <fsummary>Logs the given message as level <c>critical</c>.</fsummary> + <desc> + <p>Equivalent to + <seealso marker="#log-2"><c>log(critical,...)</c></seealso>.</p> + </desc> + </func> + + <func> + <name>error(StringOrReport[,Metadata])</name> + <name>error(Format,Args[,Metadata])</name> + <name>error(Fun,FunArgs[,Metadata])</name> + <fsummary>Logs the given message as level <c>error</c>.</fsummary> + <desc> + <p>Equivalent to + <seealso marker="#log-2"><c>log(error,...)</c></seealso>.</p> + </desc> + </func> + + <func> + <name>warning(StringOrReport[,Metadata])</name> + <name>warning(Format,Args[,Metadata])</name> + <name>warning(Fun,FunArgs[,Metadata])</name> + <fsummary>Logs the given message as level <c>warning</c>.</fsummary> + <desc> + <p>Equivalent to + <seealso marker="#log-2"><c>log(warning,...)</c></seealso>.</p> + </desc> + </func> + + <func> + <name>notice(StringOrReport[,Metadata])</name> + <name>notice(Format,Args[,Metadata])</name> + <name>notice(Fun,FunArgs[,Metadata])</name> + <fsummary>Logs the given message as level <c>notice</c>.</fsummary> + <desc> + <p>Equivalent to + <seealso marker="#log-2"><c>log(notice,...)</c></seealso>.</p> + </desc> + </func> + + <func> + <name>info(StringOrReport[,Metadata])</name> + <name>info(Format,Args[,Metadata])</name> + <name>info(Fun,FunArgs[,Metadata])</name> + <fsummary>Logs the given message as level <c>info</c>.</fsummary> + <desc> + <p>Equivalent to + <seealso marker="#log-2"><c>log(info,...)</c></seealso>.</p> + </desc> + </func> + + <func> + <name>debug(StringOrReport[,Metadata])</name> + <name>debug(Format,Args[,Metadata])</name> + <name>debug(Fun,FunArgs[,Metadata])</name> + <fsummary>Logs the given message as level <c>debug</c>.</fsummary> + <desc> + <p>Equivalent to + <seealso marker="#log-2"><c>log(debug,...)</c></seealso>.</p> + </desc> + </func> + + <func> + <name name="log" arity="2"/> + <name name="log" arity="3" clause_i="1"/> + <name name="log" arity="3" clause_i="2"/> + <name name="log" arity="3" clause_i="3"/> + <name name="log" arity="4" clause_i="1"/> + <name name="log" arity="4" clause_i="2"/> + <fsummary>Logs the given message.</fsummary> + <type variable="Level"/> + <type variable="StringOrReport" name_i="1"/> + <type variable="Format" name_i="3"/> + <type variable="Args" name_i="3"/> + <type variable="Fun" name_i="4"/> + <type variable="FunArgs" name_i="4"/> + <type variable="Metadata"/> + <desc> + <p>Log the given message.</p> + </desc> + </func> + + <func> + <name name="get_logger_config" arity="0"/> + <fsummary>Lookup the current configuration for logger.</fsummary> + <desc> + <p>Lookup the current configuration for logger.</p> + </desc> + </func> + + <func> + <name name="get_handler_config" arity="1"/> + <fsummary>Lookup the current configuration for the given handler.</fsummary> + <desc> + <p>Lookup the current configuration for the given handler.</p> + </desc> + </func> + + <func> + <name name="i" arity="0"/> + <fsummary>Get information about all logger configurations</fsummary> + <desc> + <p>Same as <seealso marker="#i/1"><c>logger:i(term)</c></seealso></p> + </desc> + </func> + + <func> + <name name="i" arity="1" clause_i="1"/> + <name name="i" arity="1" clause_i="2"/> + <name name="i" arity="1" clause_i="3"/> + <fsummary>Get information about all logger configurations</fsummary> + <desc> + <p>The <c>logger:i/1</c> function can be used to get all + current logger configuration. The way that the information + is returned depends on the <c><anno>Action</anno></c></p> + <taglist> + <tag>string</tag> + <item>Return the pretty printed current logger configuration + as iodata.</item> + <tag>term</tag> + <item>Return the current logger configuration as a term. The + format of this term may change inbetween releases. For a + stable format use <seealso marker="#get_handler_config/1"> + <c>logger:get_handler_config/1</c></seealso> + and <seealso marker="#get_logger_config/0"> + <c>logger:get_logger_config/0</c></seealso>. + The same as calling <c>logger:i()</c>.</item> + <tag>print</tag> + <item>Pretty print all the current logger configuration to + standard out. Example: + <code><![CDATA[1> logger:i(). +Current logger configuration: + Level: info + FilterDefault: log + Filters: + Handlers: + Id: logger_std_h + Module: logger_std_h + Level: info + Formatter: + Module: logger_formatter + Config: #{template => [{logger_formatter,header},"\n",msg,"\n"], + legacy_header => true} + Filter Default: stop + Filters: + Id: stop_progress + Fun: fun logger_filters:progress/2 + Config: stop + Id: remote_gl + Fun: fun logger_filters:remote_gl/2 + Config: stop + Id: domain + Fun: fun logger_filters:domain/2 + Config: {log,prefix_of,[beam,erlang,otp,sasl]} + Id: no_domain + Fun: fun logger_filters:domain/2 + Config: {log,no_domain,[]} + Handler Config: + logger_std_h: #{type => standard_io} + Level set per module: + Module: my_module + Level: debug]]></code> + </item> + </taglist> + </desc> + </func> + + <func> + <name name="add_logger_filter" arity="2"/> + <fsummary>Add a filter to the logger.</fsummary> + <desc> + <p>Add a filter to the logger.</p> + </desc> + </func> + + <func> + <name name="add_handler_filter" arity="3"/> + <fsummary>Add a filter to the specified handler.</fsummary> + <desc> + <p>Add a filter to the specified handler.</p> + </desc> + </func> + + <func> + <name name="remove_logger_filter" arity="1"/> + <fsummary>Remove a filter from the logger.</fsummary> + <desc> + <p>Remove the filter with the specified identity from the logger.</p> + </desc> + </func> + + <func> + <name name="remove_handler_filter" arity="2"/> + <fsummary>Remove a filter from the specified handler.</fsummary> + <desc> + <p>Remove the filter with the specified identity from the given handler.</p> + </desc> + </func> + + <func> + <name name="add_handler" arity="3"/> + <fsummary>Add a handler with the given configuration.</fsummary> + <desc> + <p>Add a handler with the given configuration.</p> + </desc> + </func> + + <func> + <name name="remove_handler" arity="1"/> + <fsummary>Remove the handler with the specified identity.</fsummary> + <desc> + <p>Remove the handler with the specified identity.</p> + </desc> + </func> + + <func> + <name name="set_module_level" arity="2"/> + <fsummary>Set the log level for the specified module.</fsummary> + <desc> + <p>Set the log level for the specified module.</p> + <p>To change the logging level globally, use + <seealso marker="#set_logger_config/2"><c>logger:set_logger_config(level, Level)</c></seealso>. + </p> + </desc> + </func> + + <func> + <name name="reset_module_level" arity="1"/> + <fsummary>Remove a module specific log setting.</fsummary> + <desc> + <p>Remove a module specific log setting. After this, the + global log level is used for the specified module.</p> + </desc> + </func> + + <func> + <name name="set_logger_config" arity="1"/> + <name name="set_logger_config" arity="2"/> + <fsummary>Add or update configuration data for the logger.</fsummary> + <desc> + <p>Add or update configuration data for the logger.</p> + </desc> + </func> + + <func> + <name name="set_handler_config" arity="2"/> + <name name="set_handler_config" arity="3"/> + <fsummary>Add or update configuration data for the specified + handler.</fsummary> + <desc> + <p>Add or update configuration data for the specified + handler.</p> + </desc> + </func> + + <func> + <name name="compare_levels" arity="2"/> + <fsummary>Compare the severity of two log levels.</fsummary> + <desc> + <p>Compare the severity of two log levels. Returns <c>gt</c> + if <c>Level1</c> is more severe than + <c>Level2</c>, <c>lt</c> if <c>Level1</c> is less severe, + and <c>eq</c> if the levels are equal.</p> + </desc> + </func> + + <func> + <name name="set_process_metadata" arity="1"/> + <fsummary>Set metadata to use when logging from current process.</fsummary> + <desc> + <p>Set metadata which <c>logger</c> automatically inserts it + in all log events produced on the current + process. Subsequent calls will overwrite previous data set + by this function.</p> + <p>When logging, location data produced by the log macros, + and/or metadata given as argument to the log call (API + function or macro), will be merged with the process + metadata. If the same keys occur, values from the metadata + argument to the log call will overwrite values in the + process metadata, which in turn will overwrite values from + the location data.</p> + </desc> + </func> + + <func> + <name name="get_process_metadata" arity="0"/> + <fsummary>Retrieve data set with set_process_metadata/1.</fsummary> + <desc> + <p>Retrieve data set + with <seealso marker="#set_process_metadata-1"> + <c>set_process_metadata/1</c></seealso>.</p> + </desc> + </func> + + <func> + <name name="unset_process_metadata" arity="0"/> + <fsummary>Delete data set with set_process_metadata/1.</fsummary> + <desc> + <p>Delete data set + with <seealso marker="#set_process_metadata-1"> + <c>set_process_metadata/1</c></seealso>.</p> + </desc> + </func> + + </funcs> + +</erlref> + + diff --git a/lib/kernel/doc/src/logger_arch.png b/lib/kernel/doc/src/logger_arch.png Binary files differnew file mode 100644 index 0000000000..727609a6ef --- /dev/null +++ b/lib/kernel/doc/src/logger_arch.png diff --git a/lib/kernel/doc/src/logger_chapter.xml b/lib/kernel/doc/src/logger_chapter.xml new file mode 100644 index 0000000000..0bc3b37476 --- /dev/null +++ b/lib/kernel/doc/src/logger_chapter.xml @@ -0,0 +1,815 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>2017</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>Logging</title> + <prepared></prepared> + <docno></docno> + <date></date> + <rev></rev> + <file>logger_chapter.xml</file> + </header> + + <section> + <title>Overview</title> + <p>Erlang/OTP provides a standard API for logging. The backend of + this API can be used as is, or it can be customized to suite + specific needs.</p> + <p>It consists of two parts - the <em>logger</em> part and the + <em>handler</em> part. The logger will forward log events to one + or more handler(s).</p> + + <image file="logger_arch.png"> + <icaption>Conceptual overview</icaption> + </image> + + <p><em>Filters</em> can be added to the logger and to each + handler. The filters decide if an event is to be forwarded or + not, and they can also modify all parts of the log event.</p> + + <p>A <em>formatter</em> can be set for each handler. The formatter + does the final formatting of the log event, including the log + message itself, and possibly a timestamp, header and other + metadata.</p> + + <p>In accordance with the Syslog protocol, RFC-5424, eight + severity levels can be specified:</p> + + <table align="left"> + <row> + <cell><strong>Level</strong></cell> + <cell align="center"><strong>Integer</strong></cell> + <cell><strong>Description</strong></cell> + </row> + <row> + <cell>emergency</cell> + <cell align="center">0</cell> + <cell>system is unusable</cell> + </row> + <row> + <cell>alert</cell> + <cell align="center">1</cell> + <cell>action must be taken immediately</cell> + </row> + <row> + <cell>critical</cell> + <cell align="center">2</cell> + <cell>critical contidions</cell> + </row> + <row> + <cell>error</cell> + <cell align="center">3</cell> + <cell>error conditions</cell> + </row> + <row> + <cell>warning</cell> + <cell align="center">4</cell> + <cell>warning conditions</cell> + </row> + <row> + <cell>notice</cell> + <cell align="center">5</cell> + <cell>normal but significant conditions</cell> + </row> + <row> + <cell>info</cell> + <cell align="center">6</cell> + <cell>informational messages</cell> + </row> + <row> + <cell>debug</cell> + <cell align="center">7</cell> + <cell>debug-level messages</cell> + </row> + <tcaption>Severity levels</tcaption> + </table> + + <p>A log event is allowed by Logger if the integer value of + its <c>Level</c> is less than or equal to the currently + configured log level. The log level can be configured globally, + or to allow more verbose logging from a specific part of the + system, per module.</p> + + <section> + <title>Customizable parts</title> + + <taglist> + <tag><marker id="Handler"/>Handler</tag> + <item> + <p>A handler is defined as a module exporting the following + function:</p> + + <code>log(Log, Config) -> ok</code> + + <p>A handler is called by the logger backend after filtering on + logger level and on handler level for the handler which is + about to be called. The function call is done on the client + process, and it is up to the handler implementation if other + processes are to be involved or not.</p> + + <p>Multiple instances of the same handler can be + added. Configuration is per instance.</p> + + </item> + + <tag><marker id="Filter"/>Filter</tag> + <item> + <p>Filters can be set on the logger or on a handler. Logger + filters are applied first, and if passed, the handler filters + for each handler are applied. The handler plugin is only + called if all handler filters for the handler in question also + pass.</p> + + <p>A filter is specified as:</p> + + <code>{fun((Log,Extra) -> Log | stop | ignore), Extra}</code> + + <p>The configuration parameter <c>filter_default</c> + specifies the behavior if all filters return <c>ignore</c>. + <c>filter_default</c> is by default set to <c>log</c>.</p> + + <p>The <c>Extra</c> parameter may contain any data that the + filter needs.</p> + </item> + + <tag><marker id="Formatter"/>Formatter</tag> + <item> + <p>A formatter is defined as a module exporting the following + function:</p> + + <code>format(Log,Extra) -> string()</code> + + <p>The formatter plugin is called by each handler, and the + returned string can be printed to the handler's destination + (stdout, file, ...).</p> + </item> + + </taglist> + </section> + + <section> + <title>Built-in handlers</title> + + <taglist> + <tag><c>logger_std_h</c></tag> + <item> + <p>This is the default handler used by OTP. Multiple instances + can be started, and each instance will write log events to a + given destination, console or file. Filters can be used for + selecting which event to send to which handler instance.</p> + </item> + + <tag><c>logger_disk_log_h</c></tag> + <item> + <p>This handler behaves much like logger_std_h, except it uses + <seealso marker="disk_log"><c>disk_log</c></seealso> as its + destination.</p> + </item> + + <tag><marker id="ErrorLoggerManager"/><c>error_logger</c></tag> + <item> + <p>This handler is to be used for backwards compatibility + only. It is not started by default, but will be automatically + started the first time an event handler is added + with <seealso marker="error_logger#add_report_handler-1"> + <c>error_logger:add_report_handler/1,2</c></seealso>.</p> + + <p>No built-in event handlers exist.</p> + </item> + </taglist> + </section> + + <section> + <title>Built-in filters</title> + + <taglist> + <tag><c>logger_filters:domain/2</c></tag> + <item> + <p>This filter provides a way of filtering log events based on a + <c>domain</c> field <c>Metadata</c>. See + <seealso marker="logger_filters#domain-2"> + <c>logger_filters:domain/2</c></seealso></p> + </item> + + <tag><c>logger_filters:level/2</c></tag> + <item> + <p>This filter provides a way of filtering log events based + on the log level. See <seealso marker="logger_filters#domain-2"> + <c>logger_filters:domain/2</c></seealso></p> + </item> + + <tag><c>logger_filters:progress/2</c></tag> + <item> + <p>This filter matches all progress reports + from <c>supervisor</c> and <c>application_controller</c>. + See <seealso marker="logger_filters#progress/2"> + <c>logger_filters:progress/2</c></seealso></p> + </item> + + <tag><c>logger_filters:remote_gl/2</c></tag> + <item> + <p>This filter matches all events originating from a process + that has its group leader on a remote node. + See <seealso marker="logger_filters#remote_gl/2"> + <c>logger_filters:remote_gl/2</c></seealso></p> + </item> + </taglist> + </section> + + <section> + <title>Default formatter</title> + + <p>The default formatter is <c>logger_formatter</c>. + See <seealso marker="logger_formatter#format-2"> + <c>logger_formatter:format/2</c></seealso>.</p> + </section> + </section> + + <section> + <title>Configuration</title> + + <section> + <title>Application environment variables</title> + <p>See <seealso marker="kernel_app#configuration">Kernel(6)</seealso> for + information about the application environment variables that can + be used for configuring logger.</p> + </section> + + <section> + <title>Logger configuration</title> + + <taglist> + <tag><c>level</c></tag> + <item> + <p>Specifies the severity level to log.</p> + </item> + <tag><c>filters</c></tag> + <item> + <p>Logger filters are added or removed with + <seealso marker="logger#add_logger_filter-2"> + <c>logger:add_logger_filter/2</c></seealso> and + <seealso marker="logger#remove_logger_filter-1"> + <c>logger:remove_logger_filter/1</c></seealso>, + respectively.</p> + <p>See <seealso marker="#Filter">Filter</seealso> for more + information.</p> + <p>By default, no filters exist.</p> + </item> + <tag><c>filter_default = log | stop</c></tag> + <item> + <p>Specifies what to do with an event if all filters + return <c>ignore</c>.</p> + <p>Default is <c>log</c>.</p> + </item> + <tag><c>handlers</c></tag> + <item> + <p>Handlers are added or removed with + <seealso marker="logger#add_handler-3"> + <c>logger:add_handler/3</c></seealso> and + <seealso marker="logger#remove_handler-1"> + <c>logger:remove_handler/1</c></seealso>, + respectively.</p> + <p>See <seealso marker="#Handler">Handler</seealso> for more + information.</p> + </item> + </taglist> + </section> + + <section> + <marker id="handler_configuration"/> + <title>Handler configuration</title> + <taglist> + <tag><c>level</c></tag> + <item> + <p>Specifies the severity level to log.</p> + </item> + <tag><c>filters</c></tag> + <item> + <p>Handler filters can be specified when adding the handler, + or added or removed later with + <seealso marker="logger#add_handler_filter-3"> + <c>logger:add_handler_filter/3</c></seealso> and + <seealso marker="logger#remove_handler_filter-2"> + <c>logger:remove_handler_filter/2</c></seealso>, + respectively.</p> + <p>See <seealso marker="#Filter">Filter</seealso> for more + information.</p> + <p>By default, no filters exist.</p> + </item> + <tag><c>filter_default = log | stop</c></tag> + <item> + <p>Specifies what to do with an event if all filters + return <c>ignore</c>.</p> + <p>Default is <c>log</c>.</p> + </item> + <tag><c>depth = pos_integer() | unlimited</c></tag> + <item> + <p>Specifies if the depth of terms in the log events shall + be limited by using control characters <c>~P</c> + and <c>~W</c> instead of <c>~p</c> and <c>~w</c>, + respectively. See + <seealso marker="stdlib:io#format-1"><c>io:format</c></seealso>.</p> + </item> + <tag><c>max_size = pos_integer() | unlimited</c></tag> + <item> + <p>Specifies if the size of a log event shall be limited by + truncating the formatted string.</p> + </item> + <tag><c>formatter = {Module::module(),Extra::term()}</c></tag> + <item> + <p>See <seealso marker="#Formatter">Formatter</seealso> for more + information.</p> + <p>The default module is <seealso marker="logger_formatter"> + <c>logger_formatter</c></seealso>, and <c>Extra</c> is + it's configuration map.</p> + </item> + </taglist> + + <p>Note that <c>level</c> and <c>filters</c> are obeyed by + Logger itself before forwarding the log events to each + handler, while <c>depth</c>, <c>max_size</c> + and <c>formatter</c> are left to the handler + implementation. All Logger's built-in handlers do apply these + configuration parameters before printing.</p> + </section> + + </section> + + <section> + <marker id="compatibility"/> + <title>Backwards compatibility with error_logger</title> + <p>Logger provides backwards compatibility with the old + <c>error_logger</c> in the following ways:</p> + + <taglist> + <tag>Legacy event handlers</tag> + <item> + <p>To use event handlers written for <c>error_logger</c>, just + add your event handler with</p> + <code> +error_logger:add_report_handler/1,2. + </code> + <p>This will automatically start the <c>error_logger</c> + event manager, and add <c>error_logger</c> as a + handler to <c>logger</c>, with configuration</p> +<code> +#{level=>info, + filter_default=>log, + filters=>[]}. +</code> + <p>Note that this handler will ignore events that do not + originate from the old <c>error_logger</c> API, or from + within OTP. This means that if your code uses the logger API + for logging, then your log events will be discarded by this + handler.</p> + <p>Also note that <c>error_logger</c> is not overload + protected.</p> + </item> + <tag>Logger API</tag> + <item> + <p>The old <c>error_logger</c> API still exists, but should + only be used by legacy code. It will be removed in a later + release.</p> + </item> + <tag>Output format</tag> + <item> + <p>To get log events on the same format as produced + by <c>error_logger_tty_h</c> and <c>error_logger_file_h</c>, + use the default formatter, <c>logger_formatter</c>, with + configuration parameter <c>legacy_header=>true</c>. This is + also the default.</p> + </item> + <tag>Default format of log events from OTP</tag> + <item> + <p>By default, all log events originating from within OTP, + except the former so called "SASL reports", look the same as + before.</p> + </item> + <tag>SASL reports</tag> + <item> + <p>By SASL reports we mean supervisor reports, crash reports + and progress reports.</p> + <p>In earlier releases, these reports were only logged when + the SASL application was running, and they were printed + trough specific event handlers + named <c>sasl_report_tty_h</c> + and <c>sasl_report_file_h</c>.</p> + <p>The destination of these log events were configured by + environment variables for the SASL application.</p> + <p>Due to the specific event handlers, the output format + slightly differed from other log events.</p> + <p>As of OTP-21, the concept of SASL reports is removed, + meaning that the default behavior is as follows:</p> + <list> + <item>Supervisor reports, crash reports and progress reports + are no longer connected to the SASL application.</item> + <item>Supervisor reports and crash reports are logged by + default.</item> + <item>Progress reports are not logged by default, but can be + enabled with the kernel environment + variable <c>logger_log_progress</c>.</item> + <item>The output format is the same for all log + events.</item> + </list> + <p>If the old behavior is preferred, the kernel environment + variable <c>logger_sasl_compatible</c> can be set + to <c>true</c>. The old SASL environment variables can then + be used as before, and the SASL reports will only be printed + if the SASL application is running - through a second log + handler named <c>sasl_h</c>.</p> + <p>All SASL reports have a metadata + field <c>domain=>[beam,erlang,otp,sasl]</c>, which can be + used, for example, by filters to to stop or allow the + events.</p> + </item> + </taglist> + </section> + + + <section> + <title>Error handling</title> + <p>Log data is expected to be either a format string and + arguments, a string (unicode:chardata), or a report (map or + key-value list) which can be converted to a format string and + arguments by the handler. A default report callback should be + included in the log event's metadata, which can be used for + converting the report to a format string and arguments. The + handler might also do a custom conversion if the default format + is not desired.</p> + <p><c>logger</c> does, to a certain extent, check its input data + before forwarding a log event to the handlers, but it does not + evaluate conversion funs or check the validity of format strings + and arguments. This means that any filter or handler must be + careful when formatting the data of a log event, making sure + that it does not crash due to bad input data or faulty + callbacks.</p> + <p>If a filter or handler still crashes, logger will remove the + filter or handler in question from the configuration, and then + print a short error message on the console. A debug event + containing the crash reason and other details is also issued, + and can be seen if a handler is installed which logs on debug + level.</p> + </section> + + <section> + <title>Example: add a handler to log debug events to file</title> + <p>When starting an erlang node, the default behavior is that all + log events with level info and above are logged to the + console. In order to also log debug events, you can either + change the global log level to <c>debug</c> or add a separate + handler to take care of this. In this example we will add a new + handler which prints the debug events to a separate file.</p> + <p>First, we add an instance of logger_std_h with + type <c>{file,File}</c>, and we set the handler's level + to <c>debug</c>:</p> + <pre> +1> <input>Config = #{level=>debug,logger_std_h=>#{type=>{file,"./debug.log"}}}.</input> +#{logger_std_h => #{type => {file,"./debug.log"}}, + level => debug} +2> <input>logger:add_handler(debug_handler,logger_std_h,Config).</input> +ok</pre> + <p>By default, the handler receives all events, so we need to add a filter + to stop all non-debug events:</p> + <pre> +3> <input>Fun = fun(#{level:=debug}=Log,_) -> Log; (_,_) -> stop end.</input> +#Fun<erl_eval.12.98642416> +4> <input>logger:add_handler_filter(debug_handler,allow_debug,{Fun,[]}).</input> +ok</pre> + <p>And finally, we need to make sure that the logger itself allows + debug events. This can either be done by setting the global + logger level:</p> + <pre> +5> <input>logger:set_logger_config(level,debug).</input> +ok</pre> + <p>Or by allowing debug events from one or a few modules only:</p> + <pre> +6> <input>logger:set_module_level(mymodule,debug).</input> +ok</pre> + + </section> + + <section> + <title>Example: implement a handler</title> + <p>The only requirement that a handler MUST fulfill is to export + the following function:</p> + <code>log(logger:log(),logger:config()) ->ok</code> + <p>It may also implement the following callbacks:</p> + <code> +adding_handler(logger:handler_id(),logger:config()) -> {ok,logger:config()} | {error,term()} +removing_handler(logger:handler_id()) -> ok +changing_config(logger:handler_id(),logger:config(),logger:config()) -> {ok,logger:config()} | {error,term()} + </code> + <p>When logger:add_handler(Id,Module,Config) is called, logger + will first call Module:adding_handler(Id,Config), and if it + returns {ok,NewConfig} the NewConfig is written to the + configuration database. After this, the handler may receive log + events as calls to Module:log/2.</p> + <p>A handler can be removed by calling + logger:remove_handler(Id). logger will call + Module:removing_handler(Id), and then remove the handler's + configuration from the configuration database.</p> + <p>When logger:set_handler_config is called, logger calls + Module:changing_config(Id,OldConfig,NewConfig). If this function + returns ok, the NewConfig is written to the configuration + database.</p> + + <p>A simple handler which prints to the console could be + implemented as follows:</p> + <code> +-module(myhandler). +-export([log/2]). + +log(#{msg:={report,R}},_) -> + io:format("~p~n",[R]); +log(#{msg:={string,S}},_) -> + io:put_chars(S); +log(#{msg:={F,A}},_) -> + io:format(F,A). + </code> + + <p>A simple handler which prints to file could be implemented like + this:</p> + <code> +-module(myhandler). +-export([adding_handler/2, removing_handler/1, log/2]). +-export([init/1, handle_call/3, handle_cast/2, terminate/2]). + +adding_handler(Id,Config) -> + {ok,Fd} = file:open(File,[append,{encoding,utf8}]), + {ok,Config#{myhandler_fd=>Fd}}. + +removing_handler(Id,#{myhandler_fd:=Fd}) -> + _ = file:close(Fd), + ok. + +log(#{msg:={report,R}},#{myhandler_fd:=Fd}) -> + io:format(Fd,"~p~n",[R]); +log(#{msg:={string,S}},#{myhandler_fd:=Fd}) -> + io:put_chars(Fd,S); +log(#{msg:={F,A}},#{myhandler_fd:=Fd}) -> + io:format(Fd,F,A). + </code> + + <p>Note that none of the above handlers have any overload + protection, and all log events are printed directly from the + client process. Neither do the handlers use the formatter or + in any way add time or other metadata to the printed events.</p> + + <p>For examples of overload protection, please refer to the + implementation + of <seealso marker="logger_std_h"><c>logger_std_h</c></seealso> + and <seealso marker="logger_disk_log_h"><c>logger_disk_log_h</c> + </seealso>.</p> + + <p>Below is a simpler example of a handler which logs through one + single process, and uses the default formatter to gain a common + look of the log events.</p> + <p>It also uses the metadata field <c>report_cb</c>, if it exists, + to print reports in the way the event issuer suggests. The + formatter will normally do this, but if the handler either has + an own default (as in this example) or if the + given <c>report_cb</c> should not be used at all, then the + handler must take care of this itself.</p> + <code> +-module(myhandler). +-export([adding_handler/2, removing_handler/1, log/2]). +-export([init/1, handle_call/3, handle_cast/2, terminate/2]). + +adding_handler(Id,Config) -> + {ok,Pid} = gen_server:start(?MODULE,Config), + {ok,Config#{myhandler_pid=>Pid}}. + +removing_handler(Id,#{myhandler_pid:=Pid}) -> + gen_server:stop(Pid). + +log(Log,#{myhandler_pid:=Pid} = Config) -> + gen_server:cast(Pid,{log,Log,Config}). + +init(#{myhandler_file:=File}) -> + {ok,Fd} = file:open(File,[append,{encoding,utf8}]), + {ok,#{file=>File,fd=>Fd}}. + +handle_call(_,_,State) -> + {reply,{error,bad_request},State}. + +handle_cast({log,Log,Config},#{fd:=Fd} = State) -> + do_log(Fd,Log,Config), + {noreply,State}. + +terminate(Reason,#{fd:=Fd}) -> + _ = file:close(Fd), + ok. + +do_log(Fd,#{msg:={report,R}} = Log, Config) -> + Fun = maps:get(report_cb,Config,fun my_report_cb/1, + {F,A} = Fun(R), + do_log(Fd,Log#{msg=>{F,A},Config); +do_log(Fd,Log,#{formatter:={FModule,FConfig}}) -> + String = FModule:format(Log,FConfig), + io:put_chars(Fd,String). + +my_report_cb(R) -> + {"~p",[R]}. + </code> + </section> + + <section> + <marker id="overload_protection"/> + <title>Protecting the handler from overload</title> + <p>In order for the built-in handlers to survive, and stay responsive, + during periods of high load (i.e. when huge numbers of incoming + log requests must be handled), a mechanism for overload protection + has been implemented in the + <seealso marker="logger_std_h"><c>logger_std_h</c></seealso> + and <seealso marker="logger_disk_log_h"><c>logger_disk_log_h</c> + </seealso> handler. The mechanism, used by both handlers, works + as follows:</p> + + <section> + <title>Message queue length</title> + <p>The handler process keeps track of the length of its message + queue and reacts in different ways depending on the current status. + The purpose is to keep the handler in, or (as quickly as possible), + get the handler into, a state where it can keep up with the pace + of incoming log requests. The memory usage of the handler must never + keep growing larger and larger, since that would eventually cause the + handler to crash. Three thresholds with associated actions have been + defined:</p> + + <taglist> + <tag><c>toggle_sync_qlen</c></tag> + <item> + <p>The default value of this level is <c>10</c> messages, + and as long as the length of the message queue is lower, all log + requests are handled asynchronously. This simply means that the + process sending the log request (by calling a log function in the + logger API) does not wait for a response from the handler but + continues executing immediately after the request (i.e. it will not + be affected by the time it takes the handler to print to the log + device). If the message queue grows larger than this value, however, + the handler starts handling the log requests synchronously instead, + meaning the process sending the request will have to wait for a + response. When the handler manages to reduce the message queue to a + level below the <c>toggle_sync_qlen</c> threshold, asynchronous + operation is resumed. The switch from asynchronous to synchronous + mode will force the logging tempo of few busy senders to slow down, + but can not protect the handler sufficiently in situations of many + concurrent senders.</p> + </item> + <tag><c>drop_new_reqs_qlen</c></tag> + <item> + <p>When the message queue has grown larger than this threshold, which + defaults to <c>200</c> messages, the handler switches to a mode in + which it drops any new requests being made. Dropping a message in + this state means that the log function never actually sends a message + to the handler. The log call simply returns without an action. When + the length of the message queue has been reduced to a level below this + threshold, synchronous or asynchronous request handling mode is + resumed.</p> + </item> + <tag><c>flush_reqs_qlen</c></tag> + <item> + <p>Above this threshold, which defaults to <c>1000</c> messages, a + flush operation takes place, in which all messages buffered in the + process mailbox get deleted without any logging actually taking + place. (Processes waiting for a response from a synchronous log request + will receive a reply indicating that the request has been dropped).</p> + </item> + </taglist> + + <p>For the overload protection algorithm to work properly, it is a + requirement that:</p> + + <p><c>toggle_sync_qlen < drop_new_reqs_qlen < flush_reqs_qlen</c></p> + + <p>During high load scenarios, the length of the handler message queue + rarely grows in a linear and predictable way. Instead, whenever the + handler process gets scheduled in, it can have an almost arbitrary number + of messages waiting in the mailbox. It's for this reason that the overload + protection mechanism is focused on acting quickly and quite drastically + (such as immediately dropping or flushing messages) as soon as a large + queue length is detected. </p> + + <p>The thresholds listed above may be modified by the user if, e.g, a handler + shouldn't drop or flush messages unless the message queue length grows + extremely large. (The handler must be allowed to use large amounts of memory + under such circumstances however). Another example of when the user might want + to change the settings is if, for performance reasons, the logging processes must + never get blocked by synchronous log requests, while dropping or flushing requests + is perfectly acceptable (since it doesn't affect the performance of the + loggers).</p> + + <p>A configuration example:</p> + <code type="none"> +logger:add_handler(my_standard_h, logger_std_h, + #{logger_std_h => + #{type => {file,"./system_info.log"}, + toggle_sync_qlen => 100, + drop_new_reqs_qlen => 1000, + flush_reqs_qlen => 2000}}). + </code> + </section> + + <section> + <title>Controlling bursts of log requests</title> + <p>A potential problem with large bursts of log requests, is that log files + may get full or wrapped too quickly (in the latter case overwriting + previously logged data that could be of great importance). For this reason, + both built-in handlers offer the possibility to set a maximum level of how + many requests to process with a certain time frame. With this burst control + feature enabled, the handler will take care of bursts of log requests + without choking log files, or the console, with massive amounts of + printouts. These are the configuration parameters:</p> + + <taglist> + <tag><c>enable_burst_limit</c></tag> + <item> + <p>This is set to <c>true</c> by default. The value <c>false</c> + disables the burst control feature.</p> + </item> + <tag><c>burst_limit_size</c></tag> + <item> + <p>This is how many requests should be processed within the + <c>burst_window_time</c> time frame. After this maximum has been + reached, successive requests will be dropped until the end of the + time frame. The default value is <c>500</c> messages.</p> + </item> + <tag><c>burst_window_time</c></tag> + <item> + <p>The default window is <c>1000</c> milliseconds long.</p> + </item> + </taglist> + + <p>A configuration example:</p> + <code type="none"> +logger:add_handler(my_disk_log_h, logger_disk_log_h, + #{disk_log_opts => + #{file => "./my_disk_log"}, + logger_disk_log_h => + #{burst_limit_size => 10, + burst_window_time => 500}}). + </code> + </section> + + <section> + <title>Terminating a large handler</title> + <p>A handler process may grow large even if it can manage peaks of high load + without crashing. The overload protection mechanism includes user configurable + levels for a maximum allowed message queue length and maximum allowed memory + usage. This feature is disabled by default, but can be switched on by means + of the following configuration parameters:</p> + + <taglist> + <tag><c>enable_kill_overloaded</c></tag> + <item> + <p>This is set to <c>false</c> by default. The value <c>true</c> + enables the feature.</p> + </item> + <tag><c>handler_overloaded_qlen</c></tag> + <item> + <p>This is the maximum allowed queue length. If the mailbox grows larger + than this, the handler process gets terminated.</p> + </item> + <tag><c>handler_overloaded_mem</c></tag> + <item> + <p>This is the maximum allowed memory usage of the handler process. If + the handler grows any larger, the process gets terminated.</p> + </item> + <tag><c>handler_restart_after</c></tag> + <item> + <p>If the handler gets terminated because of its queue length or + memory usage, it can get automatically restarted again after a + configurable delay time. The time is specified in milliseconds + and <c>5000</c> is the default value. The value <c>never</c> can + also be set, which prevents a restart.</p> + </item> + </taglist> + </section> + </section> + + <section> + <title>See Also</title> + <p><seealso marker="error_logger"><c>error_logger(3)</c></seealso>, + <seealso marker="sasl:sasl_app"><c>SASL(6)</c></seealso></p> + </section> +</chapter> diff --git a/lib/kernel/doc/src/logger_disk_log_h.xml b/lib/kernel/doc/src/logger_disk_log_h.xml new file mode 100644 index 0000000000..90cc4fec30 --- /dev/null +++ b/lib/kernel/doc/src/logger_disk_log_h.xml @@ -0,0 +1,146 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>2017</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>logger_disk_log_h</title> + <prepared></prepared> + <responsible></responsible> + <docno></docno> + <approved></approved> + <checked></checked> + <date></date> + <rev>A</rev> + <file>logger_disk_log_h.xml</file> + </header> + <module>logger_disk_log_h</module> + <modulesummary>A disk_log based handler for the Logger + application.</modulesummary> + + <description> + <p>This is a handler for the Logger application that offers circular + (wrapped) logs by using the disk_log application. Multiple instances + of this handler can be added to logger, and each instance will print to + its own disk_log file, created with the name and settings specified in + the handler configuration.</p> + <p>The default standard handler, + <seealso marker="logger_std_h"><c>logger_std_h</c></seealso>, can be + replaced by a disk_log handler at startup of the kernel application. + See an example of this below.</p> + <p>The handler has an overload protection mechanism that will keep the handler + process and the kernel application alive during a high load of log + requests. How this feature works, and how to modify the configuration, + is described in the + <seealso marker="logger_chapter#overload_protection"><c>User's Guide</c> + </seealso>.</p> + <p>To add a new instance of the disk_log handler, use + <seealso marker="logger#add_handler-3"><c>logger:add_handler/3</c> + </seealso>. The handler configuration argument is a map which may contain + general configuration parameters, as documented in the + <seealso marker="logger_chapter#handler_configuration"><c>User's Guide</c> + </seealso>, as well as handler specific parameters.</p> + <p>The settings for the disk_log log file should be specified with the + key <c>disk_log_opts</c>. These settings are a subset of the disk_log + datatype + <seealso marker="disk_log#open-1"><c>dlog_option()</c></seealso>.</p> + <p>Parameters in the <c>disk_log_opts</c> map:</p> + <taglist> + <tag><c>file</c></tag> + <item>This is the full name of the disk_log log file.</item> + <tag><c>type</c></tag> + <item>This is the disk_log type, <c>wrap</c> or <c>halt</c>. The + default value is <c>wrap</c>.</item> + <tag><c>max_no_files</c></tag> + <item>This is the maximum number of files that disk_log will use + for its circular logging. The default value is <c>10</c>. (The setting + has no effect on a halt log).</item> + <tag><c>max_no_bytes</c></tag> + <item>This is the maximum number of bytes that will be written to + a log file before disk_log proceeds with the next file in order (or + generates an error in case of a full halt log). The default value for + a wrap log is <c>1048576</c> bytes, and <c>infinity</c> for a halt + log.</item> + </taglist> + <p>Specific configuration for the handler (represented as a sub map) + is specified with the key <c>logger_disk_log_h</c>. It may contain the + following parameter:</p> + <taglist> + <tag><c>filesync_repeat_interval</c></tag> + <item> + <p>This value (in milliseconds) specifies how often the handler will + do a disk_log sync operation in order to make sure that buffered data + gets written to disk. The handler will repeatedly attempt this + operation, but only perform it if something has actually been logged + since the last sync. The default value is <c>5000</c> milliseconds. + If <c>no_repeat</c> is set as value, the repeated sync operation is + disabled. The user can also call the + <seealso marker="logger_disk_log_h#disk_log_sync-1"><c>disk_log_sync/1</c> + </seealso> function to perform a disk_log sync.</p></item> + </taglist> + <p>There are a number of other configuration parameters available, that are + to be used for customizing the overload protection behaviour. The same + parameters are used both in the standard handler and the disk_log handler, + and are documented in the + <seealso marker="logger_chapter#overload_protection"><c>User's Guide</c> + </seealso>.</p> + <p>Note that when changing the configuration of the handler in runtime, by + calling + <seealso marker="logger#set_handler_config-2"><c>logger:set_handler_config/2 + or logger:set_handler_config/3</c></seealso>, the <c>disk_log_opts</c> + settings may not be modified.</p> + <p>Example of adding a disk_log handler:</p> + <code type="none"> +logger:add_handler(my_disk_log_h, logger_disk_log_h, + #{level => error, + filter_default => log, + disk_log_opts => + #{file => "./my_disk_log", + type => wrap, + max_no_files => 4, + max_no_bytes => 10000}, + logger_disk_log_h => + #{filesync_repeat_interval => 1000}}). + </code> + <p>In order to use the disk_log handler instead of the default standard + handler when starting en Erlang node, use the kernel configuration parameter + <seealso marker="kernel_app#configuration"><c>logger_dest</c></seealso> with + value <c>{disk_log,FileName}</c>. Example:</p> + <code type="none"> +erl -kernel logger_dest '{disk_log,"./system_disk_log"}' + </code> + </description> + + <funcs> + + <func> + <name name="disk_log_sync" arity="1" clause_i="1"/> + <fsummary>Writes buffered data to disk.</fsummary> + <desc> + <p>Write buffered data to disk.</p> + </desc> + </func> + + </funcs> + +</erlref> + + diff --git a/lib/kernel/doc/src/logger_filters.xml b/lib/kernel/doc/src/logger_filters.xml new file mode 100644 index 0000000000..d742391e35 --- /dev/null +++ b/lib/kernel/doc/src/logger_filters.xml @@ -0,0 +1,191 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>2018</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>logger_filters</title> + <prepared></prepared> + <responsible></responsible> + <docno></docno> + <approved></approved> + <checked></checked> + <date></date> + <rev>A</rev> + <file>logger_filters.xml</file> + </header> + <module>logger_filters</module> + <modulesummary>Filters to use with logger.</modulesummary> + + <description> + <p>Filters to use with logger. All functions exported from this + module can be used as logger or handler + filters. See <seealso marker="logger#add_logger_filter-2"> + <c>logger:add_logger_filter/2</c></seealso> + and <seealso marker="logger#add_handler_filter-3"> + <c>logger:add_handler_filter/3</c></seealso> + for more information about how filters are added.</p> + </description> + + <funcs> + <func> + <name name="domain" arity="2"/> + <fsummary>Filter log events based on the domain field in metadata.</fsummary> + <desc> + <p>This filter provides a way of filtering log events based on a + <c>domain</c> field <c>Metadata</c>.</p> + + <p>The <c><anno>Extra</anno></c> parameter is specified when + adding the filter + via <seealso marker="logger#add_logger_filter-2"> + <c>logger:add_logger_filter/2</c></seealso> + or <seealso marker="logger#add_handler_filter-3"> + <c>logger:add_handler_filter/3</c></seealso>.</p> + + <p>The filter compares the value of the <c>domain</c> field + in the log event's metadata (<c>Domain</c>) + to <c><anno>MatchDomain</anno></c> as follows:</p> + + <taglist> + <tag><c><anno>Compare</anno> = starts_with</c></tag> + <item><p>The filter matches if <c>MatchDomain</c> is a prefix + of <c>Domain</c>.</p></item> + <tag><c><anno>Compare</anno> = prefix_of</c></tag> + <item><p>The filter matches if <c>Domain</c> is a prefix + of <c>MatchDomain</c>.</p></item> + <tag><c><anno>Compare</anno> = equals</c></tag> + <item><p>The filter matches if <c>Domain</c> is equal + to <c>MatchDomain</c>.</p></item> + <tag><c><anno>Compare</anno> = no_domain</c></tag> + <item><p>The filter matches if there is no domain field in + metadata. In this case <c><anno>MatchDomain</anno></c> shall + be <c>[]</c>.</p></item> + </taglist> + + <p>If the filter matches and <c><anno>Action</anno> = + log</c>, the log event is allowed. If the filter matches + and <c><anno>Action</anno> = stop</c>, the log event is + stopped.</p> + + <p>If the filter does not match, it returns <c>ignore</c>, + meaning that other filters, or the value of the + configuration parameter <c>filter_default</c>, will decide + if the event is allowed or not.</p> + + <p>Log events that do not contain any domain field, will + only match when <c><anno>Compare</anno> = no_domain</c>.</p> + + <p>Example: stop all events with + domain <c>[beam,erlang,otp,sasl|_]</c></p> + + <code> +logger:set_handler_config(h1,filter_default,log). % this is the default +Filter = {fun logger_filters:domain/2,{stop,starts_with,[beam,erlang,otp,sasl]}}. +logger:add_handler_filter(h1,no_sasl,Filter). +ok</code> + </desc> + </func> + + <func> + <name name="level" arity="2"/> + <fsummary>Filter log events based on the log level.</fsummary> + <desc> + <p>This filter provides a way of filtering log events based + on the log level. It matches log events by comparing the + log level with a predefined <c>MatchLevel</c></p> + + <p>The <c><anno>Extra</anno></c> parameter is specified when + adding the filter + via <seealso marker="logger#add_logger_filter-2"> + <c>logger:add_logger_filter/2</c></seealso> + or <seealso marker="logger#add_handler_filter-3"> + <c>logger:add_handler_filter/3</c></seealso>.</p> + + <p>The filter compares the value of the event's log level + (<c>Level</c>) to <c><anno>MatchLevel</anno></c> by + calling <seealso marker="logger#compare_levels-2"> + <c>logger:compare_levels(Level,MatchLevel) -> CmpRet</c></seealso>. It + matches the event if:</p> + + <list> + <item><c>CmpRet = eq</c> and <c><anno>Operator</anno> = + eq | lteq | gteq</c></item> + <item><c>CmpRet = lt</c> and <c><anno>Operator</anno> = + lt | lteq | neq</c></item> + <item><c>CmpRet = gt</c> and <c><anno>Operator</anno> = + gt | gteq | neq</c></item> + </list> + + <p>If the filter matches and <c><anno>Action</anno> = + log</c>, the log event is allowed. If the filter matches + and <c><anno>Action</anno> = stop</c>, the log event is + stopped.</p> + + <p>If the filter does not match, it returns <c>ignore</c>, + meaning that other filters, or the value of the + configuration parameter <c>filter_default</c>, will decide + if the event is allowed or not.</p> + + <p>Example: only allow debug level log events</p> + + <code> +logger:set_handler_config(h1,filter_default,stop). +Filter = {fun logger_filters:level/2,{log,eq,debug}}. +logger:add_handler_filter(h1,debug_only,Filter). +ok</code> + </desc> + </func> + + <func> + <name name="progress" arity="2"/> + <fsummary>Filter progress reports from supervisor and application_controller.</fsummary> + <desc> + <p>This filter matches all progress reports + from <c>supervisor</c> and <c>application_controller</c>.</p> + + <p>If <c><anno>Extra</anno> = log</c>, the progress reports + are allowed. If <c><anno>Extra</anno> = stop</c>, the + progress reports are stopped.</p> + + <p>The filter returns <c>ignore</c> for all other log events.</p> + </desc> + </func> + + <func> + <name name="remote_gl" arity="2"/> + <fsummary>Filter events with group leader on remote node.</fsummary> + <desc> + <p>This filter matches all events originating from a process + that has its group leader on a remote node.</p> + + <p>If <c><anno>Extra</anno> = log</c>, the matching events + are allowed. If <c><anno>Extra</anno> = stop</c>, the + matching events are stopped.</p> + + <p>The filter returns <c>ignore</c> for all other log events.</p> + </desc> + </func> + + </funcs> + +</erlref> + + diff --git a/lib/kernel/doc/src/logger_formatter.xml b/lib/kernel/doc/src/logger_formatter.xml new file mode 100644 index 0000000000..6a17e3641f --- /dev/null +++ b/lib/kernel/doc/src/logger_formatter.xml @@ -0,0 +1,157 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>2017</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>logger_formatter</title> + <prepared></prepared> + <responsible></responsible> + <docno></docno> + <approved></approved> + <checked></checked> + <date></date> + <rev>A</rev> + <file>logger_formatter.xml</file> + </header> + <module>logger_formatter</module> + <modulesummary>Default formatter for the Logger application.</modulesummary> + + <description> + <p>Default formatter for the Logger application.</p> + </description> + + <datatypes> + <datatype> + <name name="template"/> + <desc> + </desc> + </datatype> + </datatypes> + + <funcs> + <func> + <name name="format" arity="2"/> + <fsummary>Formats the given message.</fsummary> + <desc> + <p>Formats the given message.</p> + <p>The template is a list of atoms, tuples and strings. Atoms + can be <c>level</c> or <c>msg</c>, which are placeholders + for the severity level and the log message, + repectively. Tuples are interpreted as placeholders for + metadata. Each element in the tuple must be an atom which + matches a key in the nested metadata map, e.g. the + tuple <c>{key1,key2}</c> will be replaced by the value of + the key2 field in this nested map (the value vill be + converted to a string):</p> + +<code> +#{key1=>#{key2=>my_value, + ...}, + ...}</code> + + + <p> Strings are printed literally.</p> + + <p><c>depth</c> is a positive integer representing the maximum + depth to which terms shall be printed by this + formatter. Format strings passed to this formatter are + rewritten. The format controls ~p and ~w are replaced with + ~P and ~W, respectively, and the value is used as the depth + parameter. For details, see + <seealso marker="stdlib:io#format-2">io:format/2,3</seealso> + in STDLIB.</p> + + <p><c>chars_limit</c> is a positive integer representing the + value of the option with the same name to be used when calling + <seealso marker="stdlib:io#format-3">io:format/3</seealso>. This + value limits the total number of characters printed bu the + formatter. Notes that this is a soft limit. For a hard + truncation limit, see option <c>max_size</c>.</p> + + <p><c>max_size</c> is a positive integer representing the + maximum size a string returned from this formatter can + have. If the formatted string is longer, after possibly + being limited by <c>depth</c> and/or <c>chars_limit</c>, it + will be truncated.</p> + + <p><c>utc</c> is a boolean. If set to true, all dates are + displayed in Universal Coordinated Time. Default + is <c>false</c>.</p> + + <p><c>report_cb</c> must be a function with arity 1, + returning <c>{Format,Args}</c>. This function will replace + any <c>report_cb</c> found in metadata.</p> + + <p>If <c>single_line=true</c>, all newlines in the message are + replaced with <c>", "</c>, and whitespaces following directly + after newlines are removed. Note that newlines added by the + formatter template are not replaced.</p> + + <p>If <c>legacy_header=true</c> a header field is added to + logger_formatter's part of <c>Metadata</c>. The value of + this field is a string similar to the header created by the + old <c>error_logger</c> event handlers. It can be included + in the log event by adding the + tuple <c>{logger_formatter,header}</c> to the template.</p> + + <p>The default template when <c>legacy_header=true</c> is</p> + + <code>[{logger_formatter,header},"\n",msg,"\n"]</code> + + <p>which will cause log entries like this:</p> + + <code>=ERROR REPORT==== 29-Dec-2017::13:30:51.245123 === + process: <0.74.0> + exit_reason: "Something went wrong"</code> + + <p>Note that all eight levels might occur here, not + only <c>ERROR</c>, <c>WARNING</c> or <c>INFO</c>. And also + that micro seconds are added at the end of the + timestamp.</p> + + <p>The default template when <c>single_line=true</c> is</p> + + <code>[time," ",level,": ",msg,"\n"]</code> + + <p>which will cause log entries like this:</p> + + <code>2017-12-29 13:31:49.640317 error: process: <0.74.0>, exit_reason: "Something went wrong"</code> + + <p>The default template when both <c>legacy_header</c> and + <c>single_line</c> are set to false is:</p> + + <code>[time," ",level,":\n",msg,"\n"]</code> + + <p>which will cause log entries like this:</p> + + <code>2017-12-29 13:32:25.191925 error: + process: <0.74.0> + exit_reason: "Something went wrong"</code> + + </desc> + </func> + + </funcs> + +</erlref> + + diff --git a/lib/kernel/doc/src/logger_std_h.xml b/lib/kernel/doc/src/logger_std_h.xml new file mode 100644 index 0000000000..fe9b9ca5a9 --- /dev/null +++ b/lib/kernel/doc/src/logger_std_h.xml @@ -0,0 +1,133 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>2017</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>logger_std_h</title> + <prepared></prepared> + <responsible></responsible> + <docno></docno> + <approved></approved> + <checked></checked> + <date></date> + <rev>A</rev> + <file>logger_std_h.xml</file> + </header> + <module>logger_std_h</module> + <modulesummary>Default handler for the Logger application.</modulesummary> + + <description> + <p>This is the default handler for the Logger + application. Multiple instances of this handler can be added to + logger, and each instance will print logs to <c>standard_io</c>, + <c>standard_error</c> or to file. The default instance that starts + with kernel is named <c>logger_std_h</c> - which is the name to be used + for reconfiguration.</p> + <p>The handler has an overload protection mechanism that will keep the handler + process and the kernel application alive during a high load of log + requests. How this feature works, and how to modify the configuration, + is described in the + <seealso marker="logger_chapter#overload_protection"><c>User's Guide</c> + </seealso>.</p> + <p>To add a new instance of the standard handler, use + <seealso marker="logger#add_handler-3"><c>logger:add_handler/3</c> + </seealso>. The handler configuration argument is a map which may contain + general configuration parameters, as documented in the + <seealso marker="logger_chapter#handler_configuration"><c>User's Guide</c> + </seealso>, as well as handler specific parameters. The specific parameters + are stored in a sub map with the key <c>logger_std_h</c>. The following + keys and values may be specified:</p> + <taglist> + <tag><c>type</c></tag> + <item> + <p>This will have the value <c>standard_io</c>, <c>standard_error</c>, + <c>{file,LogFileName}</c>, or <c>{file,LogFileName,LogFileOpts}</c>, + where <c>standard_io</c> is the default value for type. It's recommended + to not specify <c>LogFileOpts</c> if not absolutely necessary. The + default options used by the handler to open a file for logging are: + <c>raw</c>, <c>append</c> and <c>delayed_write</c>. The standard + handler does not have support for circular logging. Use the + <seealso marker="logger_disk_log_h"><c>logger_disk_log_h</c> + </seealso> handler for this.</p></item> + <tag><c>filesync_repeat_interval</c></tag> + <item> + <p>This value (in milliseconds) specifies how often the handler will + do a file sync operation in order to make sure that buffered data gets + written to disk. The handler will repeatedly attempt this + operation, but only perform it if something has actually been logged + since the last sync. The default value is <c>5000</c> milliseconds. + If <c>no_repeat</c> is set as value, the repeated file sync operation + is disabled, and it will be the operating system settings that determine + how quickly or slowly data gets written to disk. The user can also call + the <seealso marker="logger_std_h#filesync-1"><c>filesync/1</c></seealso> + function to perform a file sync.</p></item> + </taglist> + <p>There are a number of other configuration parameters available, that are + to be used for customizing the overload protection behaviour. The same + parameters are used both in the standard handler and the disk_log handler, + and are documented in the + <seealso marker="logger_chapter#overload_protection"><c>User's Guide</c> + </seealso>.</p> + <p>Note that when changing the configuration of the handler in runtime, by + calling + <seealso marker="logger#set_handler_config-2"><c>logger:set_handler_config/2</c> + </seealso>, or + <seealso marker="logger#set_handler_config-3"><c>logger:set_handler_config/3</c> + </seealso>, + the <c>type</c> parameter may not be modified.</p> + <p>Example of adding a standard handler:</p> + <code type="none"> +logger:add_handler(my_standard_h, logger_std_h, + #{level => info, + filter_default => log, + logger_std_h => + #{type => {file,"./system_info.log"}, + filesync_repeat_interval => 1000}}). + </code> + <p>In order to configure the default handler (that starts initially with + the kernel application) to log to file instead of <c>standard_io</c>, + use the kernel configuration parameter + <seealso marker="kernel_app#configuration"><c>logger_dest</c></seealso> with + value <c>{file,FileName}</c>. Example:</p> + <code type="none"> +erl -kernel logger_dest '{file,"./erl.log"}' + </code> + <p>An example of how to replace the standard handler with a disk_log handler + at startup can be found in the manual of + <seealso marker="logger_disk_log_h"><c>logger_disk_log_h</c></seealso>.</p> + </description> + + <funcs> + + <func> + <name name="filesync" arity="1" clause_i="1"/> + <fsummary>Writes buffered data to disk.</fsummary> + <desc> + <p>Write buffered data to disk.</p> + </desc> + </func> + + </funcs> + +</erlref> + + diff --git a/lib/kernel/doc/src/part.xml b/lib/kernel/doc/src/part.xml new file mode 100644 index 0000000000..68eb4530e2 --- /dev/null +++ b/lib/kernel/doc/src/part.xml @@ -0,0 +1,40 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE part SYSTEM "part.dtd"> + +<part xmlns:xi="http://www.w3.org/2001/XInclude"> + <header> + <copyright> + <year>1996</year><year>2017</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>Logger User's Guide</title> + <prepared>OTP Team</prepared> + <docno></docno> + <date>2017-12-01</date> + <rev>0.1</rev> + <file>part.xml</file> + </header> + <description> + <p>The System Architecture Support Libraries SASL application + provides support for alarm handling, release handling, and + related functions.</p> + </description> + <xi:include href="introduction_chapter.xml"/> + <xi:include href="logger_chapter.xml"/> +</part> + diff --git a/lib/kernel/doc/src/ref_man.xml b/lib/kernel/doc/src/ref_man.xml index 5cd77e0f6f..c06914d23d 100644 --- a/lib/kernel/doc/src/ref_man.xml +++ b/lib/kernel/doc/src/ref_man.xml @@ -52,6 +52,11 @@ <xi:include href="inet.xml"/> <xi:include href="inet_res.xml"/> <xi:include href="init_stub.xml"/> + <xi:include href="logger.xml"/> + <xi:include href="logger_filters.xml"/> + <xi:include href="logger_formatter.xml"/> + <xi:include href="logger_std_h.xml"/> + <xi:include href="logger_disk_log_h.xml"/> <xi:include href="net_adm.xml"/> <xi:include href="net_kernel.xml"/> <xi:include href="os.xml"/> diff --git a/lib/kernel/doc/src/seq_trace.xml b/lib/kernel/doc/src/seq_trace.xml index 197851021f..69eb12a8a0 100644 --- a/lib/kernel/doc/src/seq_trace.xml +++ b/lib/kernel/doc/src/seq_trace.xml @@ -80,13 +80,18 @@ seq_trace:set_token(OldToken), % activate the trace token again <p>Sets the individual <c><anno>Component</anno></c> of the trace token to <c><anno>Val</anno></c>. Returns the previous value of the component.</p> <taglist> - <tag><c>set_token(label, <anno>Integer</anno>)</c></tag> + <tag><c>set_token(label, <anno>Label</anno>)</c></tag> <item> - <p>The <c>label</c> component is an integer which + <p>The <c>label</c> component is a term which identifies all events belonging to the same sequential trace. If several sequential traces can be active simultaneously, <c>label</c> is used to identify the separate traces. Default is 0.</p> + <warning> + <p>Labels were restricted to small signed integers (28 bits) + prior to OTP 21. The trace token will be silenty dropped if it + crosses over to a node that does not support the label.</p> + </warning> </item> <tag><c>set_token(serial, SerialValue)</c></tag> <item> diff --git a/lib/kernel/doc/src/specs.xml b/lib/kernel/doc/src/specs.xml index 29d52f23bb..bcc422930e 100644 --- a/lib/kernel/doc/src/specs.xml +++ b/lib/kernel/doc/src/specs.xml @@ -20,6 +20,11 @@ <xi:include href="../specs/specs_inet.xml"/> <xi:include href="../specs/specs_inet_res.xml"/> <xi:include href="../specs/specs_init_stub.xml"/> + <xi:include href="../specs/specs_logger.xml"/> + <xi:include href="../specs/specs_logger_filters.xml"/> + <xi:include href="../specs/specs_logger_formatter.xml"/> + <xi:include href="../specs/specs_logger_std_h.xml"/> + <xi:include href="../specs/specs_logger_disk_log_h.xml"/> <xi:include href="../specs/specs_net_adm.xml"/> <xi:include href="../specs/specs_net_kernel.xml"/> <xi:include href="../specs/specs_os.xml"/> diff --git a/lib/kernel/include/dist.hrl b/lib/kernel/include/dist.hrl index b7c35712a6..6baaa35d72 100644 --- a/lib/kernel/include/dist.hrl +++ b/lib/kernel/include/dist.hrl @@ -41,6 +41,7 @@ -define(DFLAG_MAP_TAG, 16#20000). -define(DFLAG_BIG_CREATION, 16#40000). -define(DFLAG_SEND_SENDER, 16#80000). +-define(DFLAG_BIG_SEQTRACE_LABELS, 16#100000). %% Also update dflag2str() in ../src/dist_util.erl %% when adding flags... diff --git a/lib/kernel/include/logger.hrl b/lib/kernel/include/logger.hrl new file mode 100644 index 0000000000..2143ccd297 --- /dev/null +++ b/lib/kernel/include/logger.hrl @@ -0,0 +1,49 @@ +-ifndef(LOGGER_HRL). +-define(LOGGER_HRL,true). +-define(LOG_EMERGENCY(A),?DO_LOG(emergency,[A])). +-define(LOG_EMERGENCY(A,B),?DO_LOG(emergency,[A,B])). +-define(LOG_EMERGENCY(A,B,C),?DO_LOG(emergency,[A,B,C])). + +-define(LOG_ALERT(A),?DO_LOG(alert,[A])). +-define(LOG_ALERT(A,B),?DO_LOG(alert,[A,B])). +-define(LOG_ALERT(A,B,C),?DO_LOG(alert,[A,B,C])). + +-define(LOG_CRITICAL(A),?DO_LOG(critical,[A])). +-define(LOG_CRITICAL(A,B),?DO_LOG(critical,[A,B])). +-define(LOG_CRITICAL(A,B,C),?DO_LOG(critical,[A,B,C])). + +-define(LOG_ERROR(A),?DO_LOG(error,[A])). +-define(LOG_ERROR(A,B),?DO_LOG(error,[A,B])). +-define(LOG_ERROR(A,B,C),?DO_LOG(error,[A,B,C])). + +-define(LOG_WARNING(A),?DO_LOG(warning,[A])). +-define(LOG_WARNING(A,B),?DO_LOG(warning,[A,B])). +-define(LOG_WARNING(A,B,C),?DO_LOG(warning,[A,B,C])). + +-define(LOG_NOTICE(A),?DO_LOG(notice,[A])). +-define(LOG_NOTICE(A,B),?DO_LOG(notice,[A,B])). +-define(LOG_NOTICE(A,B,C),?DO_LOG(notice,[A,B,C])). + +-define(LOG_INFO(A),?DO_LOG(info,[A])). +-define(LOG_INFO(A,B),?DO_LOG(info,[A,B])). +-define(LOG_INFO(A,B,C),?DO_LOG(info,[A,B,C])). + +-define(LOG_DEBUG(A),?DO_LOG(debug,[A])). +-define(LOG_DEBUG(A,B),?DO_LOG(debug,[A,B])). +-define(LOG_DEBUG(A,B,C),?DO_LOG(debug,[A,B,C])). + +-define(LOCATION,#{mfa=>{?MODULE,?FUNCTION_NAME,?FUNCTION_ARITY}, + line=>?LINE, + file=>?FILE}). + +%%%----------------------------------------------------------------- +%%% Internal, i.e. not intended for direct use in code - use above +%%% macros instead! +-define(DO_LOG(Level,Args), + case logger:allow(Level,?MODULE) of + true -> + apply(logger,macro_log,[?LOCATION,Level|Args]); + false -> + ok + end). +-endif. diff --git a/lib/kernel/src/Makefile b/lib/kernel/src/Makefile index 0bc9f121a0..702845512c 100644 --- a/lib/kernel/src/Makefile +++ b/lib/kernel/src/Makefile @@ -109,6 +109,17 @@ MODULES = \ kernel_refc \ local_udp \ local_tcp \ + logger \ + logger_backend \ + logger_config \ + logger_std_h \ + logger_disk_log_h \ + logger_h_common \ + logger_filters \ + logger_formatter \ + logger_server \ + logger_simple \ + logger_sup \ net \ net_adm \ net_kernel \ @@ -132,13 +143,14 @@ MODULES = \ HRL_FILES= ../include/file.hrl ../include/inet.hrl ../include/inet_sctp.hrl \ ../include/dist.hrl ../include/dist_util.hrl \ - ../include/net_address.hrl + ../include/net_address.hrl ../include/logger.hrl INTERNAL_HRL_FILES= application_master.hrl disk_log.hrl \ erl_epmd.hrl hipe_ext_format.hrl \ inet_dns.hrl inet_res.hrl \ inet_boot.hrl inet_config.hrl inet_int.hrl \ - inet_dns_record_adts.hrl + inet_dns_record_adts.hrl \ + logger_internal.hrl logger_h_common.hrl ERL_FILES= $(MODULES:%=%.erl) @@ -223,7 +235,7 @@ release_docs_spec: # Include dependencies -- list below added by Kostis Sagonas -$(EBIN)/application_controller.beam: application_master.hrl +$(EBIN)/application_controller.beam: application_master.hrl ../include/logger.hrl $(EBIN)/application_master.beam: application_master.hrl $(EBIN)/auth.beam: ../include/file.hrl $(EBIN)/code.beam: ../include/file.hrl @@ -234,6 +246,7 @@ $(EBIN)/disk_log_server.beam: disk_log.hrl $(EBIN)/dist_util.beam: ../include/dist_util.hrl ../include/dist.hrl $(EBIN)/erl_boot_server.beam: inet_boot.hrl $(EBIN)/erl_epmd.beam: inet_int.hrl erl_epmd.hrl +$(EBIN)/error_logger.beam: logger_internal.hrl ../include/logger.hrl $(EBIN)/file.beam: ../include/file.hrl file_int.hrl $(EBIN)/file_io_server.beam: ../include/file.hrl file_int.hrl $(EBIN)/gen_tcp.beam: inet_int.hrl @@ -259,6 +272,16 @@ $(EBIN)/inet_udp.beam: inet_int.hrl $(EBIN)/inet_sctp.beam: inet_int.hrl ../include/inet_sctp.hrl $(EBIN)/local_udp.beam: inet_int.hrl $(EBIN)/local_tcp.beam: inet_int.hrl +$(EBIN)/logger.beam: logger_internal.hrl ../include/logger.hrl +$(EBIN)/logger_backend.beam: logger_internal.hrl ../include/logger.hrl +$(EBIN)/logger_config.beam: logger_internal.hrl ../include/logger.hrl +$(EBIN)/logger_disk_log_h.beam: logger_h_common.hrl logger_internal.hrl ../include/logger.hrl ../include/file.hrl +$(EBIN)/logger_filters.beam: logger_internal.hrl ../include/logger.hrl +$(EBIN)/logger_formatter.beam: logger_internal.hrl ../include/logger.hrl +$(EBIN)/logger_server.beam: logger_internal.hrl ../include/logger.hrl +$(EBIN)/logger_simple.beam: logger_internal.hrl ../include/logger.hrl +$(EBIN)/logger_std_h.beam: logger_h_common.hrl logger_internal.hrl ../include/logger.hrl ../include/file.hrl +$(EBIN)/logger_h_common.beam: logger_h_common.hrl logger_internal.hrl ../include/logger.hrl $(EBIN)/net_kernel.beam: ../include/net_address.hrl $(EBIN)/os.beam: ../include/file.hrl $(EBIN)/ram_file.beam: ../include/file.hrl diff --git a/lib/kernel/src/application_controller.erl b/lib/kernel/src/application_controller.erl index 3b642f5873..b9cb722575 100644 --- a/lib/kernel/src/application_controller.erl +++ b/lib/kernel/src/application_controller.erl @@ -44,6 +44,7 @@ keyfind/3, keydelete/3, keyreplace/4]). -include("application_master.hrl"). +-include("logger.hrl"). -define(AC, ?MODULE). % Name of process @@ -1546,9 +1547,8 @@ do_change_apps(Applications, Config, OldAppls) -> %% Report errors, but do not terminate %% (backwards compatible behaviour) lists:foreach(fun({error, {SysFName, Line, Str}}) -> - Str2 = lists:flatten(io_lib:format("~tp: ~w: ~ts~n", - [SysFName, Line, Str])), - error_logger:format(Str2, []) + ?LOG_ERROR("~tp: ~w: ~ts~n",[SysFName, Line, Str], + #{error_logger=>#{tag=>error}}) end, Errors), @@ -1631,8 +1631,9 @@ make_term(Str) -> end. handle_make_term_error(Mod, Reason, Str) -> - error_logger:format("application_controller: ~ts: ~ts~n", - [Mod:format_error(Reason), Str]), + ?LOG_ERROR("application_controller: ~ts: ~ts~n", + [Mod:format_error(Reason), Str], + #{error_logger=>#{tag=>error}}), throw({error, {bad_environment_value, Str}}). get_env_i(Name, #state{conf_data = ConfData}) when is_list(ConfData) -> @@ -1913,19 +1914,25 @@ config_error() -> "configuration file must contain ONE list ended by <dot>"}}. %%----------------------------------------------------------------- -%% Info messages sent to error_logger +%% Info messages sent to logger %%----------------------------------------------------------------- info_started(Name, Node) -> - Rep = [{application, Name}, - {started_at, Node}], - error_logger:info_report(progress, Rep). + ?LOG_INFO(#{label=>{application_controller,progress}, + report=>[{application, Name}, + {started_at, Node}]}, + #{domain=>[beam,erlang,otp,sasl], + report_cb=>fun logger:format_otp_report/1, + logger_formatter=>#{title=>"PROGRESS REPORT"}, + error_logger=>#{tag=>info_report,type=>progress}}). info_exited(Name, Reason, Type) -> - Rep = [{application, Name}, - {exited, Reason}, - {type, Type}], - error_logger:info_report(Rep). - + ?LOG_INFO(#{label=>{application_controller,exit}, + report=>[{application, Name}, + {exited, Reason}, + {type, Type}]}, + #{domain=>[beam,erlang,otp], + report_cb=>fun logger:format_otp_report/1, + error_logger=>#{tag=>info_report,type=>std_info}}). %%----------------------------------------------------------------- %% Reply to all processes waiting this application to be started. diff --git a/lib/kernel/src/auth.erl b/lib/kernel/src/auth.erl index 40feee6bf0..a2116d8e8a 100644 --- a/lib/kernel/src/auth.erl +++ b/lib/kernel/src/auth.erl @@ -107,7 +107,7 @@ get_cookie() -> get_cookie(_Node) when node() =:= nonode@nohost -> nocookie; get_cookie(Node) -> - gen_server:call(auth, {get_cookie, Node}). + gen_server:call(auth, {get_cookie, Node}, infinity). -spec set_cookie(Cookie :: cookie()) -> 'true'. @@ -119,12 +119,12 @@ set_cookie(Cookie) -> set_cookie(_Node, _Cookie) when node() =:= nonode@nohost -> erlang:error(distribution_not_started); set_cookie(Node, Cookie) -> - gen_server:call(auth, {set_cookie, Node, Cookie}). + gen_server:call(auth, {set_cookie, Node, Cookie}, infinity). -spec sync_cookie() -> any(). sync_cookie() -> - gen_server:call(auth, sync_cookie). + gen_server:call(auth, sync_cookie, infinity). -spec print(Node :: node(), Format :: string(), Args :: [_]) -> 'ok'. diff --git a/lib/kernel/src/code.erl b/lib/kernel/src/code.erl index 9969021a6c..f143a49d2f 100644 --- a/lib/kernel/src/code.erl +++ b/lib/kernel/src/code.erl @@ -149,8 +149,11 @@ load_file(Mod) when is_atom(Mod) -> -spec ensure_loaded(Module) -> {module, Module} | {error, What} when Module :: module(), What :: embedded | badfile | nofile | on_load_failure. -ensure_loaded(Mod) when is_atom(Mod) -> - call({ensure_loaded,Mod}). +ensure_loaded(Mod) when is_atom(Mod) -> + case erlang:module_loaded(Mod) of + true -> {module, Mod}; + false -> call({ensure_loaded,Mod}) + end. %% XXX File as an atom is allowed only for backwards compatibility. -spec load_abs(Filename) -> load_ret() when diff --git a/lib/kernel/src/code_server.erl b/lib/kernel/src/code_server.erl index f5a890cb95..bbfa2a995d 100644 --- a/lib/kernel/src/code_server.erl +++ b/lib/kernel/src/code_server.erl @@ -1434,14 +1434,20 @@ all_loaded(Db) -> -spec error_msg(io:format(), [term()]) -> 'ok'. error_msg(Format, Args) -> - Msg = {notify,{error, group_leader(), {self(), Format, Args}}}, - error_logger ! Msg, + logger ! {log,error,Format,Args, + #{pid=>self(), + gl=>group_leader(), + time=>erlang:monotonic_time(microsecond), + error_logger=>#{tag=>error}}}, ok. -spec info_msg(io:format(), [term()]) -> 'ok'. info_msg(Format, Args) -> - Msg = {notify,{info_msg, group_leader(), {self(), Format, Args}}}, - error_logger ! Msg, + logger ! {log,info,Format,Args, + #{pid=>self(), + gl=>group_leader(), + time=>erlang:monotonic_time(microsecond), + error_logger=>#{tag=>info_msg}}}, ok. objfile_extension() -> diff --git a/lib/kernel/src/disk_log_1.erl b/lib/kernel/src/disk_log_1.erl index 93856aa7b3..b456b53d20 100644 --- a/lib/kernel/src/disk_log_1.erl +++ b/lib/kernel/src/disk_log_1.erl @@ -630,7 +630,7 @@ is_head(Bin) when is_binary(Bin) -> %% Writes MaxB bytes on each file. %% Creates a file called Name.idx in the Dir. This %% file contains the last written FileName as one byte, and -%% follwing that, the sizes of each file (size 0 number of items). +%% following that, the sizes of each file (size 0 number of items). %% On startup, this file is read, and the next available %% filename is used as first log file. %% Reports can be browsed with Report Browser Tool (rb), or diff --git a/lib/kernel/src/dist_util.erl b/lib/kernel/src/dist_util.erl index f7a84c14b4..781397e1ee 100644 --- a/lib/kernel/src/dist_util.erl +++ b/lib/kernel/src/dist_util.erl @@ -113,6 +113,8 @@ dflag2str(?DFLAG_BIG_CREATION) -> "BIG_CREATION"; dflag2str(?DFLAG_SEND_SENDER) -> "SEND_SENDER"; +dflag2str(?DFLAG_BIG_SEQTRACE_LABELS) -> + "BIG_SEQTRACE_LABELS"; dflag2str(_) -> "UNKNOWN". diff --git a/lib/kernel/src/error_logger.erl b/lib/kernel/src/error_logger.erl index 585507c545..0706220a94 100644 --- a/lib/kernel/src/error_logger.erl +++ b/lib/kernel/src/error_logger.erl @@ -19,22 +19,23 @@ %% -module(error_logger). --export([start/0,start_link/0,format/2,error_msg/1,error_msg/2,error_report/1, +-include("logger_internal.hrl"). + +-export([start/0,start_link/0,stop/0, + format/2,error_msg/1,error_msg/2,error_report/1, error_report/2,info_report/1,info_report/2,warning_report/1, warning_report/2,error_info/1, info_msg/1,info_msg/2,warning_msg/1,warning_msg/2, - logfile/1,tty/1,swap_handler/1, + logfile/1,tty/1, add_report_handler/1,add_report_handler/2, - delete_report_handler/1]). + delete_report_handler/1, + which_report_handlers/0]). --export([init/1, - handle_event/2, handle_call/2, handle_info/2, - terminate/2]). +%% logger callbacks +-export([adding_handler/2, removing_handler/1, log/2]). -export([get_format_depth/0, limit_term/1]). --define(buffer_size, 10). - %%----------------------------------------------------------------- %% Types used in this file %%----------------------------------------------------------------- @@ -43,8 +44,6 @@ | 'info' | 'info_msg' | 'info_report' | 'warning_msg' | 'warning_report'. --type state() :: {non_neg_integer(), non_neg_integer(), [term()]}. - %%% BIF -export([warning_map/0]). @@ -59,26 +58,137 @@ warning_map() -> %%----------------------------------------------------------------- --spec start() -> {'ok', pid()} | {'error', any()}. +%%%----------------------------------------------------------------- +%%% Start the event manager process under logger_sup, which is part of +%%% the kernel application's supervision tree. +-spec start() -> 'ok' | {'error', any()}. start() -> - case gen_event:start({local, error_logger}) of - {ok, Pid} -> - simple_logger(?buffer_size), - {ok, Pid}; - Error -> Error + case whereis(?MODULE) of + undefined -> + ErrorLogger = + #{id => ?MODULE, + start => {?MODULE, start_link, []}, + restart => transient, + shutdown => 2000, + type => worker, + modules => dynamic}, + case supervisor:start_child(logger_sup, ErrorLogger) of + {ok,_} -> + ok; + Error -> + Error + end; + _ -> + ok end. +%%%----------------------------------------------------------------- +%%% Start callback specified in child specification to supervisor, see start/0 -spec start_link() -> {'ok', pid()} | {'error', any()}. start_link() -> - case gen_event:start_link({local, error_logger}) of - {ok, Pid} -> - simple_logger(?buffer_size), - {ok, Pid}; - Error -> Error + gen_event:start_link({local, ?MODULE}, + [{spawn_opt,[{message_queue_data, off_heap}]}]). + +%%%----------------------------------------------------------------- +%%% Stop the event manager +-spec stop() -> ok. +stop() -> + _ = supervisor:terminate_child(logger_sup,?MODULE), + _ = supervisor:delete_child(logger_sup,?MODULE), + ok. + +%%%----------------------------------------------------------------- +%%% Callbacks for logger +-spec adding_handler(logger:handler_id(),logger:config()) -> + {ok,logger:config()} | {error,term()}. +adding_handler(?MODULE,Config) -> + case start() of + ok -> + {ok,Config}; + Error -> + Error end. +-spec removing_handler(logger:handler_id()) -> ok. +removing_handler(?MODULE) -> + stop(), + ok. + +-spec log(logger:log(),logger:config()) -> ok. +log(#{level:=Level,msg:=Msg,meta:=Meta},_Config) -> + do_log(Level,Msg,Meta). + +do_log(Level,{report,Msg},#{?MODULE:=#{tag:=Tag,type:=Type}}=Meta) -> + %% From error_logger:*_report/1,2, or logger call which added + %% error_logger data to obtain backwards compatibility with + %% error_logger:*_report/1,2 + Report = + case Msg of + #{label:=_,report:=R} -> R; + _ -> Msg + end, + notify(Level,Tag,Type,Report,Meta); +do_log(Level,{report,Msg},#{?MODULE:=#{tag:=Tag}}=Meta) -> + {Format,Args} = + case Msg of + #{label:=_,format:=F,args:=A} -> + %% From error_logger:*_msg/1,2. + %% In order to be backwards compatible with handling + %% of faulty parameters to error_logger:*_msg/1,2, + %% don't use report_cb here. + {F,A}; + _ -> + %% From logger call which added error_logger data to + %% obtain backwards compatibility with error_logger:*_msg/1,2 + RCBFun=maps:get(report_cb,Meta,fun logger:format_report/1), + try RCBFun(Msg) of + {F,A} when is_list(F), is_list(A) -> + {F,A}; + Other -> + {"REPORT_CB ERROR: ~tp; Returned: ~tp",[Msg,Other]} + catch C:R -> + {"REPORT_CB CRASH: ~tp; Reason: ~tp",[Msg,{C,R}]} + end + end, + notify(Level,Tag,Format,Args,Meta); +do_log(Level,{Format,Args},#{?MODULE:=#{tag:=Tag}}=Meta) + when is_list(Format), is_list(Args) -> + %% From logger call which added error_logger data to obtain + %% backwards compatibility with error_logger:*_msg/1,2 + notify(Level,Tag,Format,Args,Meta); +do_log(_Level,_Msg,_Meta) -> + %% Ignore the rest - i.e. to get backwards compatibility with + %% error_logger, you must use the error_logger API for logging. + %% Some modules within OTP go around this by adding an + %% error_logger field to its metadata. This is done only to allow + %% complete backwards compatibility for log events originating + %% from within OTP, while still using the new logger interface. + ok. + +-spec notify(logger:level(), msg_tag(), any(), any(), map()) -> 'ok'. +notify(Level,Tag0,FormatOrType0,ArgsOrReport,#{pid:=Pid0,gl:=GL,?MODULE:=My}) -> + Tag = fix_warning_tag(Level,Tag0), + Pid = case maps:get(emulator,My,false) of + true -> emulator; + _ -> Pid0 + end, + FormatOrType = fix_warning_type(Level,FormatOrType0), + gen_event:notify(?MODULE,{Tag,GL,{Pid,FormatOrType,ArgsOrReport}}). + +%% This is to fix the case when the client has explicitly added the +%% error logger tag and type in metadata, and not checked the warning map. +fix_warning_tag(error,warning_msg) -> error; +fix_warning_tag(error,warning_report) -> error_report; +fix_warning_tag(info,warning_msg) -> info_msg; +fix_warning_tag(info,warning_report) -> info_report; +fix_warning_tag(_,Tag) -> Tag. + +fix_warning_type(error,std_warning) -> std_error; +fix_warning_type(info,std_warning) -> std_info; +fix_warning_type(_,Type) -> Type. + %%----------------------------------------------------------------- %% These two simple old functions generate events tagged 'error' %% Used for simple messages; error or information. @@ -95,14 +205,18 @@ error_msg(Format) -> Data :: list(). error_msg(Format, Args) -> - notify({error, group_leader(), {self(), Format, Args}}). + logger:log(error, + #{label=>{?MODULE,error_msg}, + format=>Format, + args=>Args}, + meta(error)). -spec format(Format, Data) -> 'ok' when Format :: string(), Data :: list(). format(Format, Args) -> - notify({error, group_leader(), {self(), Format, Args}}). + error_msg(Format, Args). %%----------------------------------------------------------------- %% This functions should be used for error reports. Events @@ -124,7 +238,10 @@ error_report(Report) -> Report :: report(). error_report(Type, Report) -> - notify({error_report, group_leader(), {self(), Type, Report}}). + logger:log(error, + #{label=>{?MODULE,error_report}, + report=>Report}, + meta(error_report,Type)). %%----------------------------------------------------------------- %% This function should be used for warning reports. @@ -146,7 +263,8 @@ warning_report(Report) -> Report :: report(). warning_report(Type, Report) -> - {Tag, NType} = case error_logger:warning_map() of + Level = error_logger:warning_map(), + {Tag, NType} = case Level of info -> if Type =:= std_warning -> @@ -164,7 +282,10 @@ warning_report(Type, Report) -> {error_report, Type} end end, - notify({Tag, group_leader(), {self(), NType, Report}}). + logger:log(Level, + #{label=>{?MODULE,warning_report}, + report=>Report}, + meta(Tag,NType)). %%----------------------------------------------------------------- %% This function provides similar functions as error_msg for @@ -183,7 +304,8 @@ warning_msg(Format) -> Data :: list(). warning_msg(Format, Args) -> - Tag = case error_logger:warning_map() of + Level = error_logger:warning_map(), + Tag = case Level of warning -> warning_msg; info -> @@ -191,7 +313,11 @@ warning_msg(Format, Args) -> error -> error end, - notify({Tag, group_leader(), {self(), Format, Args}}). + logger:log(Level, + #{label=>{?MODULE,warning_msg}, + format=>Format, + args=>Args}, + meta(Tag)). %%----------------------------------------------------------------- %% This function should be used for information reports. Events @@ -210,7 +336,10 @@ info_report(Report) -> Report :: report(). info_report(Type, Report) -> - notify({info_report, group_leader(), {self(), Type, Report}}). + logger:log(info, + #{label=>{?MODULE,info_report}, + report=>Report}, + meta(info_report,Type)). %%----------------------------------------------------------------- %% This function provides similar functions as error_msg for @@ -228,7 +357,11 @@ info_msg(Format) -> Data :: list(). info_msg(Format, Args) -> - notify({info_msg, group_leader(), {self(), Format, Args}}). + logger:log(info, + #{label=>{?MODULE,info_msg}, + format=>Format, + args=>Args}, + meta(info_msg)). %%----------------------------------------------------------------- %% Used by the init process. Events are tagged 'info'. @@ -236,38 +369,75 @@ info_msg(Format, Args) -> -spec error_info(Error :: any()) -> 'ok'. +%% unused? error_info(Error) -> - notify({info, group_leader(), {self(), Error, []}}). - --spec notify({msg_tag(), pid(), {pid(), any(), any()}}) -> 'ok'. - -notify(Msg) -> - gen_event:notify(error_logger, Msg). - --type swap_handler_type() :: 'false' | 'silent' | 'tty' | {'logfile', string()}. --spec swap_handler(Type :: swap_handler_type()) -> any(). - -swap_handler(tty) -> - R = gen_event:swap_handler(error_logger, {error_logger, swap}, - {error_logger_tty_h, []}), - ok = simple_logger(), - R; -swap_handler({logfile, File}) -> - R = gen_event:swap_handler(error_logger, {error_logger, swap}, - {error_logger_file_h, File}), - ok = simple_logger(), - R; -swap_handler(silent) -> - _ = gen_event:delete_handler(error_logger, error_logger, delete), - ok = simple_logger(); -swap_handler(false) -> - ok. % keep primitive event handler as-is + {Format,Args} = + case string_p(Error) of + true -> {Error,[]}; + false -> {"~p",[Error]} + end, + MyMeta = #{tag=>info,type=>Error}, + logger:log(info, Format, Args, #{?MODULE=>MyMeta,domain=>[Error]}). + +%%----------------------------------------------------------------- +%% Create metadata +meta(Tag) -> + meta(Tag,undefined). +meta(Tag,Type) -> + meta(Tag,Type,#{report_cb=>fun report_to_format/1}). +meta(Tag,undefined,Meta0) -> + Meta0#{?MODULE=>#{tag=>Tag}}; +meta(Tag,Type,Meta0) -> + maybe_add_domain(Tag,Type,Meta0#{?MODULE=>#{tag=>Tag,type=>Type}}). + +%% This is to prevent events of non standard type from being printed +%% with the standard logger. Similar to how error_logger_tty_h +%% discards events of non standard type. +maybe_add_domain(error_report,std_error,Meta) -> Meta; +maybe_add_domain(info_report,std_info,Meta) -> Meta; +maybe_add_domain(warning_report,std_warning,Meta) -> Meta; +maybe_add_domain(_,Type,Meta) -> Meta#{domain=>[Type]}. + +%% ----------------------------------------------------------------- +%% Report formatting - i.e. Term => {Format,Args} +%% This was earlier done in the event handler (error_logger_tty_h, etc) +%% ----------------------------------------------------------------- +report_to_format(#{label:={?MODULE,_}, + report:=Report}) when is_map(Report) -> + %% logger:format_otp_report does maps:to_list, and for backwards + %% compatibility reasons we don't want that. + {"~tp\n",[Report]}; +report_to_format(#{label:={?MODULE,_}, + format:=Format, + args:=Args}) -> + %% This is not efficient, but needed for backwards compatibility + %% in giving faulty arguments to the *_msg functions. + try io_lib:scan_format(Format,Args) of + _ -> {Format,Args} + catch _:_ -> + {"ERROR: ~tp - ~tp",[Format,Args]} + end; +report_to_format(Term) -> + logger:format_otp_report(Term). +string_p(List) when is_list(List) -> + string_p1(lists:flatten(List)); +string_p(_) -> + false. + +string_p1([]) -> + false; +string_p1(FlatList) -> + io_lib:printable_list(FlatList). + +%% ----------------------------------------------------------------- +%% Stuff directly related to the event manager +%% ----------------------------------------------------------------- -spec add_report_handler(Handler) -> any() when Handler :: module(). add_report_handler(Module) when is_atom(Module) -> - gen_event:add_handler(error_logger, Module, []). + add_report_handler(Module, []). -spec add_report_handler(Handler, Args) -> Result when Handler :: module(), @@ -275,24 +445,37 @@ add_report_handler(Module) when is_atom(Module) -> Result :: gen_event:add_handler_ret(). add_report_handler(Module, Args) when is_atom(Module) -> - gen_event:add_handler(error_logger, Module, Args). + _ = logger:add_handler(?MODULE,?MODULE,#{level=>info,filter_default=>log}), + gen_event:add_handler(?MODULE, Module, Args). -spec delete_report_handler(Handler) -> Result when Handler :: module(), Result :: gen_event:del_handler_ret(). delete_report_handler(Module) when is_atom(Module) -> - gen_event:delete_handler(error_logger, Module, []). - -%% Start the lowest level error_logger handler with Buffer. - -simple_logger(Buffer_size) when is_integer(Buffer_size) -> - gen_event:add_handler(error_logger, error_logger, Buffer_size). - -%% Start the lowest level error_logger handler without Buffer. + case whereis(?MODULE) of + Pid when is_pid(Pid) -> + Return = gen_event:delete_handler(?MODULE, Module, []), + case gen_event:which_handlers(?MODULE) of + [] -> + %% Don't want a lot of logs here if it's not needed + _ = logger:remove_handler(?MODULE), + ok; + _ -> + ok + end, + Return; + _ -> + ok + end. -simple_logger() -> - gen_event:add_handler(error_logger, error_logger, []). +which_report_handlers() -> + case whereis(?MODULE) of + Pid when is_pid(Pid) -> + gen_event:which_handlers(?MODULE); + undefined -> + [] + end. %% Log all errors to File for all eternity @@ -308,26 +491,35 @@ simple_logger() -> FilenameReason :: no_log_file. logfile({open, File}) -> - case lists:member(error_logger_file_h, - gen_event:which_handlers(error_logger)) of + case lists:member(error_logger_file_h,which_report_handlers()) of true -> {error, allready_have_logfile}; _ -> - gen_event:add_handler(error_logger, error_logger_file_h, File) + add_report_handler(error_logger_file_h, File) end; logfile(close) -> - case gen_event:delete_handler(error_logger, error_logger_file_h, normal) of - {error,Reason} -> - {error,Reason}; - _ -> - ok + case whereis(?MODULE) of + Pid when is_pid(Pid) -> + case gen_event:delete_handler(?MODULE, error_logger_file_h, normal) of + {error,Reason} -> + {error,Reason}; + _ -> + ok + end; + _ -> + {error,module_not_found} end; logfile(filename) -> - case gen_event:call(error_logger, error_logger_file_h, filename) of - {error,_} -> - {error, no_log_file}; - Val -> - Val + case whereis(?MODULE) of + Pid when is_pid(Pid) -> + case gen_event:call(?MODULE, error_logger_file_h, filename) of + {error,_} -> + {error, no_log_file}; + Val -> + Val + end; + _ -> + {error, no_log_file} end. %% Possibly turn off all tty printouts, maybe we only want the errors @@ -337,193 +529,17 @@ logfile(filename) -> Flag :: boolean(). tty(true) -> - Hs = gen_event:which_handlers(error_logger), - case lists:member(error_logger_tty_h, Hs) of + case lists:member(error_logger_tty_h, which_report_handlers()) of false -> - gen_event:add_handler(error_logger, error_logger_tty_h, []); - true -> + add_report_handler(error_logger_tty_h, []); + true -> ignore end, ok; tty(false) -> - gen_event:delete_handler(error_logger, error_logger_tty_h, []), - ok. - + delete_report_handler(error_logger_tty_h). -%%% --------------------------------------------------- -%%% This is the default error_logger handler. -%%% --------------------------------------------------- - --spec init(term()) -> {'ok', state() | []}. - -init(Max) when is_integer(Max) -> - {ok, {Max, 0, []}}; -%% This one is called if someone took over from us, and now wants to -%% go back. -init({go_back, _PostState}) -> - {ok, {?buffer_size, 0, []}}; -init(_) -> - %% The error logger process may receive a huge amount of - %% messages. Make sure that they are stored off heap to - %% avoid exessive GCs. - process_flag(message_queue_data, off_heap), - {ok, []}. - --spec handle_event(term(), state()) -> {'ok', state()}. - -handle_event({Type, GL, Msg}, State) when node(GL) =/= node() -> - gen_event:notify({error_logger, node(GL)},{Type, GL, Msg}), - %% handle_event2({Type, GL, Msg}, State); %% Shall we do something - {ok, State}; %% at this node too ??? -handle_event({info_report, _, {_, Type, _}}, State) when Type =/= std_info -> - {ok, State}; %% Ignore other info reports here -handle_event(Event, State) -> - handle_event2(Event, State). - --spec handle_info(term(), state()) -> {'ok', state()}. - -handle_info({emulator, GL, Chars}, State) when node(GL) =/= node() -> - {error_logger, node(GL)} ! {emulator, GL, add_node(Chars,self())}, - {ok, State}; -handle_info({emulator, GL, Chars}, State) -> - handle_event2({emulator, GL, Chars}, State); -handle_info(_, State) -> - {ok, State}. - --spec handle_call(term(), state()) -> {'ok', {'error', 'bad_query'}, state()}. - -handle_call(_Query, State) -> {ok, {error, bad_query}, State}. - --spec terminate(term(), state()) -> {'error_logger', [term()]}. - -terminate(swap, {_, 0, Buff}) -> - {error_logger, Buff}; -terminate(swap, {_, Lost, Buff}) -> - Myevent = {info, group_leader(), {self(), {lost_messages, Lost}, []}}, - {error_logger, [tag_event(Myevent)|Buff]}; -terminate(_, _) -> - {error_logger, []}. - -handle_event2(Event, {1, Lost, Buff}) -> - display(tag_event(Event)), - {ok, {1, Lost+1, Buff}}; -handle_event2(Event, {N, Lost, Buff}) -> - Tagged = tag_event(Event), - display(Tagged), - {ok, {N-1, Lost, [Tagged|Buff]}}; -handle_event2(_, State) -> - {ok, State}. - -tag_event(Event) -> - {erlang:localtime(), Event}. - -display({Tag,{error,_,{_,Format,Args}}}) -> - display2(Tag,Format,Args); -display({Tag,{error_report,_,{_,Type,Report}}}) -> - display2(Tag,Type,Report); -display({Tag,{info_report,_,{_,Type,Report}}}) -> - display2(Tag,Type,Report); -display({Tag,{info,_,{_,Error,_}}}) -> - display2(Tag,Error,[]); -display({Tag,{info_msg,_,{_,Format,Args}}}) -> - display2(Tag,Format,Args); -display({Tag,{warning_report,_,{_,Type,Report}}}) -> - display2(Tag,Type,Report); -display({Tag,{warning_msg,_,{_,Format,Args}}}) -> - display2(Tag,Format,Args); -display({Tag,{emulator,_,Chars}}) -> - display2(Tag,Chars,[]). - -add_node(X, Pid) when is_atom(X) -> - add_node(atom_to_list(X), Pid); -add_node(X, Pid) -> - lists:concat([X,"** at node ",node(Pid)," **~n"]). - -%% Can't do io_lib:format - -display2({{_Y,_Mo,_D},{_H,_Mi,_S}} = Date, F, A) -> - display_date(Date), - display3(string_p(F), F, A). - -display_date({{Y,Mo,D},{H,Mi,S}}) -> - erlang:display_string( - integer_to_list(Y) ++ "-" ++ - two_digits(Mo) ++ "-" ++ - two_digits(D) ++ " " ++ - two_digits(H) ++ ":" ++ - two_digits(Mi) ++ ":" ++ - two_digits(S) ++ " "). - -two_digits(N) when 0 =< N, N =< 9 -> - [$0, $0 + N]; -two_digits(N) -> - integer_to_list(N). - -display3(true, F, A) -> - %% Format string with arguments - erlang:display_string(F ++ "\n"), - [begin - erlang:display_string("\t"), - erlang:display(Arg) - end || Arg <- A], - ok; -display3(false, Atom, A) when is_atom(Atom) -> - %% The widest atom seems to be 'supervisor_report' at 17. - ColumnWidth = 20, - AtomString = atom_to_list(Atom), - AtomLength = length(AtomString), - Padding = lists:duplicate(ColumnWidth - AtomLength, $\s), - erlang:display_string(AtomString ++ Padding), - display4(A); -display3(_, F, A) -> - erlang:display({F, A}). - -display4([A, []]) -> - %% Not sure why crash reports look like this. - display4(A); -display4(A = [_|_]) -> - case lists:all(fun({Key,_Value}) -> is_atom(Key); (_) -> false end, A) of - true -> - erlang:display_string("\n"), - lists:foreach( - fun({Key, Value}) -> - erlang:display_string( - " " ++ - atom_to_list(Key) ++ - ": "), - erlang:display(Value) - end, A); - false -> - erlang:display(A) - end; -display4(A) -> - erlang:display(A). - -string_p([]) -> - false; -string_p(Term) -> - string_p1(Term). - -string_p1([H|T]) when is_integer(H), H >= $\040, H =< $\176 -> - string_p1(T); -string_p1([H|T]) when is_integer(H), H >= 16#A0, H < 16#D800; - is_integer(H), H > 16#DFFF, H < 16#FFFE; - is_integer(H), H > 16#FFFF, H =< 16#10FFFF -> - string_p1(T); -string_p1([$\n|T]) -> string_p1(T); -string_p1([$\r|T]) -> string_p1(T); -string_p1([$\t|T]) -> string_p1(T); -string_p1([$\v|T]) -> string_p1(T); -string_p1([$\b|T]) -> string_p1(T); -string_p1([$\f|T]) -> string_p1(T); -string_p1([$\e|T]) -> string_p1(T); -string_p1([H|T]) when is_list(H) -> - case string_p1(H) of - true -> string_p1(T); - _ -> false - end; -string_p1([]) -> true; -string_p1(_) -> false. +%%%----------------------------------------------------------------- -spec limit_term(term()) -> term(). @@ -536,9 +552,4 @@ limit_term(Term) -> -spec get_format_depth() -> 'unlimited' | pos_integer(). get_format_depth() -> - case application:get_env(kernel, error_logger_format_depth) of - {ok, Depth} when is_integer(Depth) -> - max(10, Depth); - undefined -> - unlimited - end. + logger:get_format_depth(). diff --git a/lib/kernel/src/erts_debug.erl b/lib/kernel/src/erts_debug.erl index 3456c8511e..1270de4144 100644 --- a/lib/kernel/src/erts_debug.erl +++ b/lib/kernel/src/erts_debug.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2016. All Rights Reserved. +%% Copyright Ericsson AB 1999-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -32,11 +32,11 @@ %%% BIFs -export([breakpoint/2, disassemble/1, display/1, dist_ext_to_term/2, - dump_monitors/1, dump_links/1, flat_size/1, - get_internal_state/1, instructions/0, + flat_size/1, get_internal_state/1, instructions/0, map_info/1, same/2, set_internal_state/2, size_shared/1, copy_shared/1, dirty_cpu/2, dirty_io/2, dirty/3, - lcnt_control/1, lcnt_control/2, lcnt_collect/0, lcnt_clear/0]). + lcnt_control/1, lcnt_control/2, lcnt_collect/0, lcnt_clear/0, + lc_graph/0, lc_graph_to_dot/2, lc_graph_merge/2]). -spec breakpoint(MFA, Flag) -> non_neg_integer() when MFA :: {Module :: module(), @@ -70,18 +70,6 @@ display(_) -> dist_ext_to_term(_, _) -> erlang:nif_error(undef). --spec dump_monitors(Id) -> true when - Id :: pid() | atom(). - -dump_monitors(_) -> - erlang:nif_error(undef). - --spec dump_links(Id) -> true when - Id :: pid() | port() | atom(). - -dump_links(_) -> - erlang:nif_error(undef). - -spec flat_size(Term) -> non_neg_integer() when Term :: term(). @@ -420,3 +408,90 @@ cont_dis(_, {_,_,_}, _) -> ok. map_info(_) -> erlang:nif_error(undef). + +%% Create file "lc_graph.<pid>" with all actual lock dependencies +%% recorded so far by the VM. +%% Needs debug VM or --enable-lock-checking config, returns 'notsup' otherwise. +lc_graph() -> + erts_debug:set_internal_state(available_internal_state, true), + erts_debug:get_internal_state(lc_graph). + +%% Convert "lc_graph.<pid>" file to https://www.graphviz.org dot format. +lc_graph_to_dot(OutFile, InFile) -> + {ok, [LL0]} = file:consult(InFile), + + [{"NO LOCK",0} | LL] = LL0, + Map = maps:from_list([{Id, Name} || {Name, Id, _, _} <- LL]), + + case file:open(OutFile, [exclusive]) of + {ok, Out} -> + ok = file:write(Out, "digraph G {\n"), + + [dot_print_lock(Out, Lck, Map) || Lck <- LL], + + ok = file:write(Out, "}\n"), + ok = file:close(Out); + + {error,eexist} -> + {"File already exists", OutFile} + end. + +dot_print_lock(Out, {_Name, Id, Lst, _}, Map) -> + [dot_print_edge(Out, From, Id, Map) || From <- Lst], + ok. + +dot_print_edge(_, 0, _, _) -> + ignore; % "NO LOCK" +dot_print_edge(Out, From, To, Map) -> + io:format(Out, "~p -> ~p;\n", [maps:get(From,Map), maps:get(To,Map)]). + + +%% Merge several "lc_graph" files into one file. +lc_graph_merge(OutFile, InFiles) -> + LLs = lists:map(fun(InFile) -> + {ok, [LL]} = file:consult(InFile), + LL + end, + InFiles), + + Res = lists:foldl(fun(A, B) -> lcg_merge(A, B) end, + hd(LLs), + tl(LLs)), + case file:open(OutFile, [exclusive]) of + {ok, Out} -> + try + lcg_print(Out, Res) + after + file:close(Out) + end, + ok; + {error, eexist} -> + {"File already exists", OutFile} + end. + +lcg_merge(A, B) -> + lists:zipwith(fun(LA, LB) -> lcg_merge_locks(LA, LB) end, + A, B). + +lcg_merge_locks(L, L) -> + L; +lcg_merge_locks({Name, Id, DA, IA}, {Name, Id, DB, IB}) -> + Direct = lists:umerge(DA, DB), + Indirect = lists:umerge(IA, IB), + {Name, Id, Direct, Indirect -- Direct}. + + +lcg_print(Out, LL) -> + io:format(Out, "[", []), + lcg_print_locks(Out, LL), + io:format(Out, "].\n", []), + ok. + +lcg_print_locks(Out, [{_,_}=NoLock | Rest]) -> + io:format(Out, "~p,\n", [NoLock]), + lcg_print_locks(Out, Rest); +lcg_print_locks(Out, [LastLock]) -> + io:format(Out, "~w", [LastLock]); +lcg_print_locks(Out, [Lock | Rest]) -> + io:format(Out, "~w,\n", [Lock]), + lcg_print_locks(Out, Rest). diff --git a/lib/kernel/src/file.erl b/lib/kernel/src/file.erl index c2df1ee288..57d8fc7a15 100644 --- a/lib/kernel/src/file.erl +++ b/lib/kernel/src/file.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2017. All Rights Reserved. +%% Copyright Ericsson AB 1996-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -101,14 +101,25 @@ -type deep_list() :: [char() | atom() | deep_list()]. -type name() :: string() | atom() | deep_list(). -type name_all() :: string() | atom() | deep_list() | (RawFilename :: binary()). --type posix() :: 'eacces' | 'eagain' | 'ebadf' | 'ebusy' | 'edquot' - | 'eexist' | 'efault' | 'efbig' | 'eintr' | 'einval' - | 'eio' | 'eisdir' | 'eloop' | 'emfile' | 'emlink' - | 'enametoolong' - | 'enfile' | 'enodev' | 'enoent' | 'enomem' | 'enospc' - | 'enotblk' | 'enotdir' | 'enotsup' | 'enxio' | 'eperm' - | 'epipe' | 'erofs' | 'espipe' | 'esrch' | 'estale' - | 'exdev'. +-type posix() :: + 'eacces' | 'eagain' | + 'ebadf' | 'ebadmsg' | 'ebusy' | + 'edeadlk' | 'edeadlock' | 'edquot' | + 'eexist' | + 'efault' | 'efbig' | 'eftype' | + 'eintr' | 'einval' | 'eio' | 'eisdir' | + 'eloop' | + 'emfile' | 'emlink' | 'emultihop' | + 'enametoolong' | 'enfile' | + 'enobufs' | 'enodev' | 'enolck' | 'enolink' | 'enoent' | + 'enomem' | 'enospc' | 'enosr' | 'enostr' | 'enosys' | + 'enotblk' | 'enotdir' | 'enotsup' | 'enxio' | + 'eopnotsupp' | 'eoverflow' | + 'eperm' | 'epipe' | + 'erange' | 'erofs' | + 'espipe' | 'esrch' | 'estale' | + 'etxtbsy' | + 'exdev'. -type date_time() :: calendar:datetime(). -type posix_file_advise() :: 'normal' | 'sequential' | 'random' | 'no_reuse' | 'will_need' | 'dont_need'. diff --git a/lib/kernel/src/group.erl b/lib/kernel/src/group.erl index e1198d2587..2c0518ccad 100644 --- a/lib/kernel/src/group.erl +++ b/lib/kernel/src/group.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -114,7 +114,7 @@ server_loop(Drv, Shell, Buf0) -> {io_request,From,ReplyAs,Req} when is_pid(From) -> %% This io_request may cause a transition to a couple of %% selective receive loops elsewhere in this module. - Buf = io_request(Req, From, ReplyAs, Drv, Buf0), + Buf = io_request(Req, From, ReplyAs, Drv, Shell, Buf0), server_loop(Drv, Shell, Buf); {reply,{{From,ReplyAs},Reply}} -> io_reply(From, ReplyAs, Reply), @@ -135,7 +135,7 @@ server_loop(Drv, Shell, Buf0) -> exit(R); %% We want to throw away any term that we don't handle (standard %% practice in receive loops), but not any {Drv,_} tuples which are - %% handled in io_request/5. + %% handled in io_request/6. NotDrvTuple when (not is_tuple(NotDrvTuple)) orelse (tuple_size(NotDrvTuple) =/= 2) orelse (element(1, NotDrvTuple) =/= Drv) -> @@ -177,8 +177,8 @@ set_unicode_state(Drv,Bool) -> end. -io_request(Req, From, ReplyAs, Drv, Buf0) -> - case io_request(Req, Drv, {From,ReplyAs}, Buf0) of +io_request(Req, From, ReplyAs, Drv, Shell, Buf0) -> + case io_request(Req, Drv, Shell, {From,ReplyAs}, Buf0) of {ok,Reply,Buf} -> io_reply(From, ReplyAs, Reply), Buf; @@ -208,7 +208,7 @@ io_request(Req, From, ReplyAs, Drv, Buf0) -> %% %% These put requests have to be synchronous to the driver as otherwise %% there is no guarantee that the data has actually been printed. -io_request({put_chars,unicode,Chars}, Drv, From, Buf) -> +io_request({put_chars,unicode,Chars}, Drv, _Shell, From, Buf) -> case catch unicode:characters_to_binary(Chars,utf8) of Binary when is_binary(Binary) -> send_drv(Drv, {put_chars_sync, unicode, Binary, {From,ok}}), @@ -216,7 +216,7 @@ io_request({put_chars,unicode,Chars}, Drv, From, Buf) -> _ -> {error,{error,{put_chars, unicode,Chars}},Buf} end; -io_request({put_chars,unicode,M,F,As}, Drv, From, Buf) -> +io_request({put_chars,unicode,M,F,As}, Drv, _Shell, From, Buf) -> case catch apply(M, F, As) of Binary when is_binary(Binary) -> send_drv(Drv, {put_chars_sync, unicode, Binary, {From,ok}}), @@ -230,12 +230,12 @@ io_request({put_chars,unicode,M,F,As}, Drv, From, Buf) -> {error,{error,F},Buf} end end; -io_request({put_chars,latin1,Binary}, Drv, From, Buf) when is_binary(Binary) -> +io_request({put_chars,latin1,Binary}, Drv, _Shell, From, Buf) when is_binary(Binary) -> send_drv(Drv, {put_chars_sync, unicode, unicode:characters_to_binary(Binary,latin1), {From,ok}}), {noreply,Buf}; -io_request({put_chars,latin1,Chars}, Drv, From, Buf) -> +io_request({put_chars,latin1,Chars}, Drv, _Shell, From, Buf) -> case catch unicode:characters_to_binary(Chars,latin1) of Binary when is_binary(Binary) -> send_drv(Drv, {put_chars_sync, unicode, Binary, {From,ok}}), @@ -243,7 +243,7 @@ io_request({put_chars,latin1,Chars}, Drv, From, Buf) -> _ -> {error,{error,{put_chars,latin1,Chars}},Buf} end; -io_request({put_chars,latin1,M,F,As}, Drv, From, Buf) -> +io_request({put_chars,latin1,M,F,As}, Drv, _Shell, From, Buf) -> case catch apply(M, F, As) of Binary when is_binary(Binary) -> send_drv(Drv, {put_chars_sync, unicode, @@ -260,30 +260,30 @@ io_request({put_chars,latin1,M,F,As}, Drv, From, Buf) -> end end; -io_request({get_chars,Encoding,Prompt,N}, Drv, _From, Buf) -> - get_chars_n(Prompt, io_lib, collect_chars, N, Drv, Buf, Encoding); -io_request({get_line,Encoding,Prompt}, Drv, _From, Buf) -> - get_chars_line(Prompt, io_lib, collect_line, [], Drv, Buf, Encoding); -io_request({get_until,Encoding, Prompt,M,F,As}, Drv, _From, Buf) -> - get_chars_line(Prompt, io_lib, get_until, {M,F,As}, Drv, Buf, Encoding); -io_request({get_password,_Encoding},Drv,_From,Buf) -> - get_password_chars(Drv, Buf); -io_request({setopts,Opts}, Drv, _From, Buf) when is_list(Opts) -> +io_request({get_chars,Encoding,Prompt,N}, Drv, Shell, _From, Buf) -> + get_chars_n(Prompt, io_lib, collect_chars, N, Drv, Shell, Buf, Encoding); +io_request({get_line,Encoding,Prompt}, Drv, Shell, _From, Buf) -> + get_chars_line(Prompt, io_lib, collect_line, [], Drv, Shell, Buf, Encoding); +io_request({get_until,Encoding, Prompt,M,F,As}, Drv, Shell, _From, Buf) -> + get_chars_line(Prompt, io_lib, get_until, {M,F,As}, Drv, Shell, Buf, Encoding); +io_request({get_password,_Encoding},Drv,Shell,_From,Buf) -> + get_password_chars(Drv, Shell, Buf); +io_request({setopts,Opts}, Drv, _Shell, _From, Buf) when is_list(Opts) -> setopts(Opts, Drv, Buf); -io_request(getopts, Drv, _From, Buf) -> +io_request(getopts, Drv, _Shell, _From, Buf) -> getopts(Drv, Buf); -io_request({requests,Reqs}, Drv, From, Buf) -> - io_requests(Reqs, {ok,ok,Buf}, From, Drv); +io_request({requests,Reqs}, Drv, Shell, From, Buf) -> + io_requests(Reqs, {ok,ok,Buf}, From, Drv, Shell); %% New in R12 -io_request({get_geometry,columns},Drv,_From,Buf) -> +io_request({get_geometry,columns},Drv,_Shell,_From,Buf) -> case get_tty_geometry(Drv) of {W,_H} -> {ok,W,Buf}; _ -> {error,{error,enotsup},Buf} end; -io_request({get_geometry,rows},Drv,_From,Buf) -> +io_request({get_geometry,rows},Drv,_Shell,_From,Buf) -> case get_tty_geometry(Drv) of {_W,H} -> {ok,H,Buf}; @@ -292,40 +292,40 @@ io_request({get_geometry,rows},Drv,_From,Buf) -> end; %% BC with pre-R13 -io_request({put_chars,Chars}, Drv, From, Buf) -> - io_request({put_chars,latin1,Chars}, Drv, From, Buf); -io_request({put_chars,M,F,As}, Drv, From, Buf) -> - io_request({put_chars,latin1,M,F,As}, Drv, From, Buf); -io_request({get_chars,Prompt,N}, Drv, From, Buf) -> - io_request({get_chars,latin1,Prompt,N}, Drv, From, Buf); -io_request({get_line,Prompt}, Drv, From, Buf) -> - io_request({get_line,latin1,Prompt}, Drv, From, Buf); -io_request({get_until, Prompt,M,F,As}, Drv, From, Buf) -> - io_request({get_until,latin1, Prompt,M,F,As}, Drv, From, Buf); -io_request(get_password,Drv,From,Buf) -> - io_request({get_password,latin1},Drv,From,Buf); - - - -io_request(_, _Drv, _From, Buf) -> +io_request({put_chars,Chars}, Drv, Shell, From, Buf) -> + io_request({put_chars,latin1,Chars}, Drv, Shell, From, Buf); +io_request({put_chars,M,F,As}, Drv, Shell, From, Buf) -> + io_request({put_chars,latin1,M,F,As}, Drv, Shell, From, Buf); +io_request({get_chars,Prompt,N}, Drv, Shell, From, Buf) -> + io_request({get_chars,latin1,Prompt,N}, Drv, Shell, From, Buf); +io_request({get_line,Prompt}, Drv, Shell, From, Buf) -> + io_request({get_line,latin1,Prompt}, Drv, Shell, From, Buf); +io_request({get_until, Prompt,M,F,As}, Drv, Shell, From, Buf) -> + io_request({get_until,latin1, Prompt,M,F,As}, Drv, Shell, From, Buf); +io_request(get_password,Drv,Shell,From,Buf) -> + io_request({get_password,latin1},Drv,Shell,From,Buf); + + + +io_request(_, _Drv, _Shell, _From, Buf) -> {error,{error,request},Buf}. -%% Status = io_requests(RequestList, PrevStat, From, Drv) +%% Status = io_requests(RequestList, PrevStat, From, Drv, Shell) %% Process a list of output requests as long as %% the previous status is 'ok' or noreply. %% %% We use undefined as the From for all but the last request %% in order to discards acknowledgements from those requests. %% -io_requests([R|Rs], {noreply,Buf}, From, Drv) -> +io_requests([R|Rs], {noreply,Buf}, From, Drv, Shell) -> ReqFrom = if Rs =:= [] -> From; true -> undefined end, - io_requests(Rs, io_request(R, Drv, ReqFrom, Buf), From, Drv); -io_requests([R|Rs], {ok,ok,Buf}, From, Drv) -> + io_requests(Rs, io_request(R, Drv, Shell, ReqFrom, Buf), From, Drv, Shell); +io_requests([R|Rs], {ok,ok,Buf}, From, Drv, Shell) -> ReqFrom = if Rs =:= [] -> From; true -> undefined end, - io_requests(Rs, io_request(R, Drv, ReqFrom, Buf), From, Drv); -io_requests([_|_], Error, _From, _Drv) -> + io_requests(Rs, io_request(R, Drv, Shell, ReqFrom, Buf), From, Drv, Shell); +io_requests([_|_], Error, _From, _Drv, _Shell) -> Error; -io_requests([], Stat, _From, _) -> +io_requests([], Stat, _From, _, _Shell) -> Stat. %% io_reply(From, ReplyAs, Reply) @@ -333,7 +333,7 @@ io_requests([], Stat, _From, _) -> %% The ACK contains the return value. io_reply(undefined, _ReplyAs, _Reply) -> - %% Ignore these replies as they are generated from io_requests/4. + %% Ignore these replies as they are generated from io_requests/5. ok; io_reply(From, ReplyAs, Reply) -> From ! {io_reply,ReplyAs,Reply}, @@ -442,8 +442,8 @@ getopts(Drv,Buf) -> %% {Result,NewSaveBuffer} %% {error,What,NewSaveBuffer} -get_password_chars(Drv,Buf) -> - case get_password_line(Buf, Drv) of +get_password_chars(Drv,Shell,Buf) -> + case get_password_line(Buf, Drv, Shell) of {done, Line, Buf1} -> {ok, Line, Buf1}; interrupted -> @@ -452,59 +452,59 @@ get_password_chars(Drv,Buf) -> {exit, terminated} end. -get_chars_n(Prompt, M, F, Xa, Drv, Buf, Encoding) -> +get_chars_n(Prompt, M, F, Xa, Drv, Shell, Buf, Encoding) -> Pbs = prompt_bytes(Prompt, Encoding), case get(echo) of true -> - get_chars_loop(Pbs, M, F, Xa, Drv, Buf, start, Encoding); + get_chars_loop(Pbs, M, F, Xa, Drv, Shell, Buf, start, Encoding); false -> - get_chars_n_loop(Pbs, M, F, Xa, Drv, Buf, start, Encoding) + get_chars_n_loop(Pbs, M, F, Xa, Drv, Shell, Buf, start, Encoding) end. -get_chars_line(Prompt, M, F, Xa, Drv, Buf, Encoding) -> +get_chars_line(Prompt, M, F, Xa, Drv, Shell, Buf, Encoding) -> Pbs = prompt_bytes(Prompt, Encoding), - get_chars_loop(Pbs, M, F, Xa, Drv, Buf, start, Encoding). + get_chars_loop(Pbs, M, F, Xa, Drv, Shell, Buf, start, Encoding). -get_chars_loop(Pbs, M, F, Xa, Drv, Buf0, State, Encoding) -> +get_chars_loop(Pbs, M, F, Xa, Drv, Shell, Buf0, State, Encoding) -> Result = case get(echo) of true -> - get_line(Buf0, Pbs, Drv, Encoding); + get_line(Buf0, Pbs, Drv, Shell, Encoding); false -> % get_line_echo_off only deals with lists % and does not need encoding... - get_line_echo_off(Buf0, Pbs, Drv) + get_line_echo_off(Buf0, Pbs, Drv, Shell) end, case Result of {done,Line,Buf} -> - get_chars_apply(Pbs, M, F, Xa, Drv, Buf, State, Line, Encoding); + get_chars_apply(Pbs, M, F, Xa, Drv, Shell, Buf, State, Line, Encoding); interrupted -> {error,{error,interrupted},[]}; terminated -> {exit,terminated} end. -get_chars_apply(Pbs, M, F, Xa, Drv, Buf, State0, Line, Encoding) -> +get_chars_apply(Pbs, M, F, Xa, Drv, Shell, Buf, State0, Line, Encoding) -> case catch M:F(State0, cast(Line,get(read_mode), Encoding), Encoding, Xa) of {stop,Result,Rest} -> {ok,Result,append(Rest, Buf, Encoding)}; {'EXIT',_} -> {error,{error,err_func(M, F, Xa)},[]}; State1 -> - get_chars_loop(Pbs, M, F, Xa, Drv, Buf, State1, Encoding) + get_chars_loop(Pbs, M, F, Xa, Drv, Shell, Buf, State1, Encoding) end. -get_chars_n_loop(Pbs, M, F, Xa, Drv, Buf0, State, Encoding) -> +get_chars_n_loop(Pbs, M, F, Xa, Drv, Shell, Buf0, State, Encoding) -> try M:F(State, cast(Buf0, get(read_mode), Encoding), Encoding, Xa) of {stop,Result,Rest} -> {ok, Result, Rest}; State1 -> - case get_chars_echo_off(Pbs, Drv) of + case get_chars_echo_off(Pbs, Drv, Shell) of interrupted -> {error,{error,interrupted},[]}; terminated -> {exit,terminated}; Buf -> - get_chars_n_loop(Pbs, M, F, Xa, Drv, Buf, State1, Encoding) + get_chars_n_loop(Pbs, M, F, Xa, Drv, Shell, Buf, State1, Encoding) end catch _:_ -> {error,{error,err_func(M, F, Xa)},[]} @@ -523,24 +523,24 @@ err_func(_, F, _) -> %% {done,LineChars,RestChars} %% interrupted -get_line(Chars, Pbs, Drv, Encoding) -> +get_line(Chars, Pbs, Drv, Shell, Encoding) -> {more_chars,Cont,Rs} = edlin:start(Pbs), send_drv_reqs(Drv, Rs), - get_line1(edlin:edit_line(Chars, Cont), Drv, new_stack(get(line_buffer)), + get_line1(edlin:edit_line(Chars, Cont), Drv, Shell, new_stack(get(line_buffer)), Encoding). -get_line1({done,Line,Rest,Rs}, Drv, Ls, _Encoding) -> +get_line1({done,Line,Rest,Rs}, Drv, _Shell, Ls, _Encoding) -> send_drv_reqs(Drv, Rs), save_line_buffer(Line, get_lines(Ls)), {done,Line,Rest}; -get_line1({undefined,{_A,Mode,Char},Cs,Cont,Rs}, Drv, Ls0, Encoding) +get_line1({undefined,{_A,Mode,Char},Cs,Cont,Rs}, Drv, Shell, Ls0, Encoding) when ((Mode =:= none) and (Char =:= $\^P)) or ((Mode =:= meta_left_sq_bracket) and (Char =:= $A)) -> send_drv_reqs(Drv, Rs), case up_stack(save_line(Ls0, edlin:current_line(Cont))) of {none,_Ls} -> send_drv(Drv, beep), - get_line1(edlin:edit_line(Cs, Cont), Drv, Ls0, Encoding); + get_line1(edlin:edit_line(Cs, Cont), Drv, Shell, Ls0, Encoding); {Lcs,Ls} -> send_drv_reqs(Drv, edlin:erase_line(Cont)), {more_chars,Ncont,Nrs} = edlin:start(edlin:prompt(Cont)), @@ -548,16 +548,17 @@ get_line1({undefined,{_A,Mode,Char},Cs,Cont,Rs}, Drv, Ls0, Encoding) get_line1(edlin:edit_line1(lists:sublist(Lcs, 1, length(Lcs)-1), Ncont), Drv, + Shell, Ls, Encoding) end; -get_line1({undefined,{_A,Mode,Char},Cs,Cont,Rs}, Drv, Ls0, Encoding) +get_line1({undefined,{_A,Mode,Char},Cs,Cont,Rs}, Drv, Shell, Ls0, Encoding) when ((Mode =:= none) and (Char =:= $\^N)) or ((Mode =:= meta_left_sq_bracket) and (Char =:= $B)) -> send_drv_reqs(Drv, Rs), case down_stack(save_line(Ls0, edlin:current_line(Cont))) of {none,_Ls} -> send_drv(Drv, beep), - get_line1(edlin:edit_line(Cs, Cont), Drv, Ls0, Encoding); + get_line1(edlin:edit_line(Cs, Cont), Drv, Shell, Ls0, Encoding); {Lcs,Ls} -> send_drv_reqs(Drv, edlin:erase_line(Cont)), {more_chars,Ncont,Nrs} = edlin:start(edlin:prompt(Cont)), @@ -565,6 +566,7 @@ get_line1({undefined,{_A,Mode,Char},Cs,Cont,Rs}, Drv, Ls0, Encoding) get_line1(edlin:edit_line1(lists:sublist(Lcs, 1, length(Lcs)-1), Ncont), Drv, + Shell, Ls, Encoding) end; %% ^R = backward search, ^S = forward search. @@ -577,7 +579,7 @@ get_line1({undefined,{_A,Mode,Char},Cs,Cont,Rs}, Drv, Ls0, Encoding) %% new modes: search, search_quit, search_found. These are added to %% the regular ones (none, meta_left_sq_bracket) and handle special %% cases of history search. -get_line1({undefined,{_A,Mode,Char},Cs,Cont,Rs}, Drv, Ls, Encoding) +get_line1({undefined,{_A,Mode,Char},Cs,Cont,Rs}, Drv, Shell, Ls, Encoding) when ((Mode =:= none) and (Char =:= $\^R)) -> send_drv_reqs(Drv, Rs), %% drop current line, move to search mode. We store the current @@ -587,8 +589,8 @@ get_line1({undefined,{_A,Mode,Char},Cs,Cont,Rs}, Drv, Ls, Encoding) Pbs = prompt_bytes("(search)`': ", Encoding), {more_chars,Ncont,Nrs} = edlin:start(Pbs, search), send_drv_reqs(Drv, Nrs), - get_line1(edlin:edit_line1(Cs, Ncont), Drv, Ls, Encoding); -get_line1({expand, Before, Cs0, Cont,Rs}, Drv, Ls0, Encoding) -> + get_line1(edlin:edit_line1(Cs, Ncont), Drv, Shell, Ls, Encoding); +get_line1({expand, Before, Cs0, Cont,Rs}, Drv, Shell, Ls0, Encoding) -> send_drv_reqs(Drv, Rs), ExpandFun = get(expand_fun), {Found, Add, Matches} = ExpandFun(Before), @@ -603,37 +605,37 @@ get_line1({expand, Before, Cs0, Cont,Rs}, Drv, Ls0, Encoding) -> send_drv(Drv, {put_chars, unicode, unicode:characters_to_binary(MatchStr,unicode)}), [$\^L | Cs1] end, - get_line1(edlin:edit_line(Cs, Cont), Drv, Ls0, Encoding); -get_line1({undefined,_Char,Cs,Cont,Rs}, Drv, Ls, Encoding) -> + get_line1(edlin:edit_line(Cs, Cont), Drv, Shell, Ls0, Encoding); +get_line1({undefined,_Char,Cs,Cont,Rs}, Drv, Shell, Ls, Encoding) -> send_drv_reqs(Drv, Rs), send_drv(Drv, beep), - get_line1(edlin:edit_line(Cs, Cont), Drv, Ls, Encoding); + get_line1(edlin:edit_line(Cs, Cont), Drv, Shell, Ls, Encoding); %% The search item was found and accepted (new line entered on the exact %% result found) -get_line1({_What,Cont={line,_Prompt,_Chars,search_found},Rs}, Drv, Ls0, Encoding) -> +get_line1({_What,Cont={line,_Prompt,_Chars,search_found},Rs}, Drv, Shell, Ls0, Encoding) -> Line = edlin:current_line(Cont), %% this may create duplicate entries. Ls = save_line(new_stack(get_lines(Ls0)), Line), - get_line1({done, Line, "", Rs}, Drv, Ls, Encoding); + get_line1({done, Line, "", Rs}, Drv, Shell, Ls, Encoding); %% The search mode has been exited, but the user wants to remain in line %% editing mode wherever that was, but editing the search result. -get_line1({What,Cont={line,_Prompt,_Chars,search_quit},Rs}, Drv, Ls, Encoding) -> +get_line1({What,Cont={line,_Prompt,_Chars,search_quit},Rs}, Drv, Shell, Ls, Encoding) -> Line = edlin:current_chars(Cont), %% Load back the old prompt with the correct line number. case get(search_quit_prompt) of undefined -> % should not happen. Fallback. LsFallback = save_line(new_stack(get_lines(Ls)), Line), - get_line1({done, "\n", Line, Rs}, Drv, LsFallback, Encoding); + get_line1({done, "\n", Line, Rs}, Drv, Shell, LsFallback, Encoding); Prompt -> % redraw the line and keep going with the same stack position NCont = {line,Prompt,{lists:reverse(Line),[]},none}, send_drv_reqs(Drv, Rs), send_drv_reqs(Drv, edlin:erase_line(Cont)), send_drv_reqs(Drv, edlin:redraw_line(NCont)), - get_line1({What, NCont ,[]}, Drv, pad_stack(Ls), Encoding) + get_line1({What, NCont ,[]}, Drv, Shell, pad_stack(Ls), Encoding) end; %% Search mode is entered. get_line1({What,{line,Prompt,{RevCmd0,_Aft},search},Rs}, - Drv, Ls0, Encoding) -> + Drv, Shell, Ls0, Encoding) -> send_drv_reqs(Drv, Rs), %% Figure out search direction. ^S and ^R are returned through edlin %% whenever we received a search while being already in search mode. @@ -655,82 +657,88 @@ get_line1({What,{line,Prompt,{RevCmd0,_Aft},search},Rs}, {Ls2, {RevCmd, "': "++Line}} end, Cont = {line,Prompt,NewStack,search}, - more_data(What, Cont, Drv, Ls, Encoding); -get_line1({What,Cont0,Rs}, Drv, Ls, Encoding) -> + more_data(What, Cont, Drv, Shell, Ls, Encoding); +get_line1({What,Cont0,Rs}, Drv, Shell, Ls, Encoding) -> send_drv_reqs(Drv, Rs), - more_data(What, Cont0, Drv, Ls, Encoding). + more_data(What, Cont0, Drv, Shell, Ls, Encoding). -more_data(What, Cont0, Drv, Ls, Encoding) -> +more_data(What, Cont0, Drv, Shell, Ls, Encoding) -> receive {Drv,{data,Cs}} -> - get_line1(edlin:edit_line(Cs, Cont0), Drv, Ls, Encoding); + get_line1(edlin:edit_line(Cs, Cont0), Drv, Shell, Ls, Encoding); {Drv,eof} -> - get_line1(edlin:edit_line(eof, Cont0), Drv, Ls, Encoding); + get_line1(edlin:edit_line(eof, Cont0), Drv, Shell, Ls, Encoding); {io_request,From,ReplyAs,Req} when is_pid(From) -> {more_chars,Cont,_More} = edlin:edit_line([], Cont0), send_drv_reqs(Drv, edlin:erase_line(Cont)), - io_request(Req, From, ReplyAs, Drv, []), %WRONG!!! + io_request(Req, From, ReplyAs, Drv, Shell, []), %WRONG!!! send_drv_reqs(Drv, edlin:redraw_line(Cont)), - get_line1({more_chars,Cont,[]}, Drv, Ls, Encoding); + get_line1({more_chars,Cont,[]}, Drv, Shell, Ls, Encoding); {reply,{{From,ReplyAs},Reply}} -> %% We take care of replies from puts here as well io_reply(From, ReplyAs, Reply), - more_data(What, Cont0, Drv, Ls, Encoding); + more_data(What, Cont0, Drv, Shell, Ls, Encoding); {'EXIT',Drv,interrupt} -> interrupted; {'EXIT',Drv,_} -> - terminated + terminated; + {'EXIT',Shell,R} -> + exit(R) after get_line_timeout(What)-> - get_line1(edlin:edit_line([], Cont0), Drv, Ls, Encoding) + get_line1(edlin:edit_line([], Cont0), Drv, Shell, Ls, Encoding) end. -get_line_echo_off(Chars, Pbs, Drv) -> +get_line_echo_off(Chars, Pbs, Drv, Shell) -> send_drv_reqs(Drv, [{put_chars, unicode,Pbs}]), - get_line_echo_off1(edit_line(Chars,[]), Drv). + get_line_echo_off1(edit_line(Chars,[]), Drv, Shell). -get_line_echo_off1({Chars,[]}, Drv) -> +get_line_echo_off1({Chars,[]}, Drv, Shell) -> receive {Drv,{data,Cs}} -> - get_line_echo_off1(edit_line(Cs, Chars), Drv); + get_line_echo_off1(edit_line(Cs, Chars), Drv, Shell); {Drv,eof} -> - get_line_echo_off1(edit_line(eof, Chars), Drv); + get_line_echo_off1(edit_line(eof, Chars), Drv, Shell); {io_request,From,ReplyAs,Req} when is_pid(From) -> - io_request(Req, From, ReplyAs, Drv, []), - get_line_echo_off1({Chars,[]}, Drv); + io_request(Req, From, ReplyAs, Drv, Shell, []), + get_line_echo_off1({Chars,[]}, Drv, Shell); {reply,{{From,ReplyAs},Reply}} when From =/= undefined -> %% We take care of replies from puts here as well io_reply(From, ReplyAs, Reply), - get_line_echo_off1({Chars,[]},Drv); + get_line_echo_off1({Chars,[]},Drv, Shell); {'EXIT',Drv,interrupt} -> interrupted; {'EXIT',Drv,_} -> - terminated + terminated; + {'EXIT',Shell,R} -> + exit(R) end; -get_line_echo_off1({Chars,Rest}, _Drv) -> +get_line_echo_off1({Chars,Rest}, _Drv, _Shell) -> {done,lists:reverse(Chars),case Rest of done -> []; _ -> Rest end}. -get_chars_echo_off(Pbs, Drv) -> +get_chars_echo_off(Pbs, Drv, Shell) -> send_drv_reqs(Drv, [{put_chars, unicode,Pbs}]), - get_chars_echo_off1(Drv). + get_chars_echo_off1(Drv, Shell). -get_chars_echo_off1(Drv) -> +get_chars_echo_off1(Drv, Shell) -> receive {Drv, {data, Cs}} -> Cs; {Drv, eof} -> eof; {io_request,From,ReplyAs,Req} when is_pid(From) -> - io_request(Req, From, ReplyAs, Drv, []), - get_chars_echo_off1(Drv); + io_request(Req, From, ReplyAs, Drv, Shell, []), + get_chars_echo_off1(Drv, Shell); {reply,{{From,ReplyAs},Reply}} when From =/= undefined -> %% We take care of replies from puts here as well io_reply(From, ReplyAs, Reply), - get_chars_echo_off1(Drv); + get_chars_echo_off1(Drv, Shell); {'EXIT',Drv,interrupt} -> interrupted; {'EXIT',Drv,_} -> - terminated + terminated; + {'EXIT',Shell,R} -> + exit(R) end. %% We support line editing for the ICANON mode except the following @@ -861,30 +869,32 @@ search_down_stack(Stack, Substr) -> %% This is get_line without line editing (except for backspace) and %% without echo. -get_password_line(Chars, Drv) -> - get_password1(edit_password(Chars,[]),Drv). +get_password_line(Chars, Drv, Shell) -> + get_password1(edit_password(Chars,[]),Drv,Shell). -get_password1({Chars,[]}, Drv) -> +get_password1({Chars,[]}, Drv, Shell) -> receive {Drv,{data,Cs}} -> - get_password1(edit_password(Cs,Chars),Drv); + get_password1(edit_password(Cs,Chars),Drv,Shell); {io_request,From,ReplyAs,Req} when is_pid(From) -> %send_drv_reqs(Drv, [{delete_chars, -length(Pbs)}]), - io_request(Req, From, ReplyAs, Drv, []), %WRONG!!! + io_request(Req, From, ReplyAs, Drv, Shell, []), %WRONG!!! %% I guess the reason the above line is wrong is that Buf is %% set to []. But do we expect anything but plain output? - get_password1({Chars, []}, Drv); + get_password1({Chars, []}, Drv, Shell); {reply,{{From,ReplyAs},Reply}} -> %% We take care of replies from puts here as well io_reply(From, ReplyAs, Reply), - get_password1({Chars, []},Drv); + get_password1({Chars, []},Drv, Shell); {'EXIT',Drv,interrupt} -> interrupted; {'EXIT',Drv,_} -> - terminated + terminated; + {'EXIT',Shell,R} -> + exit(R) end; -get_password1({Chars,Rest},Drv) -> +get_password1({Chars,Rest},Drv,_Shell) -> send_drv_reqs(Drv,[{put_chars, unicode, "\n"}]), {done,lists:reverse(Chars),case Rest of done -> []; _ -> Rest end}. diff --git a/lib/kernel/src/inet.erl b/lib/kernel/src/inet.erl index 4bad523dff..73c53b9011 100644 --- a/lib/kernel/src/inet.erl +++ b/lib/kernel/src/inet.erl @@ -105,7 +105,20 @@ {local, binary()} | {unspec, <<>>} | {undefined, any()}. --type posix() :: exbadport | exbadseq | file:posix(). +-type posix() :: + 'eaddrinuse' | 'eaddrnotavail' | 'eafnosupport' | 'ealready' | + 'econnaborted' | 'econnrefused' | 'econnreset' | + 'edestaddrreq' | + 'ehostdown' | 'ehostunreach' | + 'einprogress' | 'eisconn' | + 'emsgsize' | + 'enetdown' | 'enetunreach' | + 'enopkg' | 'enoprotoopt' | 'enotconn' | 'enotty' | 'enotsock' | + 'eproto' | 'eprotonosupport' | 'eprototype' | + 'esocktnosupport' | + 'etimedout' | + 'ewouldblock' | + 'exbadport' | 'exbadseq' | file:posix(). -type socket() :: port(). -type socket_setopt() :: diff --git a/lib/kernel/src/kernel.app.src b/lib/kernel/src/kernel.app.src index 82a3571da9..afffcd156e 100644 --- a/lib/kernel/src/kernel.app.src +++ b/lib/kernel/src/kernel.app.src @@ -60,6 +60,17 @@ kernel_refc, local_tcp, local_udp, + logger, + logger_backend, + logger_config, + logger_disk_log_h, + logger_filters, + logger_formatter, + logger_h_common, + logger_server, + logger_simple, + logger_std_h, + logger_sup, net, net_adm, net_kernel, @@ -117,6 +128,8 @@ kernel_config, kernel_refc, kernel_sup, + logger, + logger_sup, net_kernel, net_sup, rex, @@ -127,7 +140,7 @@ inet_db, pg2]}, {applications, []}, - {env, [{error_logger, tty}]}, + {env, []}, {mod, {kernel, []}}, {runtime_dependencies, ["erts-10.0", "stdlib-3.5", "sasl-3.0"]} ] diff --git a/lib/kernel/src/kernel.appup.src b/lib/kernel/src/kernel.appup.src index 4ee497bbbd..305a1c788c 100644 --- a/lib/kernel/src/kernel.appup.src +++ b/lib/kernel/src/kernel.appup.src @@ -18,9 +18,11 @@ %% %CopyrightEnd% {"%VSN%", %% Up from - max one major revision back - [{<<"5\\.[0-3](\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-19.*, OTP-20.0 - {<<"5\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-20.1+ + [{<<"5\\.3(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-20.0 + {<<"5\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-20.1+ + {<<"6\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-21 %% Down to - max one major revision back - [{<<"5\\.[0-3](\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-19.*, OTP-20.0 - {<<"5\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-20.1+ + [{<<"5\\.3(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-20.0 + {<<"5\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-20.1+ + {<<"6\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-21 }. diff --git a/lib/kernel/src/kernel.erl b/lib/kernel/src/kernel.erl index 0382764b39..20aa47f602 100644 --- a/lib/kernel/src/kernel.erl +++ b/lib/kernel/src/kernel.erl @@ -40,8 +40,7 @@ start(_, []) -> ok = gen_event:add_handler(erl_signal_server, erl_signal_handler, []) end, %% add error handler - Type = get_error_logger_type(), - case error_logger:swap_handler(Type) of + case logger:setup_standard_handler() of ok -> {ok, Pid, []}; Error -> %% Not necessary since the node will crash anyway: @@ -62,16 +61,6 @@ config_change(Changed, New, Removed) -> do_global_groups_change(Changed, New, Removed), ok. -get_error_logger_type() -> - case application:get_env(kernel, error_logger) of - {ok, tty} -> tty; - {ok, {file, File}} when is_list(File) -> {logfile, File}; - {ok, false} -> false; - {ok, silent} -> silent; - undefined -> tty; % default value - {ok, Bad} -> exit({bad_config, {kernel, {error_logger, Bad}}}) - end. - %%%----------------------------------------------------------------- %%% The process structure in kernel is as shown in the figure. %%% @@ -153,9 +142,18 @@ init([]) -> type => supervisor, modules => [?MODULE]}, + + LoggerSup = #{id => logger_sup, + start => {logger_sup, start_link, []}, + restart => permanent, + shutdown => infinity, + type => supervisor, + modules => [logger_sup]}, + case init:get_argument(mode) of {ok, [["minimal"]]} -> - {ok, {SupFlags, [Code, File, StdError, User, Config, RefC, SafeSup]}}; + {ok, {SupFlags, + [Code, File, StdError, User, Config, RefC, SafeSup, LoggerSup]}}; _ -> Rpc = #{id => rex, start => {rpc, start_link, []}, @@ -206,7 +204,7 @@ init([]) -> {ok, {SupFlags, [Code, Rpc, Global, InetDb | DistAC] ++ [NetSup, GlGroup, File, SigSrv, - StdError, User, Config, RefC, SafeSup] ++ Timer}} + StdError, User, Config, RefC, SafeSup, LoggerSup] ++ Timer}} end; init(safe) -> SupFlags = #{strategy => one_for_one, diff --git a/lib/kernel/src/logger.erl b/lib/kernel/src/logger.erl new file mode 100644 index 0000000000..943ef8c2d1 --- /dev/null +++ b/lib/kernel/src/logger.erl @@ -0,0 +1,803 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(logger). + +%% Log interface +-export([emergency/1,emergency/2,emergency/3, + alert/1,alert/2,alert/3, + critical/1,critical/2,critical/3, + error/1,error/2,error/3, + warning/1,warning/2,warning/3, + notice/1,notice/2,notice/3, + info/1,info/2,info/3, + debug/1,debug/2,debug/3]). +-export([log/2,log/3,log/4]). + +%% Called by macro +-export([allow/2,macro_log/3,macro_log/4,macro_log/5,add_default_metadata/1]). + +%% Configuration +-export([add_handler/3, remove_handler/1, + add_logger_filter/2, add_handler_filter/3, + remove_logger_filter/1, remove_handler_filter/2, + set_module_level/2, reset_module_level/1, + set_logger_config/1, set_logger_config/2, + set_handler_config/2, set_handler_config/3, + get_logger_config/0, get_handler_config/1]). + +%% Misc +-export([compare_levels/2]). +-export([set_process_metadata/1, unset_process_metadata/0, + get_process_metadata/0]). +-export([i/0, i/1]). +-export([setup_standard_handler/0, replace_simple_handler/3]). +-export([limit_term/1, get_format_depth/0, get_max_size/0, get_utc_config/0]). + +%% Basic report formatting +-export([format_report/1, format_otp_report/1]). + +-export([internal_log/2,filter_stacktrace/2]). + +-include("logger_internal.hrl"). +-include("logger.hrl"). + +%%%----------------------------------------------------------------- +%%% Types +-type log() :: #{level=>level(), + msg=>{io:format(),[term()]} | + {report,report()} | + {string,unicode:chardata()}, + meta=>metadata()}. +-type level() :: emergency | alert | critical | error | + warning | notice | info | debug. +-type report() :: map() | [{atom(),term()}]. +-type msg_fun() :: fun((term()) -> {io:format(),[term()]} | + report() | + unicode:chardata()). +-type metadata() :: map(). + +-type handler_id() :: atom(). +-type filter_id() :: atom(). +-type filter() :: {fun((log(),term()) -> filter_return()),term()}. +-type filter_return() :: stop | ignore | log(). +-type config() :: map(). + +-export_type([log/0,level/0,report/0,msg_fun/0,metadata/0,config/0,handler_id/0, + filter_id/0,filter/0,filter_return/0]). + +%%%----------------------------------------------------------------- +%%% API +emergency(X) -> + log(emergency,X). +emergency(X,Y) -> + log(emergency,X,Y). +emergency(X,Y,Z) -> + log(emergency,X,Y,Z). + +alert(X) -> + log(alert,X). +alert(X,Y) -> + log(alert,X,Y). +alert(X,Y,Z) -> + log(alert,X,Y,Z). + +critical(X) -> + log(critical,X). +critical(X,Y) -> + log(critical,X,Y). +critical(X,Y,Z) -> + log(critical,X,Y,Z). + +error(X) -> + log(error,X). +error(X,Y) -> + log(error,X,Y). +error(X,Y,Z) -> + log(error,X,Y,Z). + +warning(X) -> + log(warning,X). +warning(X,Y) -> + log(warning,X,Y). +warning(X,Y,Z) -> + log(warning,X,Y,Z). + +notice(X) -> + log(notice,X). +notice(X,Y) -> + log(notice,X,Y). +notice(X,Y,Z) -> + log(notice,X,Y,Z). + +info(X) -> + log(info,X). +info(X,Y) -> + log(info,X,Y). +info(X,Y,Z) -> + log(info,X,Y,Z). + +debug(X) -> + log(debug,X). +debug(X,Y) -> + log(debug,X,Y). +debug(X,Y,Z) -> + log(debug,X,Y,Z). + +-spec log(Level,StringOrReport) -> ok when + Level :: level(), + StringOrReport :: unicode:chardata() | report(). +log(Level, StringOrReport) -> + do_log(Level,StringOrReport,#{}). + +-spec log(Level,StringOrReport,Metadata) -> ok when + Level :: level(), + StringOrReport :: unicode:chardata() | report(), + Metadata :: metadata(); + (Level,Format,Args) -> ok when + Level :: level(), + Format :: io:format(), + Args ::[term()]; + (Level,Fun,FunArgs) -> ok when + Level :: level(), + Fun :: msg_fun(), + FunArgs :: term(). +log(Level, StringOrReport, Metadata) + when is_map(Metadata), not is_function(StringOrReport) -> + do_log(Level,StringOrReport,Metadata); +log(Level, FunOrFormat, Args) -> + do_log(Level,{FunOrFormat,Args},#{}). + +-spec log(Level,Format, Args, Metadata) -> ok when + Level :: level(), + Format :: io:format(), + Args :: [term()], + Metadata :: metadata(); + (Level,Fun,FunArgs,Metadata) -> ok when + Level :: level(), + Fun :: msg_fun(), + FunArgs :: term(), + Metadata :: metadata(). +log(Level, FunOrFormat, Args, Metadata) -> + do_log(Level,{FunOrFormat,Args},Metadata). + +-spec allow(Level,Module) -> boolean() when + Level :: level(), + Module :: module(). +allow(Level,Module) when ?IS_LEVEL(Level), is_atom(Module) -> + logger_config:allow(?LOGGER_TABLE,Level,Module). + + +-spec macro_log(Location,Level,StringOrReport) -> ok when + Location :: map(), + Level :: level(), + StringOrReport :: unicode:chardata() | report(). +macro_log(Location,Level,StringOrReport) -> + log_allowed(Location,Level,StringOrReport,#{}). + +-spec macro_log(Location,Level,StringOrReport,Meta) -> ok when + Location :: map(), + Level :: level(), + StringOrReport :: unicode:chardata() | report(), + Meta :: metadata(); + (Location,Level,Format,Args) -> ok when + Location :: map(), + Level :: level(), + Format :: io:format(), + Args ::[term()]; + (Location,Level,Fun,FunArgs) -> ok when + Location :: map(), + Level :: level(), + Fun :: msg_fun(), + FunArgs :: term(). +macro_log(Location,Level,StringOrReport,Meta) + when is_map(Meta), not is_function(StringOrReport) -> + log_allowed(Location,Level,StringOrReport,Meta); +macro_log(Location,Level,FunOrFormat,Args) -> + log_allowed(Location,Level,{FunOrFormat,Args},#{}). + +-spec macro_log(Location,Level,Format,Args,Meta) -> ok when + Location :: map(), + Level :: level(), + Format :: io:format(), + Args ::[term()], + Meta :: metadata(); + (Location,Level,Fun,FunArgs,Meta) -> ok when + Location :: map(), + Level :: level(), + Fun :: msg_fun(), + FunArgs :: term(), + Meta :: metadata(). +macro_log(Location,Level,FunOrFormat,Args,Meta) -> + log_allowed(Location,Level,{FunOrFormat,Args},Meta). + +-spec format_otp_report(Report) -> FormatArgs when + Report :: report(), + FormatArgs :: {io:format(),[term()]}. +format_otp_report(#{label:=_,report:=Report}) -> + format_report(Report); +format_otp_report(Report) -> + format_report(Report). + +-spec format_report(Report) -> FormatArgs when + Report :: report(), + FormatArgs :: {io:format(),[term()]}. +format_report(Report) when is_map(Report) -> + format_report(maps:to_list(Report)); +format_report(Report) when is_list(Report) -> + case lists:flatten(Report) of + [] -> + {"~tp",[[]]}; + FlatList -> + case string_p1(FlatList) of + true -> + {"~ts",[FlatList]}; + false -> + format_term_list(Report,[],[]) + end + end; +format_report(Report) -> + {"~tp",[Report]}. + +format_term_list([{Tag,Data}|T],Format,Args) -> + PorS = case string_p(Data) of + true -> "s"; + false -> "p" + end, + format_term_list(T,[" ~tp: ~t"++PorS|Format],[Data,Tag|Args]); +format_term_list([Data|T],Format,Args) -> + format_term_list(T,[" ~tp"|Format],[Data|Args]); +format_term_list([],Format,Args) -> + {lists:flatten(lists:join($\n,lists:reverse(Format))),lists:reverse(Args)}. + +string_p(List) when is_list(List) -> + string_p1(lists:flatten(List)); +string_p(_) -> + false. + +string_p1([]) -> + false; +string_p1(FlatList) -> + io_lib:printable_unicode_list(FlatList). + +internal_log(Level,Term) when is_atom(Level) -> + erlang:display_string("Logger - "++ atom_to_list(Level) ++ ": "), + erlang:display(Term). + +%%%----------------------------------------------------------------- +%%% Configuration +-spec add_logger_filter(FilterId,Filter) -> ok | {error,term()} when + FilterId :: filter_id(), + Filter :: filter(). +add_logger_filter(FilterId,Filter) -> + logger_server:add_filter(logger,{FilterId,Filter}). + +-spec add_handler_filter(HandlerId,FilterId,Filter) -> ok | {error,term()} when + HandlerId :: handler_id(), + FilterId :: filter_id(), + Filter :: filter(). +add_handler_filter(HandlerId,FilterId,Filter) -> + logger_server:add_filter(HandlerId,{FilterId,Filter}). + + +-spec remove_logger_filter(FilterId) -> ok | {error,term()} when + FilterId :: filter_id(). +remove_logger_filter(FilterId) -> + logger_server:remove_filter(logger,FilterId). + +-spec remove_handler_filter(HandlerId,FilterId) -> ok | {error,term()} when + HandlerId :: handler_id(), + FilterId :: filter_id(). +remove_handler_filter(HandlerId,FilterId) -> + logger_server:remove_filter(HandlerId,FilterId). + +-spec add_handler(HandlerId,Module,Config) -> ok | {error,term()} when + HandlerId :: handler_id(), + Module :: module(), + Config :: config(). +add_handler(HandlerId,Module,Config) -> + logger_server:add_handler(HandlerId,Module,Config). + +-spec remove_handler(HandlerId) -> ok | {error,term()} when + HandlerId :: handler_id(). +remove_handler(HandlerId) -> + logger_server:remove_handler(HandlerId). + +-spec set_logger_config(Key,Value) -> ok | {error,term()} when + Key :: atom(), + Value :: term(). +set_logger_config(Key,Value) -> + logger_server:set_config(logger,Key,Value). + +-spec set_logger_config(Config) -> ok | {error,term()} when + Config :: config(). +set_logger_config(Config) -> + logger_server:set_config(logger,Config). + +-spec set_handler_config(HandlerId,Key,Value) -> ok | {error,term()} when + HandlerId :: handler_id(), + Key :: atom(), + Value :: term(). +set_handler_config(HandlerId,Key,Value) -> + logger_server:set_config(HandlerId,Key,Value). + +-spec set_handler_config(HandlerId,Config) -> ok | {error,term()} when + HandlerId :: handler_id(), + Config :: config(). +set_handler_config(HandlerId,Config) -> + logger_server:set_config(HandlerId,Config). + +-spec get_logger_config() -> {ok,Config} when + Config :: config(). +get_logger_config() -> + logger_config:get(?LOGGER_TABLE,logger). + +-spec get_handler_config(HandlerId) -> {ok,{Module,Config}} | {error,term()} when + HandlerId :: handler_id(), + Module :: module(), + Config :: config(). +get_handler_config(HandlerId) -> + logger_config:get(?LOGGER_TABLE,HandlerId). + +-spec set_module_level(Module,Level) -> ok | {error,term()} when + Module :: module(), + Level :: level(). +set_module_level(Module,Level) -> + logger_server:set_module_level(Module,Level). + +-spec reset_module_level(Module) -> ok | {error,term()} when + Module :: module(). +reset_module_level(Module) -> + logger_server:reset_module_level(Module). + +%%%----------------------------------------------------------------- +%%% Misc +-spec compare_levels(Level1,Level2) -> eq | gt | lt when + Level1 :: level(), + Level2 :: level(). +compare_levels(Level,Level) when ?IS_LEVEL(Level) -> + eq; +compare_levels(Level1,Level2) when ?IS_LEVEL(Level1), ?IS_LEVEL(Level2) -> + Int1 = logger_config:level_to_int(Level1), + Int2 = logger_config:level_to_int(Level2), + if Int1 < Int2 -> gt; + true -> lt + end; +compare_levels(Level1,Level2) -> + erlang:error(badarg,[Level1,Level2]). + +-spec set_process_metadata(Meta) -> ok when + Meta :: metadata(). +set_process_metadata(Meta) when is_map(Meta) -> + _ = put(?LOGGER_META_KEY,Meta), + ok; +set_process_metadata(Meta) -> + erlang:error(badarg,[Meta]). + +-spec get_process_metadata() -> Meta | undefined when + Meta :: metadata(). +get_process_metadata() -> + get(?LOGGER_META_KEY). + +-spec unset_process_metadata() -> ok. +unset_process_metadata() -> + _ = erase(?LOGGER_META_KEY), + ok. + +-spec i() -> #{logger=>config(), + handlers=>[{handler_id(),module(),config()}], + module_levels=>[{module(),level()}]}. +i() -> + i(term). + +-spec i(term) -> #{logger=>config(), + handlers=>[{handler_id(),module(),config()}], + module_levels=>[{module(),level()}]}; + (print) -> ok; + (string) -> iolist(). +i(_Action = print) -> + io:put_chars(i(string)); +i(_Action = string) -> + #{logger := #{level := Level, handlers := Handlers, + filters := Filters, filter_default := FilterDefault}, + handlers := HandlerConfigs, + module_levels := Modules} = i(term), + [io_lib:format("Current logger configuration:~n", []), + io_lib:format(" Level: ~p~n",[Level]), + io_lib:format(" Filter Default: ~p~n", [FilterDefault]), + io_lib:format(" Filters: ~n", []), + print_filters(4, Filters), + io_lib:format(" Handlers: ~n", []), + print_handlers([C || {Id, _, _} = C <- HandlerConfigs, + lists:member(Id, Handlers)]), + io_lib:format(" Level set per module: ~n", []), + print_module_levels(Modules) + ]; +i(_Action = term) -> + {Logger, Handlers, Modules} = logger_config:get(tid()), + #{logger=>Logger, + handlers=>Handlers, + module_levels=>Modules}. + +print_filters(Indent, {Id, {Fun, Config}}) -> + io_lib:format("~sId: ~p~n" + "~s Fun: ~p~n" + "~s Config: ~p~n",[Indent, Id, Indent, Fun, Indent, Config]); +print_filters(Indent, Filters) -> + IndentStr = io_lib:format("~.*s",[Indent, ""]), + lists:map(fun(Filter) ->print_filters(IndentStr, Filter) end, Filters). + + +print_handlers({Id,Module, + #{level := Level, + filters := Filters, filter_default := FilterDefault, + formatter := {FormatterModule,FormatterConfig}} = Config}) -> + MyKeys = [filter_default, filters, formatter, level, id], + UnhandledConfig = maps:filter(fun(Key, _) -> + not lists:member(Key, MyKeys) + end, Config), + Unhandled = lists:map(fun({Key, Value}) -> + io_lib:format(" ~p: ~p~n",[Key, Value]) + end, maps:to_list(UnhandledConfig)), + io_lib:format(" Id: ~p~n" + " Module: ~p~n" + " Level: ~p~n" + " Formatter:~n" + " Module: ~p~n" + " Config: ~p~n" + " Filter Default: ~p~n" + " Filters:~n~s" + " Handler Config:~n" + "~s" + "",[Id, Module, Level, FormatterModule, FormatterConfig, + FilterDefault, print_filters(8, Filters), Unhandled]); +print_handlers(Handlers) -> + lists:map(fun print_handlers/1, Handlers). + +print_module_levels({Module,Level}) -> + io_lib:format(" Module: ~p~n" + " Level: ~p~n", + [Module,Level]); +print_module_levels(ModuleLevels) -> + lists:map(fun print_module_levels/1, ModuleLevels). + +-spec setup_standard_handler() -> ok | {error,term()}. +setup_standard_handler() -> + case get_logger_type() of + {ok,silent} -> + Level = get_logger_level(), + ok = set_logger_config(level,Level), + remove_handler(logger_simple); + {ok,Type} -> + Level = get_logger_level(), + ok = set_logger_config(level,Level), + Filters = get_logger_filters(), + setup_standard_handler(Type,#{level=>Level, + filter_default=>stop, + filters=>Filters}); + Error -> + Error + end. + +-spec setup_standard_handler(Type,Config) -> ok | {error,term()} when + Type :: tty | standard_io | standard_error | {file,File} | + {file,File,Modes} | {disk_log,LogOpts} | false, + File :: file:filename(), + Modes :: [term()], % [file:mode()], or more specific? + Config :: config(), + LogOpts :: map(). +setup_standard_handler(false,#{level:=Level,filters:=Filters}) -> + case set_handler_config(logger_simple,level,Level) of + ok -> + set_handler_config(logger_simple,filters,Filters); + Error -> + Error + end; +setup_standard_handler(Type,Config) -> + {Module,TypeConfig} = get_type_config(Type), + replace_simple_handler(?STANDARD_HANDLER, + Module, + maps:merge(Config,TypeConfig)). + +-spec replace_simple_handler(Id,Module,Config) -> ok | {error,term()} when + Id :: handler_id(), + Module :: module(), + Config :: config(). +replace_simple_handler(Id,Module,Config) -> + _ = code:ensure_loaded(Module), + DoBuffer = erlang:function_exported(Module,swap_buffer,2), + case add_handler(Id,Module,Config#{wait_for_buffer=>DoBuffer}) of + ok -> + if DoBuffer -> + {ok,Buffered} = logger_simple:get_buffer(), + _ = remove_handler(logger_simple), + Module:swap_buffer(?STANDARD_HANDLER,Buffered); + true -> + _ = remove_handler(logger_simple), + ok + end, + ok; + Error -> + Error + end. + +get_logger_type() -> + Type0 = + case application:get_env(kernel, logger_dest) of + undefined -> + application:get_env(kernel, error_logger); + T -> + T + end, + case Type0 of + {ok, tty} -> + {ok, tty}; + {ok, {file, File}} when is_list(File) -> + {ok, {file, File}}; + {ok, {file, File, Modes}} when is_list(File), is_list(Modes) -> + {ok, {file, File, Modes}}; + {ok, {disk_log, File}} when is_list(File) -> + {ok, {disk_log, get_disk_log_config(File)}}; + {ok, false} -> + {ok, false}; + {ok, silent} -> + {ok, silent}; + undefined -> + {ok, tty}; % default value + {ok, Bad} -> + {error,{bad_config, {kernel, {logger_dest, Bad}}}} + end. + +get_disk_log_config(File) -> + Config1 = + case application:get_env(kernel,logger_disk_log_maxfiles) of + undefined -> #{}; + {ok,MF} -> #{max_no_files=>MF} + end, + Config2 = + case application:get_env(kernel,logger_disk_log_maxbytes) of + undefined -> Config1; + {ok,MB} -> Config1#{max_no_bytes=>MB} + end, + Config3 = + case application:get_env(kernel,logger_disk_log_type) of + undefined -> Config2; + {ok,T} -> Config1#{type=>T} + end, + Config3#{file=>File}. + +get_logger_level() -> + case application:get_env(kernel,logger_level) of + undefined -> info; + {ok,Level} when ?IS_LEVEL(Level) -> Level + end. + +get_logger_filters() -> + case application:get_env(kernel, logger_sasl_compatible, false) of + true -> + ?DEFAULT_HANDLER_FILTERS([beam,erlang,otp]); + false -> + Extra = + case application:get_env(kernel, logger_log_progress, false) of + true -> + []; + false -> + [{stop_progress, + {fun logger_filters:progress/2,stop}}] + end, + Extra ++ ?DEFAULT_HANDLER_FILTERS([beam,erlang,otp,sasl]) + end. + +get_type_config({disk_log,LogOpts}) -> + {logger_disk_log_h,#{disk_log_opts=>LogOpts}}; +get_type_config(tty) -> + %% This is only for backwards compatibility with error_logger and + %% old kernel and sasl environment variables + get_type_config(standard_io); +get_type_config(Type) when Type==standard_io; + Type==standard_error; + element(1,Type)==file -> + {logger_std_h,#{logger_std_h=>#{type=>Type}}}; +get_type_config(Type) -> + {error,{illegal_logger_type,Type}}. + +%%%----------------------------------------------------------------- +-spec limit_term(term()) -> term(). + +limit_term(Term) -> + try get_format_depth() of + unlimited -> Term; + D -> io_lib:limit_term(Term, D) + catch error:badarg -> + %% This could happen during system termination, after + %% application_controller process is dead. + unlimited + end. + +-spec get_format_depth() -> 'unlimited' | pos_integer(). + +get_format_depth() -> + Depth = + case application:get_env(kernel, logger_format_depth) of + {ok, D} when is_integer(D) -> + D; + undefined -> + case application:get_env(kernel, error_logger_format_depth) of + {ok, D} when is_integer(D) -> + D; + undefined -> + unlimited + end + end, + max(10, Depth). + +-spec get_max_size() -> 'unlimited' | pos_integer(). + +get_max_size() -> + case application:get_env(kernel, logger_max_size) of + {ok, Size} when is_integer(Size) -> + max(50, Size); + undefined -> + unlimited + end. + +-spec get_utc_config() -> boolean(). + +get_utc_config() -> + %% Kernel's logger_utc configuration overrides SASL utc_log, which + %% in turn overrides stdlib config - in order to have uniform + %% timestamps in log messages + case application:get_env(kernel, logger_utc) of + {ok, Val} -> Val; + undefined -> + case application:get_env(sasl, utc_log) of + {ok, Val} -> Val; + undefined -> + case application:get_env(stdlib, utc_log) of + {ok, Val} -> Val; + undefined -> false + end + end + end. + +%%%----------------------------------------------------------------- +%%% Internal +do_log(warning,Msg,Meta) -> + do_log_1(error_logger:warning_map(),Msg,Meta); +do_log(Level,Msg,Meta) -> + do_log_1(Level,Msg,Meta). + +do_log_1(Level,Msg,#{mfa:={Module,_,_}}=Meta) -> + case logger_config:allow(?LOGGER_TABLE,Level,Module) of + true -> + log_allowed(#{},Level,Msg,Meta); + false -> + ok + end; +do_log_1(Level,Msg,Meta) -> + case logger_config:allow(?LOGGER_TABLE,Level) of + true -> + log_allowed(#{},Level,Msg,Meta); + false -> + ok + end. + +-spec log_allowed(Location,Level,Msg,Meta) -> ok when + Location :: map(), + Level :: level(), + Msg :: {msg_fun(),term()} | + {io:format(),[term()]} | + report() | + unicode:chardata(), + Meta :: metadata(). +log_allowed(Location,Level,{Fun,FunArgs},Meta) when is_function(Fun,1) -> + try Fun(FunArgs) of + Msg={Format,Args} when is_list(Format), is_list(Args) -> + log_allowed(Location,Level,Msg,Meta); + Report when ?IS_REPORT(Report) -> + log_allowed(Location,Level,Report,Meta); + String when ?IS_STRING(String) -> + log_allowed(Location,Level,String,Meta); + Other -> + log_allowed(Location,Level, + {"LAZY_FUN ERROR: ~tp; Returned: ~tp", + [{Fun,FunArgs},Other]}, + Meta) + catch C:R -> + log_allowed(Location,Level, + {"LAZY_FUN CRASH: ~tp; Reason: ~tp", + [{Fun,FunArgs},{C,R}]}, + Meta) + end; +log_allowed(Location,Level,Msg,Meta0) when is_map(Meta0) -> + %% Metadata priorities are: + %% Location (added in API macros) - will be overwritten by process + %% metadata (set by set_process_metadata/1), which in turn will be + %% overwritten by the metadata given as argument in the log call + %% (function or macro). + Meta = add_default_metadata( + maps:merge(Location,maps:merge(proc_meta(),Meta0))), + case node(maps:get(gl,Meta)) of + Node when Node=/=node() -> + log_remote(Node,Level,Msg,Meta), + do_log_allowed(Level,Msg,Meta); + _ -> + do_log_allowed(Level,Msg,Meta) + end. + +do_log_allowed(Level,{Format,Args}=Msg,Meta) + when ?IS_LEVEL(Level), + is_list(Format), + is_list(Args), + is_map(Meta) -> + logger_backend:log_allowed(#{level=>Level,msg=>Msg,meta=>Meta},tid()); +do_log_allowed(Level,Report,Meta) + when ?IS_LEVEL(Level), + ?IS_REPORT(Report), + is_map(Meta) -> + logger_backend:log_allowed(#{level=>Level,msg=>{report,Report},meta=>Meta}, + tid()); +do_log_allowed(Level,String,Meta) + when ?IS_LEVEL(Level), + ?IS_STRING(String), + is_map(Meta) -> + logger_backend:log_allowed(#{level=>Level,msg=>{string,String},meta=>Meta}, + tid()). +tid() -> + ets:whereis(?LOGGER_TABLE). + +log_remote(Node,Level,{Format,Args},Meta) -> + log_remote(Node,{log,Level,Format,Args,Meta}); +log_remote(Node,Level,Msg,Meta) -> + log_remote(Node,{log,Level,Msg,Meta}). + +log_remote(Node,Request) -> + {logger,Node} ! Request, + ok. + +add_default_metadata(Meta) -> + add_default_metadata([pid,gl,time],Meta). + +add_default_metadata([Key|Keys],Meta) -> + case maps:is_key(Key,Meta) of + true -> + add_default_metadata(Keys,Meta); + false -> + add_default_metadata(Keys,Meta#{Key=>default(Key)}) + end; +add_default_metadata([],Meta) -> + Meta. + +proc_meta() -> + case get_process_metadata() of + ProcMeta when is_map(ProcMeta) -> ProcMeta; + _ -> #{} + end. + +default(pid) -> self(); +default(gl) -> group_leader(); +default(time) -> erlang:monotonic_time(microsecond). + +%% Remove everything upto and including this module from the stacktrace +filter_stacktrace(Module,[{Module,_,_,_}|_]) -> + []; +filter_stacktrace(Module,[H|T]) -> + [H|filter_stacktrace(Module,T)]; +filter_stacktrace(_,[]) -> + []. diff --git a/lib/kernel/src/logger_backend.erl b/lib/kernel/src/logger_backend.erl new file mode 100644 index 0000000000..d9f5aa6faf --- /dev/null +++ b/lib/kernel/src/logger_backend.erl @@ -0,0 +1,133 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(logger_backend). + +-export([log_allowed/2]). + +-include("logger_internal.hrl"). + +-define(OWN_KEYS,[level,filters,filter_default,handlers]). + +%%%----------------------------------------------------------------- +%%% The default logger backend +log_allowed(Log, Tid) -> + {ok,Config} = logger_config:get(Tid,logger), + Filters = maps:get(filters,Config,[]), + case apply_filters(logger,Log,Filters,Config) of + stop -> + ok; + Log1 -> + Handlers = maps:get(handlers,Config,[]), + call_handlers(Log1,Handlers,Tid) + end, + ok. + +call_handlers(#{level:=Level}=Log,[Id|Handlers],Tid) -> + case logger_config:get(Tid,Id,Level) of + {ok,{Module,Config}} -> + Filters = maps:get(filters,Config,[]), + case apply_filters(Id,Log,Filters,Config) of + stop -> + ok; + Log1 -> + Config1 = maps:without(?OWN_KEYS,Config), + try Module:log(Log1,Config1) + catch C:R:S -> + case logger:remove_handler(Id) of + ok -> + logger:internal_log( + error,{removed_failing_handler,Id}), + ?LOG_INTERNAL( + debug, + [{logger,removed_failing_handler}, + {handler,{Id,Module}}, + {log,Log1}, + {config,Config1}, + {reason,{C,R,filter_stacktrace(S)}}]); + {error,{not_found,_}} -> + %% Probably already removed by other client + %% Don't report again + ok; + {error,Reason} -> + ?LOG_INTERNAL( + debug, + [{logger,remove_handler_failed}, + {reason,Reason}]) + end + end + end; + _ -> + ok + end, + call_handlers(Log,Handlers,Tid); +call_handlers(_Log,[],_Tid) -> + ok. + +apply_filters(Owner,Log,Filters,Config) -> + case do_apply_filters(Owner,Log,Filters,ignore) of + stop -> + stop; + ignore -> + case maps:get(filter_default,Config) of + log -> + Log; + stop -> + stop + end; + Log1 -> + Log1 + end. + +do_apply_filters(Owner,Log,[{_Id,{FilterFun,FilterArgs}}=Filter|Filters],State) -> + try FilterFun(Log,FilterArgs) of + stop -> + stop; + ignore -> + do_apply_filters(Owner,Log,Filters,State); + Log1=#{level:=Level,msg:=Msg,meta:=Meta} + when is_atom(Level), ?IS_MSG(Msg), is_map(Meta) -> + do_apply_filters(Owner,Log1,Filters,log); + Bad -> + handle_filter_failed(Filter,Owner,Log,{bad_return_value,Bad}) + catch C:R:S -> + handle_filter_failed(Filter,Owner,Log,{C,R,filter_stacktrace(S)}) + end; +do_apply_filters(_Owner,_Log,[],ignore) -> + ignore; +do_apply_filters(_Owner,Log,[],log) -> + Log. + +handle_filter_failed({Id,_}=Filter,Owner,Log,Reason) -> + case logger_server:remove_filter(Owner,Id) of + ok -> + logger:internal_log(error,{removed_failing_filter,Id}), + ?LOG_INTERNAL(debug, + [{logger,removed_failing_filter}, + {filter,Filter}, + {owner,Owner}, + {log,Log}, + {reason,Reason}]); + _ -> + ok + end, + ignore. + +filter_stacktrace(Stacktrace) -> + logger:filter_stacktrace(?MODULE,Stacktrace). diff --git a/lib/kernel/src/logger_config.erl b/lib/kernel/src/logger_config.erl new file mode 100644 index 0000000000..799aea9617 --- /dev/null +++ b/lib/kernel/src/logger_config.erl @@ -0,0 +1,151 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(logger_config). + +-export([new/1,delete/2, + exist/2, + allow/2,allow/3, + get/2, get/3, get/1, + create/3, create/4, set/3, + set_module_level/3,reset_module_level/2, + cache_module_level/2, + level_to_int/1]). + +-include("logger_internal.hrl"). + +new(Name) -> + _ = ets:new(Name,[set,protected,named_table]), + ets:whereis(Name). + +delete(Tid,Id) -> + ets:delete(Tid,table_key(Id)). + +allow(Tid,Level,Module) -> + LevelInt = level_to_int(Level), + case ets:lookup(Tid,Module) of + [{Module,{ModLevel,cached}}] when is_integer(ModLevel), + LevelInt =< ModLevel -> + true; + [{Module,ModLevel}] when is_integer(ModLevel), + LevelInt =< ModLevel -> + true; + [] -> + logger_server:cache_module_level(Module), + allow(Tid,Level); + _ -> + false + end. + +allow(Tid,Level) -> + GlobalLevelInt = ets:lookup_element(Tid,?LOGGER_KEY,2), + level_to_int(Level) =< GlobalLevelInt. + +exist(Tid,What) -> + ets:member(Tid,table_key(What)). + +get(Tid,What) -> + case ets:lookup(Tid,table_key(What)) of + [{_,_,Config}] -> + {ok,Config}; + [{_,_,Config,Module}] -> + {ok,{Module,Config}}; + [] -> + {error,{not_found,What}} + end. + +get(Tid,What,Level) -> + MS = [{{table_key(What),'$1','$2'}, % logger config + [{'>=','$1',level_to_int(Level)}], + ['$2']}, + {{table_key(What),'$1','$2','$3'}, % handler config + [{'>=','$1',level_to_int(Level)}], + [{{'$3','$2'}}]}], + case ets:select(Tid,MS) of + [] -> error; + [Data] -> {ok,Data} + end. + +create(Tid,What,Module,Config) -> + LevelInt = level_to_int(maps:get(level,Config)), + ets:insert(Tid,{table_key(What),LevelInt,Config,Module}). +create(Tid,What,Config) -> + LevelInt = level_to_int(maps:get(level,Config)), + ets:insert(Tid,{table_key(What),LevelInt,Config}). + +set(Tid,What,Config) -> + LevelInt = level_to_int(maps:get(level,Config)), + %% Should do this only if the level has actually changed. Possibly + %% overwrite instead of delete? + case What of + logger -> + _ = ets:select_delete(Tid,[{{'_',{'$1',cached}}, + [{'=/=','$1',LevelInt}], + [true]}]), + ok; + _ -> + ok + end, + ets:update_element(Tid,table_key(What),[{2,LevelInt},{3,Config}]), + ok. + +set_module_level(Tid,Module,Level) -> + ets:insert(Tid,{Module,level_to_int(Level)}), + ok. + +reset_module_level(Tid,Module) -> + ets:delete(Tid,Module), % should possibley overwrite instead of delete? + ok. + +cache_module_level(Tid,Module) -> + GlobalLevelInt = ets:lookup_element(Tid,?LOGGER_KEY,2), + ets:insert_new(Tid,{Module,{GlobalLevelInt,cached}}), + ok. + +get(Tid) -> + {ok,Logger} = get(Tid,logger), + HMS = [{{table_key('$1'),'_','$2','$3'},[],[{{'$1','$3','$2'}}]}], + Handlers = ets:select(Tid,HMS), + MMS = [{{'$1','$2'},[{is_atom,'$1'},{is_integer,'$2'}],[{{'$1','$2'}}]}], + Modules = ets:select(Tid,MMS), + {Logger,Handlers,[{M,int_to_level(L)} || {M,L} <- Modules]}. + +level_to_int(emergency) -> ?EMERGENCY; +level_to_int(alert) -> ?ALERT; +level_to_int(critical) -> ?CRITICAL; +level_to_int(error) -> ?ERROR; +level_to_int(warning) -> ?WARNING; +level_to_int(notice) -> ?NOTICE; +level_to_int(info) -> ?INFO; +level_to_int(debug) -> ?DEBUG. + +int_to_level(?EMERGENCY) -> emergency; +int_to_level(?ALERT) -> alert; +int_to_level(?CRITICAL) -> critical; +int_to_level(?ERROR) -> error; +int_to_level(?WARNING) -> warning; +int_to_level(?NOTICE) -> notice; +int_to_level(?INFO) -> info; +int_to_level(?DEBUG) -> debug. + +%%%----------------------------------------------------------------- +%%% Internal + +table_key(logger) -> ?LOGGER_KEY; +table_key(HandlerId) -> {?HANDLER_KEY,HandlerId}. diff --git a/lib/kernel/src/logger_disk_log_h.erl b/lib/kernel/src/logger_disk_log_h.erl new file mode 100644 index 0000000000..3b71f936d8 --- /dev/null +++ b/lib/kernel/src/logger_disk_log_h.erl @@ -0,0 +1,694 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(logger_disk_log_h). + +-behaviour(gen_server). + +-include("logger.hrl"). +-include("logger_internal.hrl"). +-include("logger_h_common.hrl"). + +%%% API +-export([start_link/3, info/1, disk_log_sync/1, reset/1]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +%% logger callbacks +-export([log/2, + adding_handler/2, removing_handler/1, + changing_config/3, swap_buffer/2]). + +%%%=================================================================== +%%% API +%%%=================================================================== + +%%%----------------------------------------------------------------- +%%% Start a disk_log handler process and link to caller. +%%% This function is called by the kernel supervisor when this +%%% handler process gets added (as a result of calling add/3). +-spec start_link(Name, Config, HandlerState) -> {ok,Pid} | {error,Reason} when + Name :: atom(), + Config :: logger:config(), + HandlerState :: map(), + Pid :: pid(), + Reason :: term(). + +start_link(Name, Config, HandlerState) -> + proc_lib:start_link(?MODULE,init,[[Name,Config,HandlerState]]). + +%%%----------------------------------------------------------------- +%%% +-spec disk_log_sync(Name) -> ok | {error,Reason} when + Name :: atom(), + Reason :: handler_busy | {badarg,term()}. + +disk_log_sync(Name) when is_atom(Name) -> + try + gen_server:call(Name, disk_log_sync, ?DEFAULT_CALL_TIMEOUT) + catch + _:{timeout,_} -> {error,handler_busy} + end; +disk_log_sync(Name) -> + {error,{badarg,{disk_log_sync,[Name]}}}. + +%%%----------------------------------------------------------------- +%%% +-spec info(Name) -> Info | {error,Reason} when + Name :: atom(), + Info :: term(), + Reason :: handler_busy | {badarg,term()}. + +info(Name) when is_atom(Name) -> + try + gen_server:call(Name, info, ?DEFAULT_CALL_TIMEOUT) + catch + _:{timeout,_} -> {error,handler_busy} + end; +info(Name) -> + {error,{badarg,{info,[Name]}}}. + +%%%----------------------------------------------------------------- +%%% +-spec reset(Name) -> ok | {error,Reason} when + Name :: atom(), + Reason :: handler_busy | {badarg,term()}. + +reset(Name) when is_atom(Name) -> + try + gen_server:call(Name, reset, ?DEFAULT_CALL_TIMEOUT) + catch + _:{timeout,_} -> {error,handler_busy} + end; +reset(Name) -> + {error,{badarg,{reset,[Name]}}}. + + +%%%=================================================================== +%%% logger callbacks +%%%=================================================================== + +%%%----------------------------------------------------------------- +%%% Handler being added +adding_handler(Name, Config) -> + case check_config(adding, Name, Config) of + {ok, Config1} -> + %% create initial handler state by merging defaults with config + HConfig = maps:get(?MODULE, Config1, #{}), + HState = maps:merge(get_init_state(), HConfig), + case logger_h_common:overload_levels_ok(HState) of + true -> + case start(Name, Config1, HState) of + ok -> + %% Make sure wait_for_buffer is not stored, so we + %% won't hang and wait for buffer on a restart + {ok, maps:remove(wait_for_buffer,Config1)}; + Error -> + Error + end; + false -> + #{toggle_sync_qlen := TSQL, + drop_new_reqs_qlen := DNRQL, + flush_reqs_qlen := FRQL} = HState, + {error,{invalid_levels,{TSQL,DNRQL,FRQL}}} + end; + Error -> + Error + end. + +%%%----------------------------------------------------------------- +%%% Updating handler config +changing_config(Name, + OldConfig=#{id:=Id, disk_log_opts:=DLOpts}, + NewConfig=#{id:=Id, disk_log_opts:=DLOpts}) -> + case check_config(changing, Name, NewConfig) of + Result = {ok,NewConfig1} -> + try gen_server:call(Name, {change_config,OldConfig,NewConfig1}, + ?DEFAULT_CALL_TIMEOUT) of + ok -> Result; + HError -> HError + catch + _:{timeout,_} -> {error,handler_busy} + end; + Error -> + Error + end; +changing_config(_Name, OldConfig, NewConfig) -> + {error,{illegal_config_change,OldConfig,NewConfig}}. + +check_config(adding, Name, Config0) -> + %% Merge in defaults on top level + Config = maps:merge(#{id => Name}, Config0), + %% Merge in defaults on handler level + LogOpts0 = maps:get(disk_log_opts, Config, #{}), + LogOpts = merge_default_logopts(Name, LogOpts0), + case check_log_opts(maps:to_list(LogOpts)) of + ok -> + MyConfig = maps:get(?MODULE, Config, #{}), + case check_my_config(maps:to_list(MyConfig)) of + ok -> + {ok,Config#{disk_log_opts=>LogOpts, + ?MODULE=>MyConfig}}; + Error -> + Error + end; + Error -> + Error + end; +check_config(changing, _Name, Config) -> + MyConfig = maps:get(?MODULE, Config, #{}), + case check_my_config(maps:to_list(MyConfig)) of + ok -> {ok,Config}; + Error -> Error + end. + +merge_default_logopts(Name, LogOpts) -> + Type = maps:get(type, LogOpts, wrap), + {DefaultNoFiles,DefaultNoBytes} = + case Type of + halt -> {undefined,infinity}; + _wrap -> {10,1048576} + end, + {ok,Dir} = file:get_cwd(), + Default = #{file => filename:join(Dir,Name), + max_no_files => DefaultNoFiles, + max_no_bytes => DefaultNoBytes, + type => Type}, + maps:merge(Default,LogOpts). + +check_log_opts([{file,File}|Opts]) when is_list(File) -> + check_log_opts(Opts); +check_log_opts([{max_no_files,undefined}|Opts]) -> + check_log_opts(Opts); +check_log_opts([{max_no_files,N}|Opts]) when is_integer(N), N>0 -> + check_log_opts(Opts); +check_log_opts([{max_no_bytes,infinity}|Opts]) -> + check_log_opts(Opts); +check_log_opts([{max_no_bytes,N}|Opts]) when is_integer(N), N>0 -> + check_log_opts(Opts); +check_log_opts([{type,Type}|Opts]) when Type==wrap; Type==halt -> + check_log_opts(Opts); +check_log_opts([Invalid|_]) -> + {error,{invalid_config,disk_log_opt,Invalid}}; +check_log_opts([]) -> + ok. + +check_my_config([Other | Config]) -> + case logger_h_common:check_common_config(Other) of + valid -> + check_my_config(Config); + invalid -> + {error,{invalid_config,?MODULE,Other}} + end; +check_my_config([]) -> + ok. + +%%%----------------------------------------------------------------- +%%% Handler being removed +removing_handler(Name) -> + stop(Name). + +%%%----------------------------------------------------------------- +%%% Get buffer when swapping from simple handler +swap_buffer(Name,Buffer) -> + case whereis(Name) of + undefined -> + ok; + _ -> + Name ! {buffer,Buffer} + end. + +%%%----------------------------------------------------------------- +%%% Log a string or report +-spec log(Log, Config) -> ok | dropped when + Log :: logger:log(), + Config :: logger:config(). + +log(Log,Config=#{id:=Name}) -> + %% if the handler has crashed, we must drop this request + %% and hope the handler restarts so we can try again + true = is_pid(whereis(Name)), + Bin = logger_h_common:log_to_binary(Log,Config), + logger_h_common:call_cast_or_drop(Name, Bin). + + +%%%=================================================================== +%%% gen_server callbacks +%%%=================================================================== + +init([Name, Config = #{disk_log_opts := LogOpts}, + State = #{dl_sync_int := DLSyncInt}]) -> + register(Name, self()), + process_flag(trap_exit, true), + process_flag(message_queue_data, off_heap), + + ?init_test_hooks(), + ?start_observation(Name), + + case open_disk_log(Name, LogOpts) of + ok -> + catch ets:new(Name, [public, named_table]), + ?set_mode(Name, async), + proc_lib:init_ack({ok,self()}), + T0 = ?timestamp(), + State1 = + ?merge_with_stats(State#{id => Name, + mode => async, + dl_sync => DLSyncInt, + log_opts => LogOpts, + last_qlen => 0, + last_log_ts => T0, + burst_win_ts => T0, + burst_msg_count => 0, + prev_log_result => ok, + prev_sync_result => ok, + prev_disk_log_info => undefined}), + gen_server:cast(self(), {repeated_disk_log_sync,T0}), + enter_loop(Config, State1); + Error -> + logger_h_common:error_notify({open_disk_log,Name,Error}), + proc_lib:init_ack(Error) + end. + +enter_loop(#{wait_for_buffer:=true}=Config,State) -> + State1 = + receive + {buffer,Buffer} -> + lists:foldl( + fun(Log,S) -> + Bin = logger_h_common:log_to_binary(Log,Config), + {_,S1} = do_log(Bin,cast,S), + S1 + end, + State, + Buffer) + end, + gen_server:enter_loop(?MODULE,[],State1); +enter_loop(_Config,State) -> + gen_server:enter_loop(?MODULE,[],State). + +%% This is the synchronous log request. +handle_call({log, Bin}, _From, State) -> + {Result,State1} = do_log(Bin, call, State), + %% Result == ok | dropped + {reply, Result, State1}; + +handle_call(disk_log_sync, _From, State = #{id := Name}) -> + State1 = #{prev_sync_result := Result} = disk_log_sync(Name, State), + {reply, Result, State1}; + +handle_call({change_config,_OldConfig,NewConfig}, _From, + State = #{filesync_repeat_interval := FSyncInt0, + last_log_ts := LastLogTS}) -> + HConfig = maps:get(?MODULE, NewConfig, #{}), + State1 = #{toggle_sync_qlen := TSQL, + drop_new_reqs_qlen := DNRQL, + flush_reqs_qlen := FRQL} = maps:merge(State, HConfig), + case logger_h_common:overload_levels_ok(State1) of + true -> + _ = + case maps:get(filesync_repeat_interval, HConfig, undefined) of + undefined -> + ok; + no_repeat -> + _ = logger_h_common:cancel_timer(maps:get(rep_sync_tref, + State, + undefined)); + FSyncInt0 -> + ok; + _FSyncInt1 -> + _ = logger_h_common:cancel_timer(maps:get(rep_sync_tref, + State, + undefined)), + _ = gen_server:cast(self(), {repeated_disk_log_sync, + LastLogTS}) + end, + {reply, ok, State1}; + false -> + {reply, {error,{invalid_levels,{TSQL,DNRQL,FRQL}}}, State} + end; + +handle_call(info, _From, State) -> + {reply, State, State}; + +handle_call(reset, _From, State) -> + State1 = ?merge_with_stats(State), + {reply, ok, State1#{last_qlen => 0, + last_log_ts => ?timestamp(), + prev_log_result => ok, + prev_sync_result => ok, + prev_disk_log_info => undefined}}; + +handle_call(stop, _From, State) -> + {stop, {shutdown,stopped}, ok, State}. + + +%% This is the asynchronous log request. +handle_cast({log, Bin}, State) -> + {_,State1} = do_log(Bin, cast, State), + {noreply, State1}; + +%% If FILESYNC_REPEAT_INTERVAL is set to a millisec value, this +%% clause gets called repeatedly by the handler. In order to +%% guarantee that a filesync *always* happens after the last log +%% request, the repeat operation must be active! +handle_cast({repeated_disk_log_sync,LastLogTS0}, + State = #{id := Name, + filesync_repeat_interval := FSyncInt, + last_log_ts := LastLogTS1}) -> + State1 = + if is_integer(FSyncInt) -> + %% only do filesync if something has been + %% written since last time we checked + NewState = if LastLogTS1 == LastLogTS0 -> + State; + true -> + disk_log_sync(Name, State) + end, + {ok,TRef} = + timer:apply_after(FSyncInt, gen_server,cast, + [self(), + {repeated_disk_log_sync,LastLogTS1}]), + NewState#{rep_sync_tref => TRef}; + true -> + State + end, + {noreply,State1}. + +%% The disk log owner must handle status messages from disk_log. +handle_info({disk_log, _Node, _Log, {wrap,_NoLostItems}}, State) -> + {noreply, State}; +handle_info({disk_log, _Node, Log, Info = {truncated,_NoLostItems}}, + State = #{id := Name, prev_disk_log_info := PrevInfo}) -> + error_notify_new(Info, PrevInfo, {disk_log,Name,Log,Info}), + {noreply, State#{prev_disk_log_info => Info}}; +handle_info({disk_log, _Node, Log, Info = {blocked_log,_Items}}, + State = #{id := Name, prev_disk_log_info := PrevInfo}) -> + error_notify_new(Info, PrevInfo, {disk_log,Name,Log,Info}), + {noreply, State#{prev_disk_log_info => Info}}; +handle_info({disk_log, _Node, Log, full}, + State = #{id := Name, prev_disk_log_info := PrevInfo}) -> + error_notify_new(full, PrevInfo, {disk_log,Name,Log,full}), + {noreply, State#{prev_disk_log_info => full}}; +handle_info({disk_log, _Node, Log, Info = {error_status,_Status}}, + State = #{id := Name, prev_disk_log_info := PrevInfo}) -> + error_notify_new(Info, PrevInfo, {disk_log,Name,Log,Info}), + {noreply, State#{prev_disk_log_info => Info}}; + +handle_info({'EXIT',_Pid,_Why}, State = #{id := _Name}) -> + {noreply, State}; + +handle_info(_, State) -> + {noreply, State}. + +terminate(Reason, State = #{id := Name}) -> + _ = logger_h_common:cancel_timer(maps:get(rep_sync_tref, State, + undefined)), + _ = close_disk_log(Name, normal), + logger_h_common:stop_or_restart(Name, Reason, State). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%%----------------------------------------------------------------- +%%% Internal functions + +%%%----------------------------------------------------------------- +%%% +get_init_state() -> + #{toggle_sync_qlen => ?TOGGLE_SYNC_QLEN, + drop_new_reqs_qlen => ?DROP_NEW_REQS_QLEN, + flush_reqs_qlen => ?FLUSH_REQS_QLEN, + enable_burst_limit => ?ENABLE_BURST_LIMIT, + burst_limit_size => ?BURST_LIMIT_SIZE, + burst_window_time => ?BURST_WINDOW_TIME, + enable_kill_overloaded => ?ENABLE_KILL_OVERLOADED, + handler_overloaded_qlen => ?HANDLER_OVERLOADED_QLEN, + handler_overloaded_mem => ?HANDLER_OVERLOADED_MEM, + handler_restart_after => ?HANDLER_RESTART_AFTER, + dl_sync_int => ?CONTROLLER_SYNC_INTERVAL, + filesync_ok_qlen => ?FILESYNC_OK_QLEN, + filesync_repeat_interval => ?FILESYNC_REPEAT_INTERVAL}. + +%%%----------------------------------------------------------------- +%%% Add a disk_log handler to the logger. +%%% This starts a dedicated handler process which should always +%%% exist if the handler is registered with logger (and should not +%%% exist if the handler is not registered). +%%% +%%% Config is the logger:config() map containing a sub map with any of +%%% the following associations: +%%% +%%% Config = #{disk_log_opts => #{file => file:filename(), +%%% max_no_bytes => integer(), +%%% max_no_files => integer(), +%%% type => wrap | halt}}. +%%% +%%% This map will be merged with the logger configuration data for +%%% the disk_log LogName. If type == halt, then max_no_files is +%%% ignored. +%%% +%%% Handler specific config should be provided with a sub map associated +%%% with a key named the same as this module, e.g: +%%% +%%% Config = #{logger_disk_log_h => #{toggle_sync_qlen => 50} +%%% +%%% The disk_log handler process is linked to logger_sup, which is +%%% part of the kernel application's supervision tree. +start(Name, Config, HandlerState) -> + LoggerDLH = + #{id => Name, + start => {?MODULE, start_link, [Name,Config,HandlerState]}, + restart => temporary, + shutdown => 2000, + type => worker, + modules => [?MODULE]}, + case supervisor:start_child(logger_sup, LoggerDLH) of + {ok,_} -> + ok; + Error -> + Error + end. + +%%%----------------------------------------------------------------- +%%% Stop and remove the handler. +stop(Name) -> + case whereis(Name) of + undefined -> + ok; + _ -> + %% We don't want to do supervisor:terminate_child here + %% since we need to distinguish this explicit stop from a + %% system termination in order to avoid circular attempts + %% at removing the handler (implying deadlocks and + %% timeouts). + _ = gen_server:call(Name,stop), + _ = supervisor:delete_child(logger_sup, Name), + ok + end. + +%%%----------------------------------------------------------------- +%%% Logging and overload control. +-define(update_dl_sync(C, Interval), + if C == 0 -> Interval; + true -> C-1 end). + +%% check for overload between every request (and set Mode to async, +%% sync or drop accordingly), but never flush the whole mailbox +%% before LogWindowSize requests have been handled +do_log(Bin, CallOrCast, State = #{id:=Name, mode := _Mode0}) -> + T1 = ?timestamp(), + + %% check if the handler is getting overloaded, or if it's + %% recovering from overload (the check must be done for each + %% request to react quickly to large bursts of requests and + %% to ensure that the handler can never end up in drop mode + %% with an empty mailbox, which would stop operation) + {Mode1,QLen,Mem,State1} = logger_h_common:check_load(State), + + %% kill the handler if it can't keep up with the load + logger_h_common:kill_if_choked(Name, QLen, Mem, State), + + if Mode1 == flush -> + flush(Name, QLen, T1, State1); + true -> + write(Name, Mode1, T1, Bin, CallOrCast, State1) + end. + +%% this function is called by do_log/3 after an overload check +%% has been performed, where QLen > FlushQLen +flush(Name, _QLen0, T1, State=#{last_log_ts := _T0}) -> + %% flush messages in the mailbox (a limited number in + %% order to not cause long delays) + _NewFlushed = logger_h_common:flush_log_requests(?FLUSH_MAX_N), + + %% because of the receive loop when flushing messages, the + %% handler will be scheduled out often and the mailbox could + %% grow very large, so we'd better check the queue again here + {_,_QLen1} = process_info(self(), message_queue_len), + ?observe(Name,{max_qlen,_QLen1}), + + %% Add 1 for the current log request + ?observe(Name,{flushed,_NewFlushed+1}), + + State1 = ?update_max_time(?diff_time(T1,_T0),State), + {dropped,?update_other(flushed,FLUSHED,_NewFlushed, + State1#{mode => ?set_mode(Name,async), + last_qlen => 0, + last_log_ts => T1})}. + +%% this function is called to write to disk_log +write(Name, Mode, T1, Bin, _CallOrCast, + State = #{dl_sync := DLSync, + dl_sync_int := DLSyncInt, + last_qlen := LastQLen, + last_log_ts := T0}) -> + %% check if we need to limit the number of writes + %% during a burst of log requests + {DoWrite,BurstWinT,BurstMsgCount} = logger_h_common:limit_burst(State), + + %% only send a synhrounous request to the disk_log process + %% every DLSyncInt time, to give the handler time between + %% writes so it can keep up with incoming messages + {Status,LastQLen1,State1} = + if DoWrite, DLSync == 0 -> + ?observe(Name,{_CallOrCast,1}), + NewState = disk_log_write(Name, Bin, State), + {ok, element(2,process_info(self(),message_queue_len)), + NewState}; + DoWrite -> + ?observe(Name,{_CallOrCast,1}), + NewState = disk_log_write(Name, Bin, State), + {ok, LastQLen, NewState}; + not DoWrite -> + ?observe(Name,{flushed,1}), + {dropped, LastQLen, State} + end, + + %% Check if the time since the previous log request is long enough - + %% and the queue length small enough - to assume the mailbox has + %% been emptied, and if so, do filesync operation and reset mode to + %% async. Note that this is the best we can do to detect an idle + %% handler without setting a timer after each log call/cast. If the + %% time between two consecutive log requests is fast and no new + %% request comes in after the last one, idle state won't be detected! + Time = ?diff_time(T1,T0), + {Mode1,BurstMsgCount1,State2} = + if (LastQLen1 < ?FILESYNC_OK_QLEN) andalso + (Time > ?IDLE_DETECT_TIME_USEC) -> + {?change_mode(Name,Mode,async), 0, disk_log_sync(Name,State1)}; + true -> + {Mode, BurstMsgCount,State1} + end, + + State3 = + ?update_calls_or_casts(_CallOrCast,1,State2), + State4 = + ?update_max_time(Time, + State3#{mode => Mode1, + last_qlen := LastQLen1, + last_log_ts => T1, + burst_win_ts => BurstWinT, + burst_msg_count => BurstMsgCount1, + dl_sync => ?update_dl_sync(DLSync,DLSyncInt)}), + {Status,State4}. + + +open_disk_log(Name, LogOpts) -> + #{file := File, + max_no_bytes := MaxNoBytes, + max_no_files := MaxNoFiles, + type := Type} = LogOpts, + case filelib:ensure_dir(File) of + ok -> + Size = + if Type == halt -> MaxNoBytes; + Type == wrap -> {MaxNoBytes,MaxNoFiles} + end, + Opts = [{name, Name}, + {file, File}, + {size, Size}, + {type, Type}, + {linkto, self()}, + {repair, false}, + {format, external}, + {notify, true}, + {quiet, true}, + {mode, read_write}], + case disk_log:open(Opts) of + {ok,Name} -> + ok; + Error = {error,_Reason} -> + Error + end; + Error -> + Error + end. + +close_disk_log(Name, _) -> + _ = ?disk_log_sync(Name), + _ = disk_log:lclose(Name), + ok. + +disk_log_write(Name, Bin, State) -> + Result = + case ?disk_log_blog(Name, Bin) of + ok -> + ok; + LogError -> + _ = case maps:get(prev_log_result, State) of + LogError -> + %% don't report same error twice + ok; + _ -> + LogOpts = maps:get(log_opts, State), + logger_h_common:error_notify({Name,log, + LogOpts, + LogError}) + end, + LogError + end, + State#{prev_log_result => Result}. + +disk_log_sync(Name, State) -> + Result = + case ?disk_log_sync(Name) of + ok -> + ok; + SyncError -> + _ = case maps:get(prev_sync_result, State) of + SyncError -> + %% don't report same error twice + ok; + _ -> + LogOpts = maps:get(log_opts, State), + logger_h_common:error_notify({Name,sync, + LogOpts, + SyncError}) + end, + SyncError + end, + State#{prev_sync_result => Result}. + +error_notify_new(Info,Info, _Term) -> + ok; +error_notify_new(_Info0,_Info1, Term) -> + logger_h_common:error_notify(Term). diff --git a/lib/kernel/src/logger_filters.erl b/lib/kernel/src/logger_filters.erl new file mode 100644 index 0000000000..85928f0fd6 --- /dev/null +++ b/lib/kernel/src/logger_filters.erl @@ -0,0 +1,123 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(logger_filters). + +-export([domain/2, + level/2, + progress/2, + remote_gl/2]). + +-include("logger_internal.hrl"). +-define(IS_ACTION(A), (A==log orelse A==stop)). + +-spec domain(Log,Extra) -> logger:filter_return() when + Log :: logger:log(), + Extra :: {Action,Compare,MatchDomain}, + Action :: log | stop, + Compare :: prefix_of | starts_with | equals | no_domain, + MatchDomain :: list(atom()). +domain(#{meta:=Meta}=Log,{Action,Compare,MatchDomain}) + when ?IS_ACTION(Action) andalso + (Compare==prefix_of orelse + Compare==starts_with orelse + Compare==equals orelse + Compare==no_domain) andalso + is_list(MatchDomain) -> + filter_domain(Compare,Meta,MatchDomain,on_match(Action,Log)); +domain(Log,Extra) -> + erlang:error(badarg,[Log,Extra]). + +-spec level(Log,Extra) -> logger:filter_return() when + Log :: logger:log(), + Extra :: {Action,Operator,MatchLevel}, + Action :: log | stop, + Operator :: neq | eq | lt | gt | lteq | gteq, + MatchLevel :: logger:level(). +level(#{level:=L1}=Log,{Action,Op,L2}) + when ?IS_ACTION(Action) andalso + (Op==neq orelse + Op==eq orelse + Op==lt orelse + Op==gt orelse + Op==lteq orelse + Op==gteq) andalso + ?IS_LEVEL(L2) -> + filter_level(Op,L1,L2,on_match(Action,Log)); +level(Log,Extra) -> + erlang:error(badarg,[Log,Extra]). + +-spec progress(Log,Extra) -> logger:filter_return() when + Log :: logger:log(), + Extra :: log | stop. +progress(Log,Action) when ?IS_ACTION(Action) -> + filter_progress(Log,on_match(Action,Log)); +progress(Log,Action) -> + erlang:error(badarg,[Log,Action]). + +-spec remote_gl(Log,Extra) -> logger:filter_return() when + Log :: logger:log(), + Extra :: log | stop. +remote_gl(Log,Action) when ?IS_ACTION(Action) -> + filter_remote_gl(Log,on_match(Action,Log)); +remote_gl(Log,Action) -> + erlang:error(badarg,[Log,Action]). + +%%%----------------------------------------------------------------- +%%% Internal +filter_domain(prefix_of,#{domain:=Domain},MatchDomain,OnMatch) -> + is_prefix(Domain,MatchDomain,OnMatch); +filter_domain(starts_with,#{domain:=Domain},MatchDomain,OnMatch) -> + is_prefix(MatchDomain,Domain,OnMatch); +filter_domain(equals,#{domain:=Domain},Domain,OnMatch) -> + OnMatch; +filter_domain(Action,Meta,_,OnMatch) -> + case maps:is_key(domain,Meta) of + false when Action==no_domain -> OnMatch; + _ -> ignore + end. + +is_prefix(D1,D2,OnMatch) when is_list(D1), is_list(D2) -> + case lists:prefix(D1,D2) of + true -> OnMatch; + false -> ignore + end; +is_prefix(_,_,_) -> + ignore. + +filter_level(Op,L1,L2,OnMatch) -> + case logger:compare_levels(L1,L2) of + eq when Op==eq; Op==lteq; Op==gteq -> OnMatch; + lt when Op==lt; Op==lteq; Op==neq -> OnMatch; + gt when Op==gt; Op==gteq; Op==neq -> OnMatch; + _ -> ignore + end. + +filter_progress(#{msg:={report,#{label:={_,progress}}}},OnMatch) -> + OnMatch; +filter_progress(_,_) -> + ignore. + +filter_remote_gl(#{meta:=#{gl:=GL}},OnMatch) when node(GL)=/=node() -> + OnMatch; +filter_remote_gl(_,_) -> + ignore. + +on_match(log,Log) -> Log; +on_match(stop,_) -> stop. diff --git a/lib/kernel/src/logger_formatter.erl b/lib/kernel/src/logger_formatter.erl new file mode 100644 index 0000000000..386e7832e2 --- /dev/null +++ b/lib/kernel/src/logger_formatter.erl @@ -0,0 +1,295 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(logger_formatter). + +-export([format/2]). + +-include("logger_internal.hrl"). + +%%%----------------------------------------------------------------- +%%% Types +-type template() :: [atom()|tuple()|string()]. + +%%%----------------------------------------------------------------- +%%% API +-spec format(Log,Config) -> String when + Log :: logger:log(), + Config :: #{single_line=>boolean(), + legacy_header=>boolean(), + report_cb=>fun((logger:report()) -> {io:format(),[term()]}), + chars_limit=>pos_integer()| unlimited, + max_size=>pos_integer() | unlimited, + depth=>pos_integer() | unlimited, + template=>template(), + utc=>boolean()}, + String :: string(). +format(#{level:=Level,msg:=Msg0,meta:=Meta},Config0) + when is_map(Config0) -> + Config = add_default_config(Config0), + Meta1 = maybe_add_legacy_header(Level,Meta,Config), + Template = maps:get(template,Config), + {BT,AT0} = lists:splitwith(fun(msg) -> false; (_) -> true end, Template), + {DoMsg,AT} = + case AT0 of + [msg|Rest] -> {true,Rest}; + _ ->{false,AT0} + end, + B = do_format(Level,"",Meta1,BT,Config), + A = do_format(Level,"",Meta1,AT,Config), + MsgStr = + if DoMsg -> + Config1 = + case maps:get(chars_limit,Config) of + unlimited -> + Config; + Size0 -> + Size = + case Size0 - string:length([B,A]) of + S when S>=0 -> S; + _ -> 0 + end, + Config#{chars_limit=>Size} + end, + MsgStr0 = format_msg(Msg0,Meta1,Config1), + case maps:get(single_line,Config) of + true -> + %% Trim leading and trailing whitespaces, and replace + %% newlines with ", " + re:replace(string:trim(MsgStr0),",?\r?\n\s*",", ", + [{return,list},global]); + _false -> + MsgStr0 + end; + true -> + "" + end, + truncate(B ++ MsgStr ++ A,maps:get(max_size,Config)). + +do_format(Level,Msg,Data,[level|Format],Config) -> + [to_string(level,Level,Config)|do_format(Level,Msg,Data,Format,Config)]; +do_format(Level,Msg,Data,[msg|Format],Config) -> + [Msg|do_format(Level,Msg,Data,Format,Config)]; +do_format(Level,Msg,Data,[Key|Format],Config) when is_atom(Key); is_tuple(Key) -> + Value = value(Key,Data), + [to_string(Key,Value,Config)|do_format(Level,Msg,Data,Format,Config)]; +do_format(Level,Msg,Data,[Str|Format],Config) -> + [Str|do_format(Level,Msg,Data,Format,Config)]; +do_format(_Level,_Msg,_Data,[],_Config) -> + []. + +value(Key,Meta) when is_atom(Key), is_map(Meta) -> + maps:get(Key,Meta,""); +value(Key,_) when is_atom(Key) -> + ""; +value(Keys,Meta) when is_tuple(Keys) -> + value(tuple_to_list(Keys),Meta); +value([Key|Keys],Meta) -> + value(Keys,value(Key,Meta)); +value([],Value) -> + Value. + +to_string(time,Time,Config) -> + format_time(Time,Config); +to_string(mfa,MFA,_Config) -> + format_mfa(MFA); +to_string(_,Value,_Config) -> + to_string(Value). + +to_string(X) when is_atom(X) -> + atom_to_list(X); +to_string(X) when is_integer(X) -> + integer_to_list(X); +to_string(X) when is_pid(X) -> + pid_to_list(X); +to_string(X) when is_reference(X) -> + ref_to_list(X); +to_string(X) when is_list(X) -> + case io_lib:printable_unicode_list(lists:flatten(X)) of + true -> X; + _ -> io_lib:format("~tp",[X]) + end; +to_string(X) -> + io_lib:format("~tp",[X]). + +format_msg({string,Chardata},Meta,Config) -> + try unicode:characters_to_list(Chardata) + catch _:_ -> format_msg({"INVALID STRING: ~tp",[Chardata]},Meta,Config) + end; +format_msg({report,_}=Msg,Meta,#{report_cb:=Fun}=Config) when is_function(Fun,1) -> + format_msg(Msg,Meta#{report_cb=>Fun},maps:remove(report_cb,Config)); +format_msg({report,Report},#{report_cb:=Fun}=Meta,Config) when is_function(Fun,1) -> + try Fun(Report) of + {Format,Args} when is_list(Format), is_list(Args) -> + format_msg({Format,Args},maps:remove(report_cb,Meta),Config); + Other -> + format_msg({"REPORT_CB ERROR: ~tp; Returned: ~tp", + [Report,Other]},Meta,Config) + catch C:R -> + format_msg({"REPORT_CB CRASH: ~tp; Reason: ~tp", + [Report,{C,R}]},Meta,Config) + end; +format_msg({report,Report},Meta,Config) -> + format_msg({report,Report}, + Meta#{report_cb=>fun logger:format_report/1}, + Config); +format_msg(Msg,_Meta,#{depth:=Depth,chars_limit:=CharsLimit}) -> + limit_size(Msg, Depth, CharsLimit). + +limit_size(Msg,Depth,unlimited) -> + limit_size(Msg,Depth,[]); +limit_size(Msg,Depth,CharsLimit) when is_integer(CharsLimit) -> + limit_size(Msg,Depth,[{chars_limit,CharsLimit}]); +limit_size({Format,Args},unlimited,Opts) when is_list(Opts) -> + try io_lib:format(Format,Args,Opts) + catch _:_ -> + io_lib:format("FORMAT ERROR: ~tp - ~tp",[Format,Args],Opts) + end; +limit_size({Format0,Args},Depth,Opts) when is_integer(Depth) -> + try + Format1 = io_lib:scan_format(Format0, Args), + Format = limit_format(Format1, Depth), + io_lib:build_text(Format,Opts) + catch _:_ -> + limit_size({"FORMAT ERROR: ~tp - ~tp",[Format0,Args]},Depth,Opts) + end. + +limit_format([#{control_char:=C0}=M0|T], Depth) when C0 =:= $p; + C0 =:= $w -> + C = C0 - ($a - $A), %To uppercase. + #{args:=Args} = M0, + M = M0#{control_char:=C,args:=Args++[Depth]}, + [M|limit_format(T, Depth)]; +limit_format([H|T], Depth) -> + [H|limit_format(T, Depth)]; +limit_format([], _) -> + []. + +truncate(String,unlimited) -> + String; +truncate(String,Size) -> + Length = string:length(String), + if Length>Size -> + case lists:reverse(lists:flatten(String)) of + [$\n|_] -> + string:slice(String,0,Size-4)++"...\n"; + _ -> + string:slice(String,0,Size-3)++"..." + end; + true -> + String + end. + +format_time(Timestamp,Config) when is_integer(Timestamp) -> + {Date,Time,Micro} = timestamp_to_datetimemicro(Timestamp,Config), + format_time(Date,Time,Micro); +format_time(Other,_Config) -> + %% E.g. a string + to_string(Other). + +format_time({Y,M,D},{H,Min,S},Micro) -> + io_lib:format("~4w-~2..0w-~2..0w ~2w:~2..0w:~2..0w.~6..0w", + [Y,M,D,H,Min,S,Micro]). + +%% Assuming this is monotonic time in microseconds +timestamp_to_datetimemicro(Timestamp,Config) when is_integer(Timestamp) -> + SysTime = Timestamp + erlang:time_offset(microsecond), + Micro = SysTime rem 1000000, + Sec = SysTime div 1000000, + UniversalTime = erlang:posixtime_to_universaltime(Sec), + {Date,Time} = + case Config of + #{utc:=true} -> UniversalTime; + _ -> erlang:universaltime_to_localtime(UniversalTime) + end, + {Date,Time,Micro}. + +format_mfa({M,F,A}) when is_atom(M), is_atom(F), is_integer(A) -> + atom_to_list(M)++":"++atom_to_list(F)++"/"++integer_to_list(A); +format_mfa({M,F,A}) when is_atom(M), is_atom(F), is_list(A) -> + format_mfa({M,F,length(A)}); +format_mfa(MFA) -> + to_string(MFA). + +maybe_add_legacy_header(Level, + #{time:=Timestamp}=Meta, + #{legacy_header:=true}=Config) -> + #{title:=Title}=MyMeta = add_legacy_title(Level,maps:get(?MODULE,Meta,#{})), + {{Y,Mo,D},{H,Mi,S},Micro} = timestamp_to_datetimemicro(Timestamp,Config), + Header = io_lib:format("=~ts==== ~w-~s-~4w::~2..0w:~2..0w:~2..0w.~6..0w ~s===", + [Title,D,month(Mo),Y,H,Mi,S,Micro,utcstr(Config)]), + Meta#{?MODULE=>MyMeta#{header=>Header}}; +maybe_add_legacy_header(_,Meta,_) -> + Meta. + +add_legacy_title(_Level,#{title:=_}=MyMeta) -> + MyMeta; +add_legacy_title(Level,MyMeta) -> + Title = string:uppercase(atom_to_list(Level)) ++ " REPORT", + MyMeta#{title=>Title}. + +month(1) -> "Jan"; +month(2) -> "Feb"; +month(3) -> "Mar"; +month(4) -> "Apr"; +month(5) -> "May"; +month(6) -> "Jun"; +month(7) -> "Jul"; +month(8) -> "Aug"; +month(9) -> "Sep"; +month(10) -> "Oct"; +month(11) -> "Nov"; +month(12) -> "Dec". + +utcstr(#{utc:=true}) -> "UTC "; +utcstr(_) -> "". + +add_default_config(#{utc:=_}=Config0) -> + Default = + #{legacy_header=>false, + single_line=>false, + chars_limit=>unlimited}, + MaxSize = get_max_size(maps:get(max_size,Config0,false)), + Depth = get_depth(maps:get(depth,Config0,false)), + add_default_template(maps:merge(Default,Config0#{max_size=>MaxSize, + depth=>Depth})); +add_default_config(Config) -> + add_default_config(Config#{utc=>logger:get_utc_config()}). + +add_default_template(#{template:=_}=Config) -> + Config; +add_default_template(Config) -> + Config#{template=>default_template(Config)}. + +default_template(#{legacy_header:=true}) -> + ?DEFAULT_FORMAT_TEMPLATE_HEADER; +default_template(#{single_line:=true}) -> + ?DEFAULT_FORMAT_TEMPLATE_SINGLE; +default_template(_) -> + ?DEFAULT_FORMAT_TEMPLATE. + +get_max_size(false) -> + logger:get_max_size(); +get_max_size(S) -> + max(10,S). + +get_depth(false) -> + logger:get_format_depth(); +get_depth(S) -> + max(5,S). diff --git a/lib/kernel/src/logger_h_common.erl b/lib/kernel/src/logger_h_common.erl new file mode 100644 index 0000000000..7caad366ae --- /dev/null +++ b/lib/kernel/src/logger_h_common.erl @@ -0,0 +1,301 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(logger_h_common). + +-include("logger_h_common.hrl"). +-include("logger_internal.hrl"). + +-export([log_to_binary/2, + check_common_config/1, + call_cast_or_drop/2, + check_load/1, + limit_burst/1, + kill_if_choked/4, + flush_log_requests/0, + flush_log_requests/1, + handler_exit/2, + cancel_timer/1, + stop_or_restart/3, + overload_levels_ok/1, + error_notify/1, + info_notify/1]). + +%%%----------------------------------------------------------------- +%%% Covert log data on any form to binary +-spec log_to_binary(Log,Config) -> LogString when + Log :: logger:log(), + Config :: logger:config(), + LogString :: binary(). +log_to_binary(#{msg:={report,_},meta:=#{report_cb:=_}}=Log,Config) -> + do_log_to_binary(Log,Config); +log_to_binary(#{msg:={report,_},meta:=Meta}=Log,Config) -> + DefaultReportCb = fun logger:format_otp_report/1, + do_log_to_binary(Log#{meta=>Meta#{report_cb=>DefaultReportCb}},Config); +log_to_binary(Log,Config) -> + do_log_to_binary(Log,Config). + +do_log_to_binary(Log,Config) -> + {Formatter,FormatterConfig} = maps:get(formatter,Config), + String = try_format(Log,Formatter,FormatterConfig), + try unicode:characters_to_binary(String) + catch _:_ -> + ?LOG_INTERNAL(debug,[{formatter_error,Formatter}, + {config,FormatterConfig}, + {log,Log}, + {bad_return_value,String}]), + <<"FORMATTER ERROR: bad_return_value">> + end. + +try_format(Log,Formatter,FormatterConfig) -> + try Formatter:format(Log,FormatterConfig) + catch + C:R:S -> + ?LOG_INTERNAL(debug,[{formatter_crashed,Formatter}, + {config,FormatterConfig}, + {log,Log}, + {reason, + {C,R,logger:filter_stacktrace(?MODULE,S)}}]), + case {?DEFAULT_FORMATTER,?DEFAULT_FORMAT_CONFIG} of + {Formatter,FormatterConfig} -> + "DEFAULT FORMATTER CRASHED"; + {DefaultFormatter,DefaultConfig} -> + try_format(Log#{msg=>{"FORMATTER CRASH: ~tp", + [maps:get(msg,Log)]}}, + DefaultFormatter,DefaultConfig) + end + end. + +%%%----------------------------------------------------------------- +%%% Check that the configuration term is valid +check_common_config({toggle_sync_qlen,N}) when is_integer(N) -> + valid; +check_common_config({drop_new_reqs_qlen,N}) when is_integer(N) -> + valid; +check_common_config({flush_reqs_qlen,N}) when is_integer(N) -> + valid; +check_common_config({enable_burst_limit,Bool}) when Bool == true; + Bool == false -> + valid; +check_common_config({burst_limit_size,N}) when is_integer(N) -> + valid; +check_common_config({burst_window_time,N}) when is_integer(N) -> + valid; +check_common_config({enable_kill_overloaded,Bool}) when Bool == true; + Bool == false -> + valid; +check_common_config({handler_overloaded_qlen,N}) when is_integer(N) -> + valid; +check_common_config({handler_overloaded_mem,N}) when is_integer(N) -> + valid; +check_common_config({handler_restart_after,NorA}) when is_integer(NorA); + NorA == never -> + valid; +check_common_config({filesync_repeat_interval,NorA}) when is_integer(NorA); + NorA == no_repeat -> + valid; +check_common_config(_) -> + invalid. + + +%%%----------------------------------------------------------------- +%%% Overload Protection +call_cast_or_drop(Name, Bin) -> + %% If the handler process is getting overloaded, the log request + %% will be synchronous instead of asynchronous (slows down the + %% logging tempo of a process doing lots of logging. If the + %% handler is choked, drop mode is set and no request will be sent. + try ?get_mode(Name) of + async -> + gen_server:cast(Name, {log,Bin}); + sync -> + try gen_server:call(Name, {log,Bin}, ?DEFAULT_CALL_TIMEOUT) of + %% if return value from call == dropped, the + %% message has been flushed by handler and should + %% therefore not be counted as dropped in stats + ok -> ok; + dropped -> ok + catch + _:{timeout,_} -> + ?observe(Name,{dropped,1}) + end; + drop -> ?observe(Name,{dropped,1}) + catch + %% if the ETS table doesn't exist (maybe because of a + %% handler restart), we can only drop the request + _:_ -> ?observe(Name,{dropped,1}) + end, + ok. + +handler_exit(_Name, Reason) -> + exit(Reason). + +check_load(State = #{id:=Name, mode := Mode, + toggle_sync_qlen := ToggleSyncQLen, + drop_new_reqs_qlen := DropNewQLen, + flush_reqs_qlen := FlushQLen}) -> + {_,Mem} = process_info(self(), memory), + ?observe(Name,{max_mem,Mem}), + %% make sure the handler process doesn't get scheduled + %% out between the message_queue_len check below and the + %% action that follows (flush or write). + {_,QLen} = process_info(self(), message_queue_len), + ?observe(Name,{max_qlen,QLen}), + + {Mode1,_NewDrops,_NewFlushes} = + if + QLen >= FlushQLen -> + {flush, 0,1}; + QLen >= DropNewQLen -> + %% Note that drop mode will force log requests to + %% be dropped on the client side (never sent get to + %% the handler). + IncDrops = if Mode == drop -> 0; true -> 1 end, + {?change_mode(Name, Mode, drop), IncDrops,0}; + QLen >= ToggleSyncQLen -> + {?change_mode(Name, Mode, sync), 0,0}; + true -> + {?change_mode(Name, Mode, async), 0,0} + end, + State1 = ?update_other(drops,DROPS,_NewDrops,State), + {Mode1, QLen, Mem, + ?update_other(flushes,FLUSHES,_NewFlushes, + State1#{last_qlen => QLen})}. + +limit_burst(#{enable_burst_limit := false}) -> + {true,0,0}; +limit_burst(#{burst_win_ts := BurstWinT0, + burst_msg_count := BurstMsgCount, + burst_window_time := BurstWinTime, + burst_limit_size := BurstLimitSz}) -> + if (BurstMsgCount >= BurstLimitSz) -> + %% the limit for allowed messages has been reached + BurstWinT1 = ?timestamp(), + case ?diff_time(BurstWinT1,BurstWinT0) of + BurstCheckTime when BurstCheckTime < (BurstWinTime*1000) -> + %% we're still within the burst time frame + {false,BurstWinT0,BurstMsgCount}; + _BurstCheckTime -> + %% burst time frame passed, reset counters + {true,BurstWinT1,0} + end; + true -> + %% the limit for allowed messages not yet reached + {true,BurstWinT0,BurstMsgCount+1} + end. + +kill_if_choked(Name, QLen, Mem, + #{enable_kill_overloaded := KillIfOL, + handler_overloaded_qlen := HOLQLen, + handler_overloaded_mem := HOLMem}) -> + if KillIfOL andalso + ((QLen > HOLQLen) orelse (Mem > HOLMem)) -> + handler_exit(Name, {shutdown,{overloaded,Name,QLen,Mem}}); + true -> + ok + end. + +flush_log_requests() -> + flush_log_requests(-1). + +flush_log_requests(Limit) -> + process_flag(priority, high), + Flushed = flush_log_requests(0, Limit), + process_flag(priority, normal), + Flushed. + +flush_log_requests(Limit, Limit) -> + Limit; +flush_log_requests(N, Limit) -> + %% flush log requests but leave other requests, such as + %% file/disk_log_sync, info and change_config, so that these + %% have a chance to be processed even under heavy load + receive + {'$gen_cast',{log,_}} -> + flush_log_requests(N+1, Limit); + {'$gen_call',{Pid,MRef},{log,_}} -> + Pid ! {MRef, dropped}, + flush_log_requests(N+1, Limit) + after + 0 -> N + end. + +cancel_timer(TRef) when is_atom(TRef) -> ok; +cancel_timer(TRef) -> timer:cancel(TRef). + + +stop_or_restart(Name, {shutdown,Reason={overloaded,_Name,_QLen,_Mem}}, + #{handler_restart_after := RestartAfter}) -> + %% If we're terminating because of an overload situation (see + %% logger_h_common:kill_if_choked/4), we need to remove the handler + %% and set a restart timer. A separate process must perform this + %% in order to avoid deadlock. + HandlerPid = self(), + RemoveAndRestart = + fun() -> + MRef = erlang:monitor(process, HandlerPid), + receive + {'DOWN',MRef,_,_,_} -> + ok + after 30000 -> + error_notify(Reason), + exit(HandlerPid, kill) + end, + case logger:get_handler_config(Name) of + {ok,{HMod,HConfig}} when is_integer(RestartAfter) -> + _ = logger:remove_handler(Name), + _ = timer:apply_after(RestartAfter, logger, add_handler, + [Name,HMod,HConfig]); + {ok,_} -> + _ = logger:remove_handler(Name); + {error,CfgReason} when is_integer(RestartAfter) -> + error_notify({Name,restart_impossible,CfgReason}); + {error,_} -> + ok + end + end, + spawn(RemoveAndRestart), + ok; + +stop_or_restart(Name, shutdown, _State) -> + %% Probably terminated by supervisor. Remove the handler to avoid + %% error printouts due to failing handler. + _ = case logger:get_handler_config(Name) of + {ok,_} -> + %% Spawning to avoid deadlock + spawn(logger,remove_handler,[Name]); + _ -> + ok + end, + ok; + +stop_or_restart(_Name, _Reason, _State) -> + ok. + +overload_levels_ok(HandlerConfig) -> + TSQL = maps:get(toggle_sync_qlen, HandlerConfig, ?TOGGLE_SYNC_QLEN), + DNRQL = maps:get(drop_new_reqs_qlen, HandlerConfig, ?DROP_NEW_REQS_QLEN), + FRQL = maps:get(flush_reqs_qlen, HandlerConfig, ?FLUSH_REQS_QLEN), + (TSQL < DNRQL) andalso (DNRQL < FRQL). + +error_notify(Term) -> + ?internal_log(error, Term). + +info_notify(Term) -> + ?internal_log(info, Term). diff --git a/lib/kernel/src/logger_h_common.hrl b/lib/kernel/src/logger_h_common.hrl new file mode 100644 index 0000000000..89378dbb10 --- /dev/null +++ b/lib/kernel/src/logger_h_common.hrl @@ -0,0 +1,262 @@ + +%%%----------------------------------------------------------------- +%%% Overload protection configuration + +%%! *** NOTE *** +%%! It's important that: +%%! TOGGLE_SYNC_QLEN < DROP_NEW_REQS_QLEN < FLUSH_REQS_QLEN +%%! and that DROP_NEW_REQS_QLEN >= 2. +%%! Otherwise the handler could end up in drop mode with no new +%%! log requests to process. This would cause all future requests +%%! to be dropped (no switch to async mode would ever take place). + +%% This specifies the message_queue_len value where the log +%% requests switch from asynchronous casts to synchronous calls. +-define(TOGGLE_SYNC_QLEN, 10). +%% Above this message_queue_len, log requests will be dropped, +%% i.e. no log requests get sent to the handler process. +-define(DROP_NEW_REQS_QLEN, 200). +%% Above this message_queue_len, the handler process will flush +%% its mailbox and only leave this number of messages in it. +-define(FLUSH_REQS_QLEN, 1000). + +%% Never flush more than this number of messages in one go, +%% or the handler will be unresponsive for seconds (keep this +%% number as large as possible or the mailbox could grow large). +-define(FLUSH_MAX_N, 5000). + +%% BURST_LIMIT is the max number of log requests allowed to be +%% written within a BURST_WINDOW_TIME time frame. +-define(ENABLE_BURST_LIMIT, true). +-define(BURST_LIMIT_SIZE, 500). +-define(BURST_WINDOW_TIME, 1000). + +%% This enables/disables the feature to automatically get the +%% handler terminated if it gets too loaded (and can't keep up). +-define(ENABLE_KILL_OVERLOADED, false). +%% If the message_queue_len goes above this size even after +%% flushing has been performed, the handler is terminated. +-define(HANDLER_OVERLOADED_QLEN, 20000). +%% If the memory usage exceeds this level +-define(HANDLER_OVERLOADED_MEM, 3000000). + +%% This is the default time that the handler will wait before +%% restarting and accepting new requests. The value 'never' +%% disables restarts. +-define(HANDLER_RESTART_AFTER, 5000). +%%-define(HANDLER_RESTART_AFTER, never). + +%% The handler sends asynchronous write requests to the process +%% controlling the i/o device, but every once in this interval +%% will the write request be synchronous, so that the i/o device +%% process doesn't get overloaded. This gives the handler time +%% to keep up with its mailbox in overload situations, even if +%% the i/o is slow. +-define(CONTROLLER_SYNC_INTERVAL, 20). +%% The handler will not perform a file sync operation if the +%% mailbox size is greater than this number. This is to ensure +%% the handler process doesn't get overloaded while waiting for +%% an expensive file sync operation to finish. +-define(FILESYNC_OK_QLEN, 2). +%% Do a file/disk_log sync operation every integer() millisec +%% (if necessary) or set to 'no_repeat' to only do file sync when +%% the handler is idle. Note that file sync is not guaranteed to +%% happen automatically if this operation is disabled. +-define(FILESYNC_REPEAT_INTERVAL, 5000). +%%-define(FILESYNC_REPEAT_INTERVAL, no_repeat). + +%% This is the time after last message received that we think/hope +%% that the handler has an empty mailbox (no new log request has +%% come in). +-define(IDLE_DETECT_TIME_MSEC, 100). +-define(IDLE_DETECT_TIME_USEC, 100000). + +%% Default disk log option values +-define(DISK_LOG_TYPE, wrap). +-define(DISK_LOG_MAX_NO_FILES, 10). +-define(DISK_LOG_MAX_NO_BYTES, 1048576). + +%%%----------------------------------------------------------------- +%%% Overload protection macros + +-define(timestamp(), erlang:monotonic_time(microsecond)). + +-define(get_mode(HandlerName), + case ets:lookup(HandlerName, mode) of + [{mode,sync}] -> + case whereis(HandlerName)==self() of + true -> async; + _ -> sync + end; + [{mode,M}] -> M; + _ -> async + end). + +-define(set_mode(HandlerName, M), + begin ets:insert(HandlerName, {mode,M}), M end). + +-define(change_mode(HandlerName, M0, M1), + if M0 == M1 -> + M0; + true -> + ets:insert(HandlerName, {mode,M1}), + M1 + end). + +-define(min(X1, X2), + if X2 == undefined -> X1; + X2 < X1 -> X2; + true -> X1 + end). + +-define(max(X1, X2), + if + X2 == undefined -> X1; + X2 > X1 -> X2; + true -> X1 + end). + +-define(diff_time(OS_T1, OS_T0), OS_T1-OS_T0). + +%%%----------------------------------------------------------------- +%%% The test hook macros make it possible to observe and manipulate +%%% internal handler functionality. When enabled, these macros will +%%% slow down execution and therefore should not be include in code +%%% to be officially released. + +%% -define(TEST_HOOKS, true). +-ifdef(TEST_HOOKS). + -define(TEST_HOOKS_TAB, logger_h_test_hooks). + + -define(init_test_hooks(), + _ = case ets:whereis(?TEST_HOOKS_TAB) of + undefined -> ets:new(?TEST_HOOKS_TAB, [named_table,public]); + _ -> ok + end, + ets:insert(?TEST_HOOKS_TAB, {internal_log,{logger,internal_log}}), + ets:insert(?TEST_HOOKS_TAB, {file_write,ok}), + ets:insert(?TEST_HOOKS_TAB, {file_datasync,ok}), + ets:insert(?TEST_HOOKS_TAB, {disk_log_blog,ok}), + ets:insert(?TEST_HOOKS_TAB, {disk_log_sync,ok})). + + -define(set_internal_log(MOD_FUNC), + ets:insert(?TEST_HOOKS_TAB, {internal_log,MOD_FUNC})). + + -define(set_result(OPERATION, RESULT), + ets:insert(?TEST_HOOKS_TAB, {OPERATION,RESULT})). + + -define(set_defaults(), + ets:insert(?TEST_HOOKS_TAB, {internal_log,{logger,internal_log}}), + ets:insert(?TEST_HOOKS_TAB, {file_write,ok}), + ets:insert(?TEST_HOOKS_TAB, {file_datasync,ok}), + ets:insert(?TEST_HOOKS_TAB, {disk_log_blog,ok}), + ets:insert(?TEST_HOOKS_TAB, {disk_log_sync,ok})). + + -define(internal_log(TYPE, TERM), + try ets:lookup(?TEST_HOOKS_TAB, internal_log) of + [{_,{LMOD,LFUNC}}] -> apply(LMOD, LFUNC, [TYPE,TERM]); + _ -> logger:internal_log(TYPE, TERM) + catch _:_ -> logger:internal_log(TYPE, TERM) end). + + -define(file_write(DEVICE, DATA), + try ets:lookup(?TEST_HOOKS_TAB, file_write) of + [{_,ok}] -> file:write(DEVICE, DATA); + [{_,ERROR}] -> ERROR + catch _:_ -> file:write(DEVICE, DATA) end). + + -define(file_datasync(DEVICE), + try ets:lookup(?TEST_HOOKS_TAB, file_datasync) of + [{_,ok}] -> file:datasync(DEVICE); + [{_,ERROR}] -> ERROR + catch _:_ -> file:datasync(DEVICE) end). + + -define(disk_log_blog(LOG, DATA), + try ets:lookup(?TEST_HOOKS_TAB, disk_log_blog) of + [{_,ok}] -> disk_log:blog(LOG, DATA); + [{_,ERROR}] -> ERROR + catch _:_ -> disk_log:blog(LOG, DATA) end). + + -define(disk_log_sync(LOG), + try ets:lookup(?TEST_HOOKS_TAB, disk_log_sync) of + [{_,ok}] -> disk_log:sync(LOG); + [{_,ERROR}] -> ERROR + catch _:_ -> disk_log:sync(LOG) end). + + -define(DEFAULT_CALL_TIMEOUT, 5000). + +-else. % DEFAULTS! + -define(TEST_HOOKS_TAB, undefined). + -define(init_test_hooks(), ok). + -define(set_internal_log(_MOD_FUNC), ok). + -define(set_result(_OPERATION, _RESULT), ok). + -define(set_defaults(), ok). + -define(internal_log(TYPE, TERM), logger:internal_log(TYPE, TERM)). + -define(file_write(DEVICE, DATA), file:write(DEVICE, DATA)). + -define(file_datasync(DEVICE), file:datasync(DEVICE)). + -define(disk_log_blog(LOG, DATA), disk_log:blog(LOG, DATA)). + -define(disk_log_sync(LOG), disk_log:sync(LOG)). + -define(DEFAULT_CALL_TIMEOUT, 10000). +-endif. + +%%%----------------------------------------------------------------- +%%% These macros enable statistics counters in the state of the +%%% handler which is useful for analysing the overload protection +%%% behaviour. These counters should not be included in code to be +%%% officially released (as some counters will grow very large +%%% over time). + +%%-define(SAVE_STATS, true). +-ifdef(SAVE_STATS). + -define(merge_with_stats(STATE), + STATE#{flushes => 0, flushed => 0, drops => 0, + casts => 0, calls => 0, + max_qlen => 0, max_time => 0}). + + -define(update_max_qlen(QLEN, STATE), + begin #{max_qlen := QLEN0} = STATE, + STATE#{max_qlen => ?max(QLEN0,QLEN)} end). + + -define(update_calls_or_casts(CALL_OR_CAST, INC, STATE), + case CALL_OR_CAST of + cast -> + #{casts := CASTS0} = STATE, + STATE#{casts => CASTS0+INC}; + call -> + #{calls := CALLS0} = STATE, + STATE#{calls => CALLS0+INC} + end). + + -define(update_max_time(TIME, STATE), + begin #{max_time := TIME0} = STATE, + STATE#{max_time => ?max(TIME0,TIME)} end). + + -define(update_other(OTHER, VAR, INCVAL, STATE), + begin #{OTHER := VAR} = STATE, + STATE#{OTHER => VAR+INCVAL} end). + +-else. % DEFAULT! + -define(merge_with_stats(STATE), STATE). + -define(update_max_qlen(_QLEN, STATE), STATE). + -define(update_calls_or_casts(_CALL_OR_CAST, _INC, STATE), STATE). + -define(update_max_time(_TIME, STATE), STATE). + -define(update_other(_OTHER, _VAR, _INCVAL, STATE), STATE). +-endif. + +%%%----------------------------------------------------------------- +%%% These macros enable callbacks that make it possible to analyse +%%% the overload protection behaviour from outside the handler +%%% process (including dropped requests on the client side). +%%% An external callback module (?OBSERVER_MOD) is required which +%%% is not part of the kernel application. For this reason, these +%%% callbacks should not be included in code to be officially released. + +%%-define(OBSERVER_MOD, logger_test). +-ifdef(OBSERVER_MOD). + -define(start_observation(NAME), ?OBSERVER:start_observation(NAME)). + -define(observe(NAME,EVENT), ?OBSERVER:observe(NAME,EVENT)). + +-else. % DEFAULT! + -define(start_observation(_NAME), ok). + -define(observe(_NAME,_EVENT), ok). +-endif. +%%! <--- diff --git a/lib/kernel/src/logger_internal.hrl b/lib/kernel/src/logger_internal.hrl new file mode 100644 index 0000000000..82df499c2b --- /dev/null +++ b/lib/kernel/src/logger_internal.hrl @@ -0,0 +1,98 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-include_lib("kernel/include/logger.hrl"). +-define(LOGGER_TABLE,logger). +-define(LOGGER_KEY,'$logger_config$'). +-define(HANDLER_KEY,'$handler_config$'). +-define(LOGGER_META_KEY,'$logger_metadata$'). +-define(STANDARD_HANDLER, logger_std_h). +-define(DEFAULT_HANDLER_FILTERS, + ?DEFAULT_HANDLER_FILTERS([beam,erlang,otp])). +-define(DEFAULT_HANDLER_FILTERS(Domain), + [{remote_gl,{fun logger_filters:remote_gl/2,stop}}, + {domain,{fun logger_filters:domain/2,{log,prefix_of,Domain}}}, + {no_domain,{fun logger_filters:domain/2,{log,no_domain,[]}}}]). +-define(DEFAULT_FORMATTER,logger_formatter). +-define(DEFAULT_FORMAT_CONFIG,#{legacy_header=>true, + template=>?DEFAULT_FORMAT_TEMPLATE_HEADER}). +-define(DEFAULT_FORMAT_TEMPLATE_HEADER, + [{logger_formatter,header},"\n",msg,"\n"]). +-define(DEFAULT_FORMAT_TEMPLATE_SINGLE, + [time," ",level,": ",msg,"\n"]). +-define(DEFAULT_FORMAT_TEMPLATE, + [time," ",level,":\n",msg,"\n"]). + +-define(DEFAULT_LOGGER_CALL_TIMEOUT, infinity). + +-define(LOG_INTERNAL(Level,Report), + case logger:allow(Level,?MODULE) of + true -> + %% Spawn this to avoid deadlocks + _ = spawn(logger,macro_log,[?LOCATION,Level,Report, + logger:add_default_metadata(#{})]), + ok; + false -> + ok + end). + +%%%----------------------------------------------------------------- +%%% Levels +%%% Using same as syslog +-define(LEVELS,[emergency, + alert, + critical, + error, + warning, + notice, + info, + debug]). +-define(EMERGENCY,0). +-define(ALERT,1). +-define(CRITICAL,2). +-define(ERROR,3). +-define(WARNING,4). +-define(NOTICE,5). +-define(INFO,6). +-define(DEBUG,7). + +-define(IS_LEVEL(L), + (L=:=emergency orelse + L=:=alert orelse + L=:=critical orelse + L=:=error orelse + L=:=warning orelse + L=:=notice orelse + L=:=info orelse + L=:=debug)). + +-define(IS_MSG(Msg), + ((is_tuple(Msg) andalso tuple_size(Msg)==2) + andalso + (is_list(element(1,Msg)) andalso is_list(element(2,Msg))) + orelse + (element(1,Msg)==report andalso ?IS_REPORT(element(2,Msg))) + orelse + (element(1,Msg)==string andalso ?IS_STRING(element(2,Msg))))). + +-define(IS_REPORT(Report), + (is_map(Report) orelse (is_list(Report) andalso is_tuple(hd(Report))))). + +-define(IS_STRING(String), + (is_list(String) orelse is_binary(String))). diff --git a/lib/kernel/src/logger_server.erl b/lib/kernel/src/logger_server.erl new file mode 100644 index 0000000000..6ef3b8582a --- /dev/null +++ b/lib/kernel/src/logger_server.erl @@ -0,0 +1,440 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(logger_server). + +-behaviour(gen_server). + +%% API +-export([start_link/0, + add_handler/3, remove_handler/1, + add_filter/2, remove_filter/2, + set_module_level/2, reset_module_level/1, + cache_module_level/1, + set_config/2, set_config/3]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2]). + +-include("logger_internal.hrl"). + +-define(SERVER, logger). + +-record(state, {tid}). + +%%%=================================================================== +%%% API +%%%=================================================================== + +start_link() -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). + +add_handler(Id,Module,Config0) -> + case sanity_check(logger,handlers,[Id]) of + ok -> + try check_mod(Module) of + ok -> + case sanity_check(Id,Config0) of + ok -> + Default = default_config(Id), + Config = maps:merge(Default,Config0), + call({add_handler,Id,Module,Config}); + Error -> + Error + end + catch throw:Error -> + {error,Error} + end; + Error -> + Error + end. + +remove_handler(HandlerId) -> + call({remove_handler,HandlerId}). + +add_filter(Owner,Filter) -> + case sanity_check(Owner,filters,[Filter]) of + ok -> call({add_filter,Owner,Filter}); + Error -> Error + end. + +remove_filter(Owner,FilterId) -> + call({remove_filter,Owner,FilterId}). + +set_module_level(Module,Level) when is_atom(Module) -> + case sanity_check(logger,level,Level) of + ok -> call({set_module_level,Module,Level}); + Error -> Error + end; +set_module_level(Module,_) -> + {error,{not_a_module,Module}}. + +reset_module_level(Module) when is_atom(Module) -> + call({reset_module_level,Module}); +reset_module_level(Module) -> + {error,{not_a_module,Module}}. + +cache_module_level(Module) -> + gen_server:cast(?SERVER,{cache_module_level,Module}). + + +set_config(Owner,Key,Value) -> + case sanity_check(Owner,Key,Value) of + ok -> call({update_config,Owner,#{Key=>Value}}); + Error -> Error + end. + +set_config(Owner,Config0) -> + case sanity_check(Owner,Config0) of + ok -> + Config = maps:merge(default_config(Owner),Config0), + call({set_config,Owner,Config}); + Error -> + Error + end. + +%%%=================================================================== +%%% gen_server callbacks +%%%=================================================================== + +init([]) -> + process_flag(trap_exit, true), + Tid = logger_config:new(?LOGGER_TABLE), + LoggerConfig = maps:merge(default_config(logger), + #{handlers=>[logger_simple]}), + logger_config:create(Tid,logger,LoggerConfig), + SimpleConfig0 = maps:merge(default_config(logger_simple), + #{filter_default=>stop, + filters=>?DEFAULT_HANDLER_FILTERS, + logger_simple=>#{buffer=>true}}), + %% If this fails, then the node should crash + {ok,SimpleConfig} = + logger_simple:adding_handler(logger_simple,SimpleConfig0), + logger_config:create(Tid,logger_simple,logger_simple,SimpleConfig), + {ok, #state{tid=Tid}}. + +handle_call({add_handler,Id,Module,HConfig}, _From, #state{tid=Tid}=State) -> + Reply = + case logger_config:exist(Tid,Id) of + true -> + {error,{already_exist,Id}}; + false -> + %% inform the handler + case call_h(Module,adding_handler,[Id,HConfig],{ok,HConfig}) of + {ok,HConfig1} -> + logger_config:create(Tid,Id,Module,HConfig1), + {ok,Config} = do_get_config(Tid,logger), + Handlers = maps:get(handlers,Config,[]), + do_set_config(Tid,logger, + Config#{handlers=>[Id|Handlers]}), + ok; + {error,HReason} -> + {error,{handler_not_added,HReason}} + end + end, + {reply,Reply,State}; +handle_call({remove_handler,HandlerId}, _From, #state{tid=Tid}=State) -> + Reply = + case logger_config:get(Tid,HandlerId) of + {ok,{Module,_}} -> + {ok,Config} = do_get_config(Tid,logger), + Handlers0 = maps:get(handlers,Config,[]), + Handlers = lists:delete(HandlerId,Handlers0), + %% inform the handler + _ = call_h(Module,removing_handler,[HandlerId],ok), + do_set_config(Tid,logger,Config#{handlers=>Handlers}), + logger_config:delete(Tid,HandlerId), + ok; + _ -> + {error,{not_found,HandlerId}} + end, + {reply,Reply,State}; +handle_call({add_filter,Id,Filter}, _From,#state{tid=Tid}=State) -> + Reply = do_add_filter(Tid,Id,Filter), + {reply,Reply,State}; +handle_call({remove_filter,Id,FilterId}, _From, #state{tid=Tid}=State) -> + Reply = do_remove_filter(Tid,Id,FilterId), + {reply,Reply,State}; +handle_call({update_config,Id,NewConfig}, _From, #state{tid=Tid}=State) -> + Reply = + case logger_config:get(Tid,Id) of + {ok,{Module,OldConfig}} -> + Config = maps:merge(OldConfig,NewConfig), + case call_h(Module,changing_config,[Id,OldConfig,Config], + {ok,Config}) of + {ok,Config1} -> + do_set_config(Tid,Id,Config1); + Error -> + Error + end; + {ok,OldConfig} -> + Config = maps:merge(OldConfig,NewConfig), + do_set_config(Tid,Id,Config); + Error -> + Error + end, + {reply,Reply,State}; +handle_call({set_config,logger,Config}, _From, #state{tid=Tid}=State) -> + Reply = do_set_config(Tid,logger,Config), + {reply,Reply,State}; +handle_call({set_config,HandlerId,Config}, _From, #state{tid=Tid}=State) -> + Reply = + case logger_config:get(Tid,HandlerId) of + {ok,{Module,OldConfig}} -> + case call_h(Module,changing_config,[HandlerId,OldConfig,Config], + {ok,Config}) of + {ok,Config1} -> + do_set_config(Tid,HandlerId,Config1); + Error -> + Error + end; + _ -> + {error,{not_found,HandlerId}} + end, + {reply,Reply,State}; +handle_call({set_module_level,Module,Level}, _From, #state{tid=Tid}=State) -> + Reply = logger_config:set_module_level(Tid,Module,Level), + {reply,Reply,State}; +handle_call({reset_module_level,Module}, _From, #state{tid=Tid}=State) -> + Reply = logger_config:reset_module_level(Tid,Module), + {reply,Reply,State}. + +handle_cast({cache_module_level,Module}, #state{tid=Tid}=State) -> + logger_config:cache_module_level(Tid,Module), + {noreply, State}. + +%% Interface for those who can't call the API - e.g. the emulator, or +%% places related to code loading. +%% +%% This can also be log events from remote nodes which are sent from +%% logger.erl when the group leader of the client process is on a +%% same node as the client process itself. +handle_info({log,Level,Format,Args,Meta}, State) -> + logger:log(Level,Format,Args,Meta), + {noreply, State}; +handle_info({log,Level,Report,Meta}, State) -> + logger:log(Level,Report,Meta), + {noreply, State}; +handle_info({Ref,_Reply},State) when is_reference(Ref) -> + %% Assuming this is a timed-out gen_server reply - ignoring + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== +call(Request) -> + case whereis(?SERVER) of + Pid when Pid==self() -> + {error,{attempting_syncronous_call_to_self,Request}}; + _ -> + gen_server:call(?SERVER,Request,?DEFAULT_LOGGER_CALL_TIMEOUT) + end. + +do_add_filter(Tid,Id,{FId,_} = Filter) -> + case do_get_config(Tid,Id) of + {ok,Config} -> + Filters = maps:get(filters,Config,[]), + case lists:keymember(FId,1,Filters) of + true -> + {error,{already_exist,FId}}; + false -> + do_set_config(Tid,Id,Config#{filters=>[Filter|Filters]}) + end; + Error -> + Error + end. + +do_remove_filter(Tid,Id,FilterId) -> + case do_get_config(Tid,Id) of + {ok,Config} -> + Filters0 = maps:get(filters,Config,[]), + case lists:keytake(FilterId,1,Filters0) of + {value,_,Filters} -> + do_set_config(Tid,Id,Config#{filters=>Filters}); + false -> + {error,{not_found,FilterId}} + end; + Error -> + Error + end. + +do_get_config(Tid,Id) -> + case logger_config:get(Tid,Id) of + {ok,{_,Config}} -> + {ok,Config}; + {ok,Config} -> + {ok,Config}; + Error -> + Error + end. + +do_set_config(Tid,Id,Config) -> + logger_config:set(Tid,Id,Config), + ok. + +default_config(logger) -> + #{level=>info, + filters=>[], + filter_default=>log, + handlers=>[]}; +default_config(_) -> + #{level=>info, + filters=>[], + filter_default=>log, + formatter=>{?DEFAULT_FORMATTER,?DEFAULT_FORMAT_CONFIG}}. + +sanity_check(Owner,Key,Value) -> + sanity_check_1(Owner,[{Key,Value}]). + +sanity_check(HandlerId,Config) when is_map(Config) -> + sanity_check_1(HandlerId,maps:to_list(Config)); +sanity_check(_,Config) -> + {error,{invalid_handler_config,Config}}. + +sanity_check_1(Owner,Config) when is_list(Config) -> + try + Type = get_type(Owner), + check_config(Type,Config) + catch throw:Error -> {error,Error} + end. + +get_type(logger) -> + logger; +get_type(Id) -> + check_id(Id), + handler. + +check_config(Owner,[{level,Level}|Config]) -> + check_level(Level), + check_config(Owner,Config); +check_config(logger,[{handlers,Handlers}|Config]) -> + check_handlers(Handlers), + check_config(logger,Config); +check_config(Owner,[{filters,Filters}|Config]) -> + check_filters(Filters), + check_config(Owner,Config); +check_config(Owner,[{filter_default,FD}|Config]) -> + check_filter_default(FD), + check_config(Owner,Config); +check_config(handler,[{formatter,Formatter}|Config]) -> + check_formatter(Formatter), + check_config(handler,Config); +check_config(logger,[C|_]) -> + throw({invalid_logger_config,C}); +check_config(handler,[{_,_}|Config]) -> + %% Arbitrary config elements are allowed for handlers + check_config(handler,Config); +check_config(_,[]) -> + ok. + +check_id(Id) when is_atom(Id) -> + ok; +check_id(Id) -> + throw({invalid_id,Id}). + +check_mod(Mod) when is_atom(Mod) -> + ok; +check_mod(Mod) -> + throw({invalid_module,Mod}). + +check_level({LevelInt,cached}) when LevelInt>=?EMERGENCY, LevelInt=<?DEBUG -> + ok; +check_level(Level) -> + case lists:member(Level,?LEVELS) of + true -> + ok; + false -> + throw({invalid_level,Level}) + end. + +check_handlers([Id|Handlers]) -> + check_id(Id), + check_handlers(Handlers); +check_handlers([]) -> + ok; +check_handlers(Handlers) -> + throw({invalid_handlers,Handlers}). + +check_filters([{Id,{Fun,_Args}}|Filters]) when is_atom(Id), is_function(Fun,2) -> + check_filters(Filters); +check_filters([Filter|_]) -> + throw({invalid_filter,Filter}); +check_filters([]) -> + ok; +check_filters(Filters) -> + throw({invalid_filters,Filters}). + +check_filter_default(FD) when FD==stop; FD==log -> + ok; +check_filter_default(FD) -> + throw({invalid_filter_default,FD}). + +check_formatter({logger_formatter,Config}) when is_map(Config) -> + check_logger_formatter_config(maps:to_list(Config)); +check_formatter({logger_formatter,Config}) -> + throw({invalid_formatter_config,Config}); +check_formatter({Mod,_}) -> + %% no knowledge of other formatters + check_mod(Mod); +check_formatter(Formatter) -> + throw({invalid_formatter,Formatter}). + + +check_logger_formatter_config([{template,T}|Config]) when is_list(T) -> + case lists:all(fun(X) when is_atom(X) -> true; + (X) when is_tuple(X), is_atom(element(1,X)) -> true; + (X) when is_list(X) -> io_lib:printable_unicode_list(X); + (_) -> false + end, + T) of + true -> + check_logger_formatter_config(Config); + false -> + throw({invalid_formatter_template,T}) + end; +check_logger_formatter_config([{legacy_header,LH}|Config]) when is_boolean(LH) -> + check_logger_formatter_config(Config); +check_logger_formatter_config([{single_line,SL}|Config]) when is_boolean(SL) -> + check_logger_formatter_config(Config); +check_logger_formatter_config([{utc,Utc}|Config]) when is_boolean(Utc) -> + check_logger_formatter_config(Config); +check_logger_formatter_config([C|_]) -> + throw({invalid_formatter_config,C}); +check_logger_formatter_config([]) -> + ok. + +call_h(Module, Function, Args, DefRet) -> + %% Not calling code:ensure_loaded + erlang:function_exported here, + %% since in some rare terminal cases, the code_server might not + %% exist and we'll get a deadlock in removing the handler. + try apply(Module, Function, Args) + catch + C:R:S -> + case {C,R,S} of + {error,undef,[{Module,Function,Args,_}|_]} -> + DefRet; + _ -> + {error,{callback_crashed, + {C,R,logger:filter_stacktrace(?MODULE,S)}}} + end + end. diff --git a/lib/kernel/src/logger_simple.erl b/lib/kernel/src/logger_simple.erl new file mode 100644 index 0000000000..23ff6ccd2e --- /dev/null +++ b/lib/kernel/src/logger_simple.erl @@ -0,0 +1,236 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(logger_simple). + +-export([adding_handler/2, removing_handler/1, log/2]). +-export([get_buffer/0]). + +%% This module implements a simple handler for logger. It is the +%% default used during system start. + +%%%----------------------------------------------------------------- +%%% API +get_buffer() -> + case whereis(?MODULE) of + undefined -> + {error,noproc}; + Pid -> + Ref = erlang:monitor(process,Pid), + Pid ! {get_buffer,self()}, + receive + {buffer,Buffer} -> + erlang:demonitor(Ref,[flush]), + {ok,Buffer}; + {'DOWN',Ref,process,Pid,Reason} -> + {error,Reason} + end + end. + +%%%----------------------------------------------------------------- +%%% Logger callback + +adding_handler(?MODULE,Config) -> + Me = self(), + case whereis(?MODULE) of + undefined -> + {Pid,Ref} = spawn_opt(fun() -> init(Me,Config) end, + [link,monitor,{message_queue_data,off_heap}]), + receive + {'DOWN',Ref,process,Pid,Reason} -> + {error,Reason}; + {Pid,started} -> + erlang:demonitor(Ref), + {ok,Config} + end; + _ -> + {error,{handler_process_name_already_exists,?MODULE}} + end. + +removing_handler(?MODULE) -> + case whereis(?MODULE) of + undefined -> + ok; + Pid -> + Ref = erlang:monitor(process,Pid), + unlink(Pid), + Pid ! stop, + receive {'DOWN',Ref,process,Pid,_} -> + ok + end + end. + +log(#{meta:=#{error_logger:=#{tag:=info_report,type:=Type}}},_Config) + when Type=/=std_info -> + %% Skip info reports that are not 'std_info' (ref simple logger in + %% error_logger) + ok; +log(#{msg:=_,meta:=#{time:=_}}=Log,_Config) -> + _ = case whereis(?MODULE) of + undefined -> + %% Is the node on the way down? Real emergency? + %% Log directly from client just to get it out + do_log( + #{level=>error, + msg=>{report,{error,simple_handler_process_dead}}, + meta=>#{time=>erlang:monotonic_time(microsecond)}}), + do_log(Log); + _ -> + ?MODULE ! {log,Log} + end, + ok; +log(_,_) -> + %% Unexpected log. + %% We don't want to crash the simple logger, so ignore this. + ok. + +%%%----------------------------------------------------------------- +%%% Process +init(Starter,Config) -> + register(?MODULE,self()), + Starter ! {self(),started}, + BufferSize = + case Config of + #{?MODULE:=#{buffer:=true}} -> + 10; + _ -> + infinity + end, + loop(#{buffer_size=>BufferSize,dropped=>0,buffer=>[]},infinity). + +loop(Buffer,Timeout) -> + receive + stop -> + ok; + {get_buffer,From} -> + loop(Buffer#{send_to=>From},0); + {log,#{msg:=_,meta:=#{time:=_}}=Log} -> + do_log(Log), + loop(update_buffer(Buffer,Log),Timeout); + _ -> + %% Unexpected message - flush it! + loop(Buffer,Timeout) + after Timeout -> + #{dropped:=D,buffer:=B,send_to:=Pid} = Buffer, + LogList = lists:reverse(B) ++ drop_msg(D), + Pid ! {buffer,LogList}, + loop(Buffer#{buffer_size=>infinity, + dropped=>0, + buffer=>[], + send_to=>false}, + infinity) + end. + +update_buffer(#{buffer_size:=infinity}=Buffer,_Log) -> + Buffer; +update_buffer(#{buffer_size:=0,dropped:=D}=Buffer,_Log) -> + Buffer#{dropped=>D+1}; +update_buffer(#{buffer_size:=S,buffer:=B}=Buffer,Log) -> + Buffer#{buffer_size=>S-1,buffer=>[Log|B]}. + +drop_msg(0) -> + []; +drop_msg(N) -> + [#{level=>info, + msg=>{"Simple handler buffer full, dropped ~w messages",[N]}, + meta=>#{time=>erlang:monotonic_time(microsecond)}}]. + +%%%----------------------------------------------------------------- +%%% Internal + +%% Can't do io_lib:format + +do_log(#{msg:={report,Report}, + meta:=#{time:=T,error_logger:=#{type:=Type}}}) -> + display_date(T), + display_report(Type,Report); +do_log(#{msg:=Msg,meta:=#{time:=T}}) -> + display_date(T), + display(Msg). + +display_date(Timestamp0) when is_integer(Timestamp0) -> + Timestamp = Timestamp0 + erlang:time_offset(microsecond), + Micro = Timestamp rem 1000000, + Sec = Timestamp div 1000000, + {{Y,Mo,D},{H,Mi,S}} = erlang:universaltime_to_localtime( + erlang:posixtime_to_universaltime(Sec)), + erlang:display_string( + integer_to_list(Y) ++ "-" ++ + pad(Mo,2) ++ "-" ++ + pad(D,2) ++ " " ++ + pad(H,2) ++ ":" ++ + pad(Mi,2) ++ ":" ++ + pad(S,2) ++ "." ++ + pad(Micro,6) ++ " "). + +pad(Int,Size) when is_integer(Int) -> + pad(integer_to_list(Int),Size); +pad(Str,Size) when length(Str)==Size -> + Str; +pad(Str,Size) -> + pad([$0|Str],Size). + +display({string,Chardata}) -> + try unicode:characters_to_list(Chardata) of + String -> erlang:display_string(String), erlang:display_string("\n") + catch _:_ -> erlang:display(Chardata) + end; +display({report,Report}) when is_map(Report) -> + display_report(maps:to_list(Report)); +display({report,Report}) -> + display_report(Report); +display({F, A}) when is_list(F), is_list(A) -> + erlang:display_string(F ++ "\n"), + [begin + erlang:display_string("\t"), + erlang:display(Arg) + end || Arg <- A], + ok. + +display_report(Atom, A) when is_atom(Atom) -> + %% The widest atom seems to be 'supervisor_report' at 17. + ColumnWidth = 20, + AtomString = atom_to_list(Atom), + AtomLength = length(AtomString), + Padding = lists:duplicate(ColumnWidth - AtomLength, $\s), + erlang:display_string(AtomString ++ Padding), + display_report(A); +display_report(F, A) -> + erlang:display({F, A}). + +display_report([A, []]) -> + %% Special case for crash reports when process has no links + display_report(A); +display_report(A = [_|_]) -> + case lists:all(fun({Key,_Value}) -> is_atom(Key); (_) -> false end, A) of + true -> + erlang:display_string("\n"), + lists:foreach( + fun({Key, Value}) -> + erlang:display_string( + " " ++ + atom_to_list(Key) ++ + ": "), + erlang:display(Value) + end, A); + false -> + erlang:display(A) + end; +display_report(A) -> + erlang:display(A). diff --git a/lib/kernel/src/logger_std_h.erl b/lib/kernel/src/logger_std_h.erl new file mode 100644 index 0000000000..cbc9db372c --- /dev/null +++ b/lib/kernel/src/logger_std_h.erl @@ -0,0 +1,799 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(logger_std_h). + +-behaviour(gen_server). + +-include("logger.hrl"). +-include("logger_internal.hrl"). +-include("logger_h_common.hrl"). + +-include_lib("kernel/include/file.hrl"). + +%% API +-export([start_link/3, info/1, filesync/1, reset/1]). + +%% gen_server and proc_lib callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +%% logger callbacks +-export([log/2, adding_handler/2, removing_handler/1, + changing_config/3, swap_buffer/2]). + +%%%=================================================================== +%%% API +%%%=================================================================== + +%%%----------------------------------------------------------------- +%%% Start a standard handler process and link to caller. +%%% This function is called by the kernel supervisor when this +%%% handler process gets added +-spec start_link(Name, Config, HandlerState) -> {ok,Pid} | {error,Reason} when + Name :: atom(), + Config :: logger:config(), + HandlerState :: map(), + Pid :: pid(), + Reason :: term(). + +start_link(Name, Config, HandlerState) -> + proc_lib:start_link(?MODULE,init,[[Name,Config,HandlerState]]). + +%%%----------------------------------------------------------------- +%%% +-spec filesync(Name) -> ok | {error,Reason} when + Name :: atom(), + Reason :: handler_busy | {badarg,term()}. + +filesync(Name) when is_atom(Name) -> + try + gen_server:call(Name, filesync, ?DEFAULT_CALL_TIMEOUT) + catch + _:{timeout,_} -> {error,handler_busy} + end; +filesync(Name) -> + {error,{badarg,{filesync,[Name]}}}. + +%%%----------------------------------------------------------------- +%%% +-spec info(Name) -> Info | {error,Reason} when + Name :: atom(), + Info :: term(), + Reason :: handler_busy | {badarg,term()}. + +info(Name) when is_atom(Name) -> + try + gen_server:call(Name, info, ?DEFAULT_CALL_TIMEOUT) + catch + _:{timeout,_} -> {error,handler_busy} + end; +info(Name) -> + {error,{badarg,{info,[Name]}}}. + +%%%----------------------------------------------------------------- +%%% +-spec reset(Name) -> ok | {error,Reason} when + Name :: atom(), + Reason :: handler_busy | {badarg,term()}. + +reset(Name) when is_atom(Name) -> + try + gen_server:call(Name, reset, ?DEFAULT_CALL_TIMEOUT) + catch + _:{timeout,_} -> {error,handler_busy} + end; +reset(Name) -> + {error,{badarg,{reset,[Name]}}}. + + +%%%=================================================================== +%%% logger callbacks +%%%=================================================================== + +%%%----------------------------------------------------------------- +%%% Handler being added +adding_handler(Name, Config) -> + case check_config(adding, Name, Config) of + {ok, Config1} -> + %% create initial handler state by merging defaults with config + HConfig = maps:get(?MODULE, Config1, #{}), + HState = maps:merge(get_init_state(), HConfig), + case logger_h_common:overload_levels_ok(HState) of + true -> + case start(Name, Config1, HState) of + ok -> + %% Make sure wait_for_buffer is not stored, so we + %% won't hang and wait for buffer on a restart + {ok, maps:remove(wait_for_buffer,Config1)}; + Error -> + Error + end; + false -> + #{toggle_sync_qlen := TSQL, + drop_new_reqs_qlen := DNRQL, + flush_reqs_qlen := FRQL} = HState, + {error,{invalid_levels,{TSQL,DNRQL,FRQL}}} + end; + Error -> + Error + end. + +%%%----------------------------------------------------------------- +%%% Updating handler config +changing_config(Name, + OldConfig=#{id:=Id, ?MODULE:=#{type:=Type}}, + NewConfig=#{id:=Id}) -> + MyConfig = maps:get(?MODULE, NewConfig, #{}), + case maps:get(type, MyConfig, Type) of + Type -> + MyConfig1 = MyConfig#{type=>Type}, + changing_config1(Name, OldConfig, + NewConfig#{?MODULE=>MyConfig1}); + _ -> + {error,{illegal_config_change,OldConfig,NewConfig}} + end; +changing_config(_Name, OldConfig, NewConfig) -> + {error,{illegal_config_change,OldConfig,NewConfig}}. + +changing_config1(Name, OldConfig, NewConfig) -> + case check_config(changing, Name, NewConfig) of + Result = {ok,NewConfig1} -> + try gen_server:call(Name, {change_config,OldConfig,NewConfig1}, + ?DEFAULT_CALL_TIMEOUT) of + ok -> Result; + HError -> HError + catch + _:{timeout,_} -> {error,handler_busy} + end; + Error -> + Error + end. + +check_config(adding, Name, Config0) -> + %% Merge in defaults on top level + Config = maps:merge(#{id => Name}, Config0), + %% Merge in defaults on handler level + MyConfig0 = maps:get(?MODULE, Config, #{}), + MyConfig = maps:merge(#{type => standard_io}, + MyConfig0), + case check_my_config(maps:to_list(MyConfig)) of + ok -> + {ok,Config#{?MODULE=>MyConfig}}; + Error -> + Error + end; +check_config(changing, _Name, Config) -> + MyConfig = maps:get(?MODULE, Config, #{}), + case check_my_config(maps:to_list(MyConfig)) of + ok -> {ok,Config}; + Error -> Error + end. + +check_my_config([{type,Type} | Config]) when Type == standard_io; + Type == standard_error -> + check_my_config(Config); +check_my_config([{type,{file,File}} | Config]) when is_list(File) -> + check_my_config(Config); +check_my_config([{type,{file,File,Modes}} | Config]) when is_list(File), + is_list(Modes) -> + check_my_config(Config); +check_my_config([Other | Config]) -> + case logger_h_common:check_common_config(Other) of + valid -> + check_my_config(Config); + invalid -> + {error,{invalid_config,?MODULE,Other}} + end; +check_my_config([]) -> + ok. + + +%%%----------------------------------------------------------------- +%%% Handler being removed +removing_handler(Name) -> + stop(Name). + +%%%----------------------------------------------------------------- +%%% Get buffer when swapping from simple handler +swap_buffer(Name,Buffer) -> + case whereis(Name) of + undefined -> + ok; + _ -> + Name ! {buffer,Buffer} + end. + +%%%----------------------------------------------------------------- +%%% Log a string or report +-spec log(Log, Config) -> ok | dropped when + Log :: logger:log(), + Config :: logger:config(). + +log(Log,Config=#{id:=Name}) -> + %% if the handler has crashed, we must drop this request + %% and hope the handler restarts so we can try again + true = is_pid(whereis(Name)), + Bin = logger_h_common:log_to_binary(Log,Config), + logger_h_common:call_cast_or_drop(Name, Bin). + +%%%=================================================================== +%%% gen_server callbacks +%%%=================================================================== + +init([Name, Config, + State0 = #{type := Type, file_ctrl_sync_int := FileCtrlSyncInt}]) -> + register(Name, self()), + process_flag(trap_exit, true), + process_flag(message_queue_data, off_heap), + + ?init_test_hooks(), + ?start_observation(Name), + + case do_init(Name, Type) of + {ok,InitState} -> + catch ets:new(Name, [public, named_table]), + ?set_mode(Name, async), + State = maps:merge(State0, InitState), + T0 = ?timestamp(), + State1 = + ?merge_with_stats(State#{mode => async, + file_ctrl_sync => FileCtrlSyncInt, + last_qlen => 0, + last_log_ts => T0, + burst_win_ts => T0, + burst_msg_count => 0}), + proc_lib:init_ack({ok,self()}), + gen_server:cast(self(), {repeated_filesync,T0}), + enter_loop(Config, State1); + Error -> + logger_h_common:error_notify({init_handler,Name,Error}), + proc_lib:init_ack(Error) + end. + +do_init(Name, Std) when Std=:=standard_io; Std=:=standard_error -> + case open_log_file(Name, Std) of + {ok,FileCtrlPid} -> + {ok,#{id=>Name,type=>Std,file_ctrl_pid=>FileCtrlPid}}; + Error -> + Error + end; +do_init(Name, FileInfo) when is_tuple(FileInfo) -> + case open_log_file(Name, FileInfo) of + {ok,FileCtrlPid} -> + {ok,#{id=>Name,type=>FileInfo,file_ctrl_pid=>FileCtrlPid}}; + Error -> + Error + end. + +enter_loop(#{wait_for_buffer:=true}=Config,State) -> + State1 = + receive + {buffer,Buffer} -> + lists:foldl( + fun(Log,S) -> + Bin = logger_h_common:log_to_binary(Log,Config), + {_,S1} = do_log(Bin,cast,S), + S1 + end, + State, + Buffer) + end, + gen_server:enter_loop(?MODULE,[],State1); +enter_loop(_Config,State) -> + gen_server:enter_loop(?MODULE,[],State). + +%% This is the synchronous log request. +handle_call({log, Bin}, _From, State) -> + {Result,State1} = do_log(Bin, call, State), + %% Result == ok | dropped + {reply,Result, State1}; + +handle_call(filesync, _From, State = #{type := Type, + file_ctrl_pid := FileCtrlPid}) -> + if is_atom(Type) -> + {reply, ok, State}; + true -> + {reply, file_ctrl_filesync_sync(FileCtrlPid), State} + end; + +handle_call({change_config,_OldConfig,NewConfig}, _From, + State = #{filesync_repeat_interval := FSyncInt0, + last_log_ts := LastLogTS}) -> + HConfig = maps:get(?MODULE, NewConfig, #{}), + State1 = maps:merge(State, HConfig), + case logger_h_common:overload_levels_ok(State1) of + true -> + _ = + case maps:get(filesync_repeat_interval, HConfig, undefined) of + undefined -> + ok; + no_repeat -> + _ = logger_h_common:cancel_timer(maps:get(rep_sync_tref, + State, + undefined)); + FSyncInt0 -> + ok; + _FSyncInt1 -> + _ = logger_h_common:cancel_timer(maps:get(rep_sync_tref, + State, + undefined)), + gen_server:cast(self(), {repeated_filesync, + LastLogTS}) + end, + {reply, ok, State1}; + false -> + #{toggle_sync_qlen := TSQL, + drop_new_reqs_qlen := DNRQL, + flush_reqs_qlen := FRQL} = State1, + {reply, {error,{invalid_levels,{TSQL,DNRQL,FRQL}}}, State} + end; + +handle_call(info, _From, State) -> + {reply, State, State}; + +handle_call(reset, _From, State) -> + State1 = ?merge_with_stats(State), + {reply, ok, State1#{last_qlen => 0, + last_log_ts => ?timestamp()}}; + +handle_call(stop, _From, State) -> + {stop, {shutdown,stopped}, ok, State}. + +%% This is the asynchronous log request. +handle_cast({log, Bin}, State) -> + {_,State1} = do_log(Bin, cast, State), + {noreply, State1}; + +%% If FILESYNC_REPEAT_INTERVAL is set to a millisec value, this +%% clause gets called repeatedly by the handler. In order to +%% guarantee that a filesync *always* happens after the last log +%% request, the repeat operation must be active! +handle_cast({repeated_filesync,LastLogTS0}, + State = #{type := Type, + file_ctrl_pid := FileCtrlPid, + filesync_repeat_interval := FSyncInt, + last_log_ts := LastLogTS1}) -> + State1 = + if not is_atom(Type), is_integer(FSyncInt) -> + %% only do filesync if something has been + %% written since last time we checked + if LastLogTS1 == LastLogTS0 -> + ok; + true -> + file_ctrl_filesync_async(FileCtrlPid) + end, + {ok,TRef} = + timer:apply_after(FSyncInt, gen_server,cast, + [self(),{repeated_filesync,LastLogTS1}]), + State#{rep_sync_tref => TRef}; + true -> + State + end, + {noreply,State1}. + +handle_info({'EXIT',Pid,Why}, State = #{id := Name, type := FileInfo}) -> + case maps:get(file_ctrl_pid, State, undefined) of + Pid -> + %% file error, terminate handler + logger_h_common:handler_exit(Name, + {error,{write_failed,FileInfo,Why}}); + _Other -> + %% ignore EXIT + ok + end, + {noreply, State}; + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(Reason, State = #{id:=Name, file_ctrl_pid:=FWPid, + type:=_FileInfo}) -> + _ = logger_h_common:cancel_timer(maps:get(rep_sync_tref, State, + undefined)), + case is_process_alive(FWPid) of + true -> + unlink(FWPid), + _ = file_ctrl_stop(FWPid), + MRef = erlang:monitor(process, FWPid), + receive + {'DOWN',MRef,_,_,_} -> + ok + after + ?DEFAULT_CALL_TIMEOUT -> + exit(FWPid, kill) + end; + false -> + ok + end, + logger_h_common:stop_or_restart(Name, Reason, State). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== + +%%%----------------------------------------------------------------- +%%% +get_init_state() -> + #{toggle_sync_qlen => ?TOGGLE_SYNC_QLEN, + drop_new_reqs_qlen => ?DROP_NEW_REQS_QLEN, + flush_reqs_qlen => ?FLUSH_REQS_QLEN, + enable_burst_limit => ?ENABLE_BURST_LIMIT, + burst_limit_size => ?BURST_LIMIT_SIZE, + burst_window_time => ?BURST_WINDOW_TIME, + enable_kill_overloaded => ?ENABLE_KILL_OVERLOADED, + handler_overloaded_qlen => ?HANDLER_OVERLOADED_QLEN, + handler_overloaded_mem => ?HANDLER_OVERLOADED_MEM, + handler_restart_after => ?HANDLER_RESTART_AFTER, + file_ctrl_sync_int => ?CONTROLLER_SYNC_INTERVAL, + filesync_ok_qlen => ?FILESYNC_OK_QLEN, + filesync_repeat_interval => ?FILESYNC_REPEAT_INTERVAL}. + +%%%----------------------------------------------------------------- +%%% Add a standard handler to the logger. +%%% This starts a dedicated handler process which should always +%%% exist if the handler is registered with logger (and should not +%%% exist if the handler is not registered). +%%% +%%% Handler specific config should be provided with a sub map associated +%%% with a key named the same as this module, e.g: +%%% +%%% Config = #{logger_std_h => #{toggle_sync_qlen => 50} +%%% +%%% The standard handler process is linked to logger_sup, which is +%%% part of the kernel application's supervision tree. +start(Name, Config, HandlerState) -> + LoggerStdH = + #{id => Name, + start => {?MODULE, start_link, [Name,Config,HandlerState]}, + restart => temporary, + shutdown => 2000, + type => worker, + modules => [?MODULE]}, + case supervisor:start_child(logger_sup, LoggerStdH) of + {ok,_Pid} -> + ok; + Error -> + Error + end. + +%%%----------------------------------------------------------------- +%%% Stop and remove the handler. +stop(Name) -> + case whereis(Name) of + undefined -> + ok; + _ -> + %% We don't want to do supervisor:terminate_child here + %% since we need to distinguish this explicit stop from a + %% system termination in order to avoid circular attempts + %% at removing the handler (implying deadlocks and + %% timeouts). + _ = gen_server:call(Name,stop), + _ = supervisor:delete_child(logger_sup, Name), + ok + end. + +%%%----------------------------------------------------------------- +%%% Logging and overload control. +-define(update_file_ctrl_sync(C, Interval), + if C == 0 -> Interval; + true -> C-1 end). + +%% check for overload between every request (and set Mode to async, +%% sync or drop accordingly), but never flush the whole mailbox +%% before LogWindowSize requests have been handled +do_log(Bin, CallOrCast, State = #{id:=Name}) -> + T1 = ?timestamp(), + + %% check if the handler is getting overloaded, or if it's + %% recovering from overload (the check must be done for each + %% request to react quickly to large bursts of requests and + %% to ensure that the handler can never end up in drop mode + %% with an empty mailbox, which would stop operation) + {Mode1,QLen,Mem,State1} = logger_h_common:check_load(State), + + %% kill the handler if it can't keep up with the load + logger_h_common:kill_if_choked(Name, QLen, Mem, State), + + if Mode1 == flush -> + flush(Name, QLen, T1, State1); + true -> + write(Name, Mode1, T1, Bin, CallOrCast, State1) + end. + +%% this clause is called by do_log/3 after an overload check +%% has been performed, where QLen > FlushQLen +flush(Name, _QLen0, T1, State=#{last_log_ts := _T0}) -> + %% flush messages in the mailbox (a limited number in + %% order to not cause long delays) + _NewFlushed = logger_h_common:flush_log_requests(?FLUSH_MAX_N), + + %% because of the receive loop when flushing messages, the + %% handler will be scheduled out often and the mailbox could + %% grow very large, so we'd better check the queue again here + {_,_QLen1} = process_info(self(), message_queue_len), + ?observe(Name,{max_qlen,_QLen1}), + + %% Add 1 for the current log request + ?observe(Name,{flushed,_NewFlushed+1}), + + State1 = ?update_max_time(?diff_time(T1,_T0),State), + {dropped,?update_other(flushed,FLUSHED,_NewFlushed, + State1#{mode => ?set_mode(Name,async), + last_qlen => 0, + last_log_ts => T1})}. + +%% this clause is called to write to file +write(Name, Mode, T1, Bin, _CallOrCast, + State = #{file_ctrl_pid := FileCtrlPid, + file_ctrl_sync := FileCtrlSync, + last_qlen := LastQLen, + last_log_ts := T0, + file_ctrl_sync_int := FileCtrlSyncInt}) -> + %% check if we need to limit the number of writes + %% during a burst of log requests + {DoWrite,BurstWinT,BurstMsgCount} = logger_h_common:limit_burst(State), + + %% only send a synhrounous request to the file controller process + %% every FileCtrlSyncInt time, to give the handler time between + %% file writes so it can keep up with incoming messages + {Result,LastQLen1} = + if DoWrite, FileCtrlSync == 0 -> + ?observe(Name,{_CallOrCast,1}), + file_write_sync(FileCtrlPid, Bin, false), + {ok,element(2, process_info(self(), message_queue_len))}; + DoWrite -> + ?observe(Name,{_CallOrCast,1}), + file_write_async(FileCtrlPid, Bin), + {ok,LastQLen}; + not DoWrite -> + ?observe(Name,{flushed,1}), + {dropped,LastQLen} + end, + + %% Check if the time since the previous log request is long enough - + %% and the queue length small enough - to assume the mailbox has + %% been emptied, and if so, do filesync operation and reset mode to + %% async. Note that this is the best we can do to detect an idle + %% handler without setting a timer after each log call/cast. If the + %% time between two consecutive log requests is fast and no new + %% request comes in after the last one, idle state won't be detected! + Time = ?diff_time(T1,T0), + {Mode1,BurstMsgCount1} = + if (LastQLen1 < ?FILESYNC_OK_QLEN) andalso + (Time > ?IDLE_DETECT_TIME_USEC) -> + %% do filesync if necessary + case maps:get(type, State) of + Std when is_atom(Std) -> + ok; + _File -> + file_ctrl_filesync_async(FileCtrlPid) + end, + {?change_mode(Name, Mode, async),0}; + true -> + {Mode,BurstMsgCount} + end, + State1 = + ?update_calls_or_casts(_CallOrCast,1,State), + State2 = + ?update_max_time(Time, + State1#{mode => Mode1, + last_qlen := LastQLen1, + last_log_ts => T1, + burst_win_ts => BurstWinT, + burst_msg_count => BurstMsgCount1, + file_ctrl_sync => + ?update_file_ctrl_sync(FileCtrlSync, + FileCtrlSyncInt)}), + {Result,State2}. + +open_log_file(HandlerName, FileInfo) -> + case file_ctrl_start(HandlerName, FileInfo) of + OK = {ok,_FileCtrlPid} -> OK; + Error -> Error + end. + +do_open_log_file({file,File}) -> + do_open_log_file({file,File,[raw,append,delayed_write]}); + +do_open_log_file({file,File,[]}) -> + do_open_log_file({file,File,[raw,append,delayed_write]}); + +do_open_log_file({file,File,Modes}) -> + try + case filelib:ensure_dir(File) of + ok -> + file:open(File, Modes); + Error -> + Error + end + catch + _:Reason -> {error,Reason} + end. + +close_log_file(Std) when Std == standard_io; Std == standard_error -> + ok; +close_log_file(Fd) -> + _ = file:datasync(Fd), + _ = file:close(Fd). + +%%%----------------------------------------------------------------- +%%% File control process + +file_ctrl_start(HandlerName, FileInfo) -> + Starter = self(), + FileCtrlPid = + spawn_link(fun() -> + file_ctrl_init(HandlerName, FileInfo, Starter) + end), + receive + {FileCtrlPid,ok} -> + {ok,FileCtrlPid}; + {FileCtrlPid,Error} -> + Error + after + ?DEFAULT_CALL_TIMEOUT -> + {error,file_ctrl_process_not_started} + end. + +file_ctrl_stop(Pid) -> + Pid ! stop. + +file_write_async(Pid, Bin) -> + Pid ! {log,Bin}, + ok. + +file_write_sync(Pid, Bin, FileSync) -> + case file_ctrl_call(Pid, {log,self(),Bin,FileSync}) of + {error,Reason} -> + {error,{write_failed,Bin,Reason}}; + Result -> + Result + end. + +file_ctrl_filesync_async(Pid) -> + Pid ! filesync, + ok. + +file_ctrl_filesync_sync(Pid) -> + file_ctrl_call(Pid, {filesync,self()}). + +file_ctrl_call(Pid, Msg) -> + MRef = monitor(process, Pid), + Pid ! {Msg,MRef}, + receive + {MRef,Result} -> + demonitor(MRef, [flush]), + Result; + {'DOWN',MRef,_Type,_Object,Reason} -> + {error,Reason} + after + ?DEFAULT_CALL_TIMEOUT -> + {error,{no_response,Pid}} + end. + +file_ctrl_init(HandlerName, FileInfo, Starter) when is_tuple(FileInfo) -> + process_flag(message_queue_data, off_heap), + FileName = element(2, FileInfo), + case do_open_log_file(FileInfo) of + {ok,Fd} -> + Starter ! {self(),ok}, + file_ctrl_loop(Fd, file, FileName, false, ok, ok, HandlerName); + {error,Reason} -> + Starter ! {self(),{error,{open_failed,FileName,Reason}}} + end; +file_ctrl_init(HandlerName, StdDev, Starter) -> + Starter ! {self(),ok}, + file_ctrl_loop(StdDev, standard_io, StdDev, false, ok, ok, HandlerName). + +file_ctrl_loop(Fd, Type, DevName, Synced, + PrevWriteResult, PrevSyncResult, HandlerName) -> + receive + %% asynchronous request + {log,Bin} -> + Result = if Type == file -> + write_to_dev(Fd, Bin, DevName, + PrevWriteResult, HandlerName); + true -> + io:put_chars(Fd, Bin) + end, + file_ctrl_loop(Fd, Type, DevName, false, + Result, PrevSyncResult, HandlerName); + + %% synchronous request + {{log,From,Bin,FileSync},MRef} -> + if Type == file -> + %% check that file hasn't been deleted + CheckFile = + fun() -> {ok,_} = file:read_file_info(DevName) end, + spawn_link(CheckFile), + WResult = write_to_dev(Fd, Bin, DevName, + PrevWriteResult, HandlerName), + {Synced1,SResult} = + if not FileSync -> + {false,PrevSyncResult}; + true -> + case sync_dev(Fd, DevName, + PrevSyncResult, HandlerName) of + ok -> {true,ok}; + Error -> {false,Error} + end + end, + From ! {MRef,ok}, + file_ctrl_loop(Fd, Type, DevName, Synced1, + WResult, SResult, HandlerName); + true -> + _ = io:put_chars(Fd, Bin), + From ! {MRef,ok}, + file_ctrl_loop(Fd, Type, DevName, false, + ok, PrevSyncResult, HandlerName) + end; + + filesync when not Synced -> + Result = sync_dev(Fd, DevName, PrevSyncResult, HandlerName), + file_ctrl_loop(Fd, Type, DevName, true, + PrevWriteResult, Result, HandlerName); + + filesync -> + file_ctrl_loop(Fd, Type, DevName, true, + PrevWriteResult, PrevSyncResult, HandlerName); + + {{filesync,From},MRef} -> + Result = if not Synced -> + sync_dev(Fd, DevName, PrevSyncResult, HandlerName); + true -> + ok + end, + From ! {MRef,ok}, + file_ctrl_loop(Fd, Type, DevName, true, + PrevWriteResult, Result, HandlerName); + + stop -> + _ = close_log_file(Fd), + stopped + end. + +write_to_dev(Fd, Bin, FileName, PrevWriteResult, HandlerName) -> + case ?file_write(Fd, Bin) of + ok -> + ok; + PrevWriteResult -> + %% don't report same error twice + PrevWriteResult; + Error -> + logger_h_common:error_notify({HandlerName,write,FileName,Error}), + Error + end. + +sync_dev(Fd, DevName, PrevSyncResult, HandlerName) -> + case ?file_datasync(Fd) of + ok -> + ok; + PrevSyncResult -> + %% don't report same error twice + PrevSyncResult; + Error -> + logger_h_common:error_notify({HandlerName,filesync,DevName,Error}), + Error + end. + diff --git a/lib/kernel/src/logger_sup.erl b/lib/kernel/src/logger_sup.erl new file mode 100644 index 0000000000..4e4de94d5c --- /dev/null +++ b/lib/kernel/src/logger_sup.erl @@ -0,0 +1,53 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(logger_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/0]). + +%% Supervisor callbacks +-export([init/1]). + +-define(SERVER, ?MODULE). + +%%%=================================================================== +%%% API functions +%%%=================================================================== + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +%%%=================================================================== +%%% Supervisor callbacks +%%%=================================================================== + +init([]) -> + + SupFlags = #{strategy => one_for_one, + intensity => 1, + period => 5}, + + {ok, {SupFlags, []}}. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== diff --git a/lib/kernel/src/net_kernel.erl b/lib/kernel/src/net_kernel.erl index f38989d103..669adefdf8 100644 --- a/lib/kernel/src/net_kernel.erl +++ b/lib/kernel/src/net_kernel.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2017. All Rights Reserved. +%% Copyright Ericsson AB 1996-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -222,8 +222,7 @@ get_net_ticktime() -> Error :: error | {error, term()}. monitor_nodes(Flag) -> case catch process_flag(monitor_nodes, Flag) of - true -> ok; - false -> ok; + N when is_integer(N) -> ok; _ -> mk_monitor_nodes_error(Flag, []) end. @@ -236,8 +235,7 @@ monitor_nodes(Flag) -> Error :: error | {error, term()}. monitor_nodes(Flag, Opts) -> case catch process_flag({monitor_nodes, Opts}, Flag) of - true -> ok; - false -> ok; + N when is_integer(N) -> ok; _ -> mk_monitor_nodes_error(Flag, Opts) end. diff --git a/lib/kernel/src/seq_trace.erl b/lib/kernel/src/seq_trace.erl index cc0c10909b..8d7aba0f27 100644 --- a/lib/kernel/src/seq_trace.erl +++ b/lib/kernel/src/seq_trace.erl @@ -41,7 +41,7 @@ -type flag() :: 'send' | 'receive' | 'print' | 'timestamp' | 'monotonic_timestamp' | 'strict_monotonic_timestamp'. -type component() :: 'label' | 'serial' | flag(). --type value() :: (Integer :: non_neg_integer()) +-type value() :: (Label :: term()) | {Previous :: non_neg_integer(), Current :: non_neg_integer()} | (Bool :: boolean()). @@ -59,10 +59,6 @@ set_token({Flags,Label,Serial,_From,Lastcnt}) -> F = decode_flags(Flags), set_token2([{label,Label},{serial,{Lastcnt, Serial}} | F]). -%% We limit the label type to always be a small integer because erl_interface -%% expects that, the BIF can however "unofficially" handle atoms as well, and -%% atoms can be used if only Erlang nodes are involved - -spec set_token(Component, Val) -> {Component, OldVal} when Component :: component(), Val :: value(), diff --git a/lib/kernel/test/Makefile b/lib/kernel/test/Makefile index efe3a68531..8599a3d814 100644 --- a/lib/kernel/test/Makefile +++ b/lib/kernel/test/Makefile @@ -70,6 +70,15 @@ MODULES= \ interactive_shell_SUITE \ init_SUITE \ kernel_config_SUITE \ + logger_SUITE \ + logger_bench_SUITE \ + logger_disk_log_h_SUITE \ + logger_env_var_SUITE \ + logger_filters_SUITE \ + logger_formatter_SUITE \ + logger_legacy_SUITE \ + logger_simple_SUITE \ + logger_std_h_SUITE \ os_SUITE \ pg2_SUITE \ seq_trace_SUITE \ @@ -80,7 +89,8 @@ MODULES= \ loose_node \ sendfile_SUITE \ standard_error_SUITE \ - multi_load_SUITE + multi_load_SUITE \ + zzz_SUITE APP_FILES = \ appinc.app \ @@ -101,7 +111,7 @@ TARGET_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) INSTALL_PROGS= $(TARGET_FILES) EMAKEFILE=Emakefile -COVERFILE=kernel.cover +COVERFILE=kernel.cover logger.cover # ---------------------------------------------------- # Release directory specification @@ -148,7 +158,8 @@ release_tests_spec: make_emakefile $(INSTALL_DIR) "$(RELSYSDIR)" $(INSTALL_DATA) $(ERL_FILES) "$(RELSYSDIR)" $(INSTALL_DATA) $(APP_FILES) "$(RELSYSDIR)" - $(INSTALL_DATA) kernel.spec kernel_smoke.spec kernel_bench.spec \ + $(INSTALL_DATA) \ + kernel.spec kernel_smoke.spec kernel_bench.spec logger.spec \ $(EMAKEFILE) $(COVERFILE) "$(RELSYSDIR)" chmod -R u+w "$(RELSYSDIR)" @tar cf - *_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -) diff --git a/lib/kernel/test/application_SUITE.erl b/lib/kernel/test/application_SUITE.erl index 866043cfb4..c00fb44c46 100644 --- a/lib/kernel/test/application_SUITE.erl +++ b/lib/kernel/test/application_SUITE.erl @@ -1568,7 +1568,8 @@ loop5606(Pid) -> %% Tests get_env/* functions. get_env(Conf) when is_list(Conf) -> - {ok, _} = application:get_env(kernel, error_logger), + ok = application:set_env(kernel, new_var, new_val), + {ok, new_val} = application:get_env(kernel, new_var), undefined = application:get_env(undefined_app, a), undefined = application:get_env(kernel, error_logger_xyz), default = application:get_env(kernel, error_logger_xyz, default), diff --git a/lib/kernel/test/erl_distribution_SUITE.erl b/lib/kernel/test/erl_distribution_SUITE.erl index f6791adf86..0470f09f29 100644 --- a/lib/kernel/test/erl_distribution_SUITE.erl +++ b/lib/kernel/test/erl_distribution_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2017. All Rights Reserved. +%% Copyright Ericsson AB 1997-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1144,17 +1144,16 @@ monitor_nodes_otp_6481_test(Config, TestType) when is_list(Config) -> TestMonNodeState = monitor_node_state(), %% io:format("~p~n", [TestMonNodeState]), TestMonNodeState = - MonNodeState + case TestType of + nodedown -> []; + nodeup -> [{self(), []}] + end + ++ lists:map(fun (_) -> {MN, []} end, Seq) ++ case TestType of nodedown -> [{self(), []}]; nodeup -> [] end - ++ lists:map(fun (_) -> {MN, []} end, Seq) - ++ case TestType of - nodedown -> []; - nodeup -> [{self(), []}] - end, - + ++ MonNodeState, {ok, Node} = start_node(Name, "", this), receive {nodeup, Node} -> ok end, diff --git a/lib/kernel/test/error_logger_SUITE.erl b/lib/kernel/test/error_logger_SUITE.erl index 2d26a7246c..6c4526d0cf 100644 --- a/lib/kernel/test/error_logger_SUITE.erl +++ b/lib/kernel/test/error_logger_SUITE.erl @@ -32,7 +32,8 @@ init_per_group/2,end_per_group/2, off_heap/1, error_report/1, info_report/1, error/1, info/1, - emulator/1, tty/1, logfile/1, add/1, delete/1]). + emulator/1, via_logger_process/1, other_node/1, + tty/1, logfile/1, add/1, delete/1]). -export([generate_error/2]). @@ -46,16 +47,19 @@ suite() -> {timetrap,{minutes,1}}]. all() -> - [off_heap, error_report, info_report, error, info, emulator, tty, - logfile, add, delete]. + [off_heap, error_report, info_report, error, info, emulator, + via_logger_process, other_node, tty, logfile, add, delete]. groups() -> []. init_per_suite(Config) -> + logger:add_handler(error_logger,error_logger, + #{level=>info,filter_default=>log}), Config. end_per_suite(_Config) -> + logger:remove_handler(error_logger), ok. init_per_group(_GroupName, Config) -> @@ -226,6 +230,40 @@ generate_error(Error, Stack) -> erlang:raise(error, Error, Stack). %%----------------------------------------------------------------- + +via_logger_process(Config) -> + case os:type() of + {win32,_} -> + {skip,"Skip on windows - cant change file mode"}; + _ -> + error_logger:add_report_handler(?MODULE, self()), + Dir = filename:join(?config(priv_dir,Config),"dummydir"), + Msg = "File operation error: eacces. Target: " ++ + Dir ++ ". Function: list_dir. ", + ok = file:make_dir(Dir), + ok = file:change_mode(Dir,8#0222), + error = erl_prim_loader:list_dir(Dir), + ok = file:change_mode(Dir,8#0664), + _ = file:del_dir(Dir), + reported(error_report, std_error, Msg), + my_yes = error_logger:delete_report_handler(?MODULE), + ok + end. + +%%----------------------------------------------------------------- + +other_node(_Config) -> + error_logger:add_report_handler(?MODULE, self()), + {ok,Node} = test_server:start_node(?FUNCTION_NAME,slave,[]), + ok = rpc:call(Node,logger,add_handler,[error_logger,error_logger, + #{level=>info,filter_default=>log}]), + rpc:call(Node,error_logger,error_report,[hi_from_remote]), + reported(error_report,std_error,hi_from_remote), + test_server:stop_node(Node), + ok. + + +%%----------------------------------------------------------------- %% We don't enables or disables tty error logging here. We do not %% want to interact with the test run. %%----------------------------------------------------------------- @@ -279,7 +317,7 @@ reported(Tag, Type, Report) -> test_server:messages_get(), ok after 1000 -> - ct:fail(no_report_received) + ct:fail({no_report_received,test_server:messages_get()}) end. %%----------------------------------------------------------------- diff --git a/lib/kernel/test/file_SUITE.erl b/lib/kernel/test/file_SUITE.erl index 9a77454432..ff93f25e25 100644 --- a/lib/kernel/test/file_SUITE.erl +++ b/lib/kernel/test/file_SUITE.erl @@ -56,7 +56,8 @@ open1/1, old_modes/1, new_modes/1, path_open/1, open_errors/1]). -export([ file_info_basic_file/1, file_info_basic_directory/1, - file_info_bad/1, file_info_times/1, file_write_file_info/1]). + file_info_bad/1, file_info_times/1, file_write_file_info/1, + file_wfi_helpers/1]). -export([rename/1, access/1, truncate/1, datasync/1, sync/1, read_write/1, pread_write/1, append/1, exclusive/1]). -export([ e_delete/1, e_rename/1, e_make_dir/1, e_del_dir/1]). @@ -152,7 +153,8 @@ groups() -> {pos, [], [pos1, pos2, pos3]}, {file_info, [], [file_info_basic_file, file_info_basic_directory, - file_info_bad, file_info_times, file_write_file_info]}, + file_info_bad, file_info_times, file_write_file_info, + file_wfi_helpers]}, {consult, [], [consult1, path_consult]}, {eval, [], [eval1, path_eval]}, {script, [], [script1, path_script]}, @@ -1608,6 +1610,39 @@ file_write_file_info(Config) when is_list(Config) -> [] = flush(), ok. +file_wfi_helpers(Config) when is_list(Config) -> + RootDir = get_good_directory(Config), + io:format("RootDir = ~p", [RootDir]), + + Name = filename:join(RootDir, + atom_to_list(?MODULE) ++ "_wfi_helpers"), + + ok = ?FILE_MODULE:write_file(Name, "hello again"), + NewTime = {{1997, 02, 15}, {13, 18, 20}}, + ok = ?FILE_MODULE:change_time(Name, NewTime, NewTime), + + {ok, #file_info{atime=NewActAtime, mtime=NewTime}} = + ?FILE_MODULE:read_file_info(Name), + + NewFilteredAtime = filter_atime(NewTime, Config), + NewFilteredAtime = filter_atime(NewActAtime, Config), + + %% Make the file unwritable + ok = ?FILE_MODULE:change_mode(Name, 8#400), + {error, eacces} = ?FILE_MODULE:write_file(Name, "hello again"), + + %% ... and writable again + ok = ?FILE_MODULE:change_mode(Name, 8#600), + ok = ?FILE_MODULE:write_file(Name, "hello again"), + + %% We have no idea which users will work, so all we can do is to check + %% that it returns enoent instead of crashing. + {error, enoent} = ?FILE_MODULE:change_group("bogus file name", 0), + {error, enoent} = ?FILE_MODULE:change_owner("bogus file name", 0), + + [] = flush(), + ok. + %% Returns a directory on a file system that has correct file times. get_good_directory(Config) -> @@ -2177,7 +2212,7 @@ e_delete(Config) when is_list(Config) -> Base, #file_info {mode=0}), {error, eacces} = ?FILE_MODULE:delete(Afile), ?FILE_MODULE:write_file_info( - Base, #file_info {mode=8#600}) + Base, #file_info {mode=8#700}) end, [] = flush(), @@ -2308,7 +2343,7 @@ e_make_dir(Config) when is_list(Config) -> ?FILE_MODULE:write_file_info(Base, #file_info {mode=0}), {error, eacces} = ?FILE_MODULE:make_dir(filename:join(Base, "xxxx")), ?FILE_MODULE:write_file_info( - Base, #file_info {mode=8#600}) + Base, #file_info {mode=8#700}) end, ok. @@ -2354,7 +2389,7 @@ e_del_dir(Config) when is_list(Config) -> ok = ?FILE_MODULE:make_dir(ADirectory), ?FILE_MODULE:write_file_info( Base, #file_info {mode=0}), {error, eacces} = ?FILE_MODULE:del_dir(ADirectory), - ?FILE_MODULE:write_file_info( Base, #file_info {mode=8#600}) + ?FILE_MODULE:write_file_info( Base, #file_info {mode=8#700}) end, [] = flush(), ok. diff --git a/lib/kernel/test/init_SUITE.erl b/lib/kernel/test/init_SUITE.erl index 2b59eb2bfe..6a006cdc01 100644 --- a/lib/kernel/test/init_SUITE.erl +++ b/lib/kernel/test/init_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -299,7 +299,7 @@ many_restarts() -> many_restarts(Config) when is_list(Config) -> {ok, Node} = loose_node:start(init_test, "", ?DEFAULT_TIMEOUT_SEC), - loop_restart(50,Node,rpc:call(Node,erlang,whereis,[error_logger])), + loop_restart(50,Node,rpc:call(Node,erlang,whereis,[logger])), loose_node:stop(Node), ok. @@ -316,13 +316,13 @@ loop_restart(N,Node,EHPid) -> ct:fail(not_stopping) end, ok = wait_for(30, Node, EHPid), - loop_restart(N-1,Node,rpc:call(Node,erlang,whereis,[error_logger])). + loop_restart(N-1,Node,rpc:call(Node,erlang,whereis,[logger])). wait_for(0,Node,_) -> loose_node:stop(Node), error; wait_for(N,Node,EHPid) -> - case rpc:call(Node, erlang, whereis, [error_logger]) of + case rpc:call(Node, erlang, whereis, [logger]) of Pid when is_pid(Pid), Pid =/= EHPid -> %% erlang:display(ok), ok; @@ -365,7 +365,9 @@ restart(Config) when is_list(Config) -> %% Ok, the node is up, now the real test test begins. erlang:monitor_node(Node, true), SysProcs0 = rpc:call(Node, ?MODULE, find_system_processes, []), - [InitPid, PurgerPid, LitCollectorPid, DirtyCodePid] = SysProcs0, + io:format("SysProcs0=~p~n", [SysProcs0]), + [InitPid, PurgerPid, LitCollectorPid, + DirtySigNPid, DirtySigHPid, DirtySigMPid] = SysProcs0, InitPid = rpc:call(Node, erlang, whereis, [init]), PurgerPid = rpc:call(Node, erlang, whereis, [erts_code_purger]), Procs = rpc:call(Node, erlang, processes, []), @@ -381,7 +383,9 @@ restart(Config) when is_list(Config) -> ok = wait_restart(30, Node), SysProcs1 = rpc:call(Node, ?MODULE, find_system_processes, []), - [InitPid1, PurgerPid1, LitCollectorPid1, DirtyCodePid1] = SysProcs1, + io:format("SysProcs1=~p~n", [SysProcs1]), + [InitPid1, PurgerPid1, LitCollectorPid1, + DirtySigNPid1, DirtySigHPid1, DirtySigMPid1] = SysProcs1, %% Still the same init process! InitPid1 = rpc:call(Node, erlang, whereis, [init]), @@ -394,20 +398,18 @@ restart(Config) when is_list(Config) -> PurgerP = pid_to_list(PurgerPid1), %% and same literal area collector process! - case LitCollectorPid of - undefined -> undefined = LitCollectorPid1; - _ -> - LitCollectorP = pid_to_list(LitCollectorPid), - LitCollectorP = pid_to_list(LitCollectorPid1) - end, - - %% and same dirty process code checker process! - case DirtyCodePid of - undefined -> undefined = DirtyCodePid1; - _ -> - DirtyCodeP = pid_to_list(DirtyCodePid), - DirtyCodeP = pid_to_list(DirtyCodePid1) - end, + LitCollectorP = pid_to_list(LitCollectorPid), + LitCollectorP = pid_to_list(LitCollectorPid1), + + %% and same normal dirty signal handler process! + DirtySigNP = pid_to_list(DirtySigNPid), + DirtySigNP = pid_to_list(DirtySigNPid1), + %% and same high dirty signal handler process! + DirtySigHP = pid_to_list(DirtySigHPid), + DirtySigHP = pid_to_list(DirtySigHPid1), + %% and same max dirty signal handler process! + DirtySigMP = pid_to_list(DirtySigMPid), + DirtySigMP = pid_to_list(DirtySigMPid1), NewProcs0 = rpc:call(Node, erlang, processes, []), NewProcs = NewProcs0 -- SysProcs1, @@ -433,7 +435,9 @@ restart(Config) when is_list(Config) -> -record(sys_procs, {init, code_purger, literal_collector, - dirty_proc_checker}). + dirty_sig_handler_normal, + dirty_sig_handler_high, + dirty_sig_handler_max}). find_system_processes() -> find_system_procs(processes(), #sys_procs{}). @@ -442,21 +446,32 @@ find_system_procs([], SysProcs) -> [SysProcs#sys_procs.init, SysProcs#sys_procs.code_purger, SysProcs#sys_procs.literal_collector, - SysProcs#sys_procs.dirty_proc_checker]; + SysProcs#sys_procs.dirty_sig_handler_normal, + SysProcs#sys_procs.dirty_sig_handler_high, + SysProcs#sys_procs.dirty_sig_handler_max]; find_system_procs([P|Ps], SysProcs) -> - case process_info(P, initial_call) of - {initial_call,{otp_ring0,start,2}} -> + case process_info(P, [initial_call, priority]) of + [{initial_call,{otp_ring0,start,2}},_] -> undefined = SysProcs#sys_procs.init, find_system_procs(Ps, SysProcs#sys_procs{init = P}); - {initial_call,{erts_code_purger,start,0}} -> + [{initial_call,{erts_code_purger,start,0}},_] -> undefined = SysProcs#sys_procs.code_purger, find_system_procs(Ps, SysProcs#sys_procs{code_purger = P}); - {initial_call,{erts_literal_area_collector,start,0}} -> + [{initial_call,{erts_literal_area_collector,start,0}},_] -> undefined = SysProcs#sys_procs.literal_collector, find_system_procs(Ps, SysProcs#sys_procs{literal_collector = P}); - {initial_call,{erts_dirty_process_code_checker,start,0}} -> - undefined = SysProcs#sys_procs.dirty_proc_checker, - find_system_procs(Ps, SysProcs#sys_procs{dirty_proc_checker = P}); + [{initial_call,{erts_dirty_process_signal_handler,start,0}}, + {priority,normal}] -> + undefined = SysProcs#sys_procs.dirty_sig_handler_normal, + find_system_procs(Ps, SysProcs#sys_procs{dirty_sig_handler_normal = P}); + [{initial_call,{erts_dirty_process_signal_handler,start,0}}, + {priority,high}] -> + undefined = SysProcs#sys_procs.dirty_sig_handler_high, + find_system_procs(Ps, SysProcs#sys_procs{dirty_sig_handler_high = P}); + [{initial_call,{erts_dirty_process_signal_handler,start,0}}, + {priority,max}] -> + undefined = SysProcs#sys_procs.dirty_sig_handler_max, + find_system_procs(Ps, SysProcs#sys_procs{dirty_sig_handler_max = P}); _ -> find_system_procs(Ps, SysProcs) end. diff --git a/lib/kernel/test/kernel.spec b/lib/kernel/test/kernel.spec index 62afc9f97b..86d2155828 100644 --- a/lib/kernel/test/kernel.spec +++ b/lib/kernel/test/kernel.spec @@ -2,3 +2,4 @@ {config, "../test_server/ts.unix.config"}. {suites,"../kernel_test", all}. +{skip_suites,"../kernel_test",[logger_bench_SUITE],"Not ready"}. diff --git a/lib/kernel/test/logger.cover b/lib/kernel/test/logger.cover new file mode 100644 index 0000000000..b30bcfe920 --- /dev/null +++ b/lib/kernel/test/logger.cover @@ -0,0 +1,14 @@ +%% -*- erlang -*- +{incl_mods,[error_logger, + logger, + logger_backend, + logger_config, + logger_disk_log_h, + logger_h_common, + logger_filters, + logger_formatter, + logger_server, + logger_simple, + logger_std_h, + logger_sup]}. + diff --git a/lib/kernel/test/logger.spec b/lib/kernel/test/logger.spec new file mode 100644 index 0000000000..cd76a754a4 --- /dev/null +++ b/lib/kernel/test/logger.spec @@ -0,0 +1,11 @@ +%% -*-erlang-*- +{suites,"../kernel_test", [error_logger_SUITE, + error_logger_warn_SUITE, + logger_SUITE, + logger_disk_log_h_SUITE, + logger_env_var_SUITE, + logger_filters_SUITE, + logger_formatter_SUITE, + logger_legacy_SUITE, + logger_simple_SUITE, + logger_std_h_SUITE]}. diff --git a/lib/kernel/test/logger_SUITE.erl b/lib/kernel/test/logger_SUITE.erl new file mode 100644 index 0000000000..0edce3e34c --- /dev/null +++ b/lib/kernel/test/logger_SUITE.erl @@ -0,0 +1,828 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(logger_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("kernel/include/logger.hrl"). +-include_lib("kernel/src/logger_internal.hrl"). + +-define(str,"Log from "++atom_to_list(?FUNCTION_NAME)++ + ":"++integer_to_list(?LINE)). +-define(map_rep,#{function=>?FUNCTION_NAME, line=>?LINE}). +-define(keyval_rep,[{function,?FUNCTION_NAME}, {line,?LINE}]). + +-define(MY_LOC(N),#{mfa=>{?MODULE,?FUNCTION_NAME,?FUNCTION_ARITY}, + file=>?FILE, line=>?LINE-N}). + +-define(TRY(X), my_try(fun() -> X end)). + + +suite() -> + [{timetrap,{seconds,30}}]. + +init_per_suite(Config) -> + case logger:get_handler_config(logger_std_h) of + {ok,StdH} -> + ok = logger:remove_handler(logger_std_h), + [{logger_std_h,StdH}|Config]; + _ -> + Config + end. + +end_per_suite(Config) -> + case ?config(logger_std_h,Config) of + {HMod,HConfig} -> + ok = logger:add_handler(logger_std_h,HMod,HConfig); + _ -> + ok + end. + +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + {ok,LC} = logger:get_logger_config(), + [{logger_config,LC}|Config]. + +end_per_testcase(Case, Config) -> + try apply(?MODULE,Case,[cleanup,Config]) + catch error:undef -> ok + end, + ok. + +groups() -> + []. + +all() -> + [start_stop, + add_remove_handler, + multiple_handlers, + add_remove_filter, + change_config, + set_formatter, + log_all_levels_api, + macros, + set_level, + set_level_module, + cache_level_module, + format_report, + filter_failed, + handler_failed, + config_sanity_check, + log_failed, + emulator, + via_logger_process, + other_node, + compare_levels, + process_metadata]. + +start_stop(_Config) -> + S = whereis(logger), + true = is_pid(S), + ok. + +add_remove_handler(_Config) -> + register(callback_receiver,self()), + {ok,#{handlers:=Hs0}} = logger:get_logger_config(), + {error,{not_found,h1}} = logger:get_handler_config(h1), + ok = logger:add_handler(h1,?MODULE,#{}), + [add] = test_server:messages_get(), + {ok,#{handlers:=Hs}} = logger:get_logger_config(), + [h1|Hs0] = Hs, + {ok,{?MODULE,#{level:=info,filters:=[],filter_default:=log}}} = % defaults + logger:get_handler_config(h1), + ok = logger:set_handler_config(h1,filter_default,stop), + [changing_config] = test_server:messages_get(), + ?LOG_INFO("hello",[]), + ok = check_no_log(), + ok = logger:set_handler_config(h1,filter_default,log), + [changing_config] = test_server:messages_get(), + {ok,{?MODULE,#{filter_default:=log}}} = logger:get_handler_config(h1), + ?LOG_INFO("hello",[]), + ok = check_logged(info,"hello",[],?MY_LOC(1)), + ok = logger:remove_handler(h1), + [remove] = test_server:messages_get(), + {ok,#{handlers:=Hs0}} = logger:get_logger_config(), + {error,{not_found,h1}} = logger:get_handler_config(h1), + {error,{not_found,h1}} = logger:remove_handler(h1), + logger:info("hello",[]), + ok = check_no_log(), + ok. + +add_remove_handler(cleanup,_Config) -> + logger:remove_handler(h1), + ok. + +multiple_handlers(_Config) -> + ok = logger:add_handler(h1,?MODULE,#{level=>info,filter_default=>log}), + ok = logger:add_handler(h2,?MODULE,#{level=>error,filter_default=>log}), + ?LOG_ERROR("hello",[]), + ok = check_logged(error,"hello",[],?MY_LOC(1)), + ok = check_logged(error,"hello",[],?MY_LOC(2)), + ?LOG_INFO("hello",[]), + ok = check_logged(info,"hello",[],?MY_LOC(1)), + ok = check_no_log(), + ok. + +multiple_handlers(cleanup,_Config) -> + logger:remove_handler(h1), + logger:remove_handler(h2), + ok. + +add_remove_filter(_Config) -> + ok = logger:add_handler(h1,?MODULE,#{level=>info,filter_default=>log}), + LF = {fun(Log,_) -> Log#{level=>error} end, []}, + ok = logger:add_logger_filter(lf,LF), + {error,{already_exist,lf}} = logger:add_logger_filter(lf,LF), + {error,{already_exist,lf}} = logger:add_logger_filter(lf,{fun(Log,_) -> + Log + end, []}), + ?LOG_INFO("hello",[]), + ok = check_logged(error,"hello",[],?MY_LOC(1)), + ok = check_no_log(), + + ok = logger:add_handler(h2,?MODULE,#{level=>info,filter_default=>log}), + HF = {fun(#{level:=error}=Log,_) -> + Log#{level=>mylevel}; + (_,_) -> + ignore + end, + []}, + ok = logger:add_handler_filter(h1,hf,HF), + {error,{already_exist,hf}} = logger:add_handler_filter(h1,hf,HF), + {error,{already_exist,hf}} = logger:add_handler_filter(h1,hf,{fun(Log,_) -> + Log + end, []}), + ?LOG_INFO("hello",[]), + ok = check_logged(mylevel,"hello",[],?MY_LOC(1)), + ok = check_logged(error,"hello",[],?MY_LOC(2)), + + ok = logger:remove_logger_filter(lf), + {error,{not_found,lf}} = logger:remove_logger_filter(lf), + + ?LOG_INFO("hello",[]), + ok = check_logged(info,"hello",[],?MY_LOC(1)), + ok = check_logged(info,"hello",[],?MY_LOC(2)), + + ?LOG_ERROR("hello",[]), + ok = check_logged(mylevel,"hello",[],?MY_LOC(1)), + ok = check_logged(error,"hello",[],?MY_LOC(2)), + + ok = logger:remove_handler_filter(h1,hf), + {error,{not_found,hf}} = logger:remove_handler_filter(h1,hf), + ?LOG_INFO("hello",[]), + ok = check_logged(info,"hello",[],?MY_LOC(1)), + ok = check_logged(info,"hello",[],?MY_LOC(2)), + + ?LOG_ERROR("hello",[]), + ok = check_logged(error,"hello",[],?MY_LOC(1)), + ok = check_logged(error,"hello",[],?MY_LOC(2)), + ok. + +add_remove_filter(cleanup,_Config) -> + logger:remove_logger_filter(lf), + logger:remove_handler(h1), + logger:remove_handler(h2), + ok. + +change_config(_Config) -> + %% Overwrite handler config - check that defaults are added + ok = logger:add_handler(h1,?MODULE,#{level=>debug,custom=>custom}), + {ok,{?MODULE,#{level:=debug,filter_default:=log,custom:=custom}}} = + logger:get_handler_config(h1), + register(callback_receiver,self()), + ok = logger:set_handler_config(h1,#{filter_default=>stop}), + [changing_config] = test_server:messages_get(), + {ok,{?MODULE,#{level:=info,filter_default:=stop}=C2}} = + logger:get_handler_config(h1), + false = maps:is_key(custom,C2), + {error,fail} = logger:set_handler_config(h1,#{fail=>true}), + {error,{attempting_syncronous_call_to_self,_}} = + logger:set_handler_config( + h1,#{call=>fun() -> logger:set_module_level(?MODULE,debug) end}), + {ok,{?MODULE,C2}} = logger:get_handler_config(h1), + + %% Change one key only + {error,fail} = logger:set_handler_config(h1,fail,true), + ok = logger:set_handler_config(h1,custom,custom), + [changing_config] = test_server:messages_get(), + {ok,{?MODULE,#{custom:=custom}=C3}} = logger:get_handler_config(h1), + C2 = maps:remove(custom,C3), + + %% Overwrite logger config - check that defaults are added + {ok,LConfig} = logger:get_logger_config(), + ok = logger:set_logger_config(#{filter_default=>stop}), + {ok,#{level:=info,filters:=[],handlers:=[],filter_default:=stop}=LC1} = + logger:get_logger_config(), + 4 = maps:size(LC1), + + %% Change one key only + ok = logger:set_logger_config(handlers,[h1]), + {ok,#{level:=info,filters:=[],handlers:=[h1],filter_default:=stop}} = + logger:get_logger_config(), + + %% Cleanup + ok = logger:set_logger_config(LConfig), + [] = test_server:messages_get(), + + ok. + +change_config(cleanup,Config) -> + logger:remove_handler(h1), + LC = ?config(logger_config,Config), + logger:set_logger_config(LC), + ok. + +set_formatter(_Config) -> + {error,{not_found,h1}}=logger:set_handler_config(h1,formatter,{?MODULE,[]}), + ok = logger:add_handler(h1,?MODULE,#{level=>info,filter_default=>log}), + ok = logger:set_handler_config(h1,formatter,{?MODULE,[]}), + logger:info("hello",[]), + receive + {_Log,#{formatter:={?MODULE,[]}}} -> + ok + after 500 -> + ct:fail({timeout,no_log,process_info(self(),messages)}) + end, + ok. + +set_formatter(cleanup,_Config) -> + logger:remove_handler(h1), + ok. + +log_all_levels_api(_Config) -> + ok = logger:set_logger_config(level,debug), + ok = logger:add_handler(h1,?MODULE,#{level=>debug,filter_default=>log}), + test_api(emergency), + test_api(alert), + test_api(critical), + test_api(error), + test_api(warning), + test_api(notice), + test_api(info), + test_api(debug), + test_log_function(emergency), + ok. + +log_all_levels_api(cleanup,_Config) -> + logger:remove_handler(h1), + logger:set_logger_config(level,info), + ok. + +macros(_Config) -> + ok = logger:set_module_level(?MODULE,debug), + ok = logger:add_handler(h1,?MODULE,#{level=>debug,filter_default=>log}), + test_macros(emergency), + ok. + +macros(cleanup,_Config) -> + logger:remove_handler(h1), + logger:reset_module_level(?MODULE), + ok. + +set_level(_Config) -> + ok = logger:add_handler(h1,?MODULE,#{level=>debug,filter_default=>log}), + logger:debug(?map_rep), + ok = check_no_log(), + logger:info(M1=?map_rep), + ok = check_logged(info,M1,#{}), + ok = logger:set_logger_config(level,debug), + logger:debug(M2=?map_rep), + ok = check_logged(debug,M2,#{}), + ok. + +set_level(cleanup,_Config) -> + logger:remove_handler(h1), + logger:set_logger_config(level,info), + ok. + +set_level_module(_Config) -> + ok = logger:add_handler(h1,?MODULE,#{level=>info,filter_default=>log}), + {error,{invalid_level,bad}} = logger:set_module_level(?MODULE,bad), + {error,{not_a_module,{bad}}} = logger:set_module_level({bad},warning), + ok = logger:set_module_level(?MODULE,warning), + logger:info(?map_rep,?MY_LOC(0)), + ok = check_no_log(), + logger:warning(M1=?map_rep,?MY_LOC(0)), + ok = check_logged(warning,M1,?MY_LOC(1)), + ok = logger:set_module_level(?MODULE,info), + logger:info(M2=?map_rep,?MY_LOC(0)), + ok = check_logged(info,M2,?MY_LOC(1)), + + {error,{not_a_module,{bad}}} = logger:reset_module_level({bad}), + ok = logger:reset_module_level(?MODULE), + + ok. + +set_level_module(cleanup,_Config) -> + logger:remove_handler(h1), + logger:reset_module_level(?MODULE), + ok. + +cache_level_module(_Config) -> + ok = logger:reset_module_level(?MODULE), + [] = ets:lookup(logger,?MODULE), %dirty - add API in logger_config? + ?LOG_INFO(?map_rep), + %% Caching is done asynchronously, so wait a bit for the update + timer:sleep(100), + [_] = ets:lookup(logger,?MODULE), %dirty - add API in logger_config? + ok = logger:reset_module_level(?MODULE), + [] = ets:lookup(logger,?MODULE), %dirty - add API in logger_config? + ok. + +cache_level_module(cleanup,_Config) -> + logger:reset_module_level(?MODULE), + ok. + +format_report(_Config) -> + {"~ts",["string"]} = logger:format_report("string"), + {"~tp",[term]} = logger:format_report(term), + {"~tp",[[]]} = logger:format_report([]), + {" ~tp: ~tp",[key,value]} = logger:format_report([{key,value}]), + KeyVals = [{key1,value1},{key2,"value2"},{key3,[]}], + KeyValRes = + {" ~tp: ~tp\n ~tp: ~ts\n ~tp: ~tp", + [key1,value1,key2,"value2",key3,[]]} = + logger:format_report(KeyVals), + KeyValRes = logger:format_report(maps:from_list(KeyVals)), + KeyValRes = logger:format_otp_report(#{label=>{?MODULE,test},report=>KeyVals}), + {" ~tp: ~tp\n ~tp: ~tp", + [label,{?MODULE,test},report,KeyVals]} = + logger:format_report(#{label=>{?MODULE,test},report=>KeyVals}), + + {" ~tp: ~tp\n ~tp",[key1,value1,term]} = + logger:format_report([{key1,value1},term]), + + {" ~tp: ~tp\n ~tp",[key1,value1,[]]} = + logger:format_report([{key1,value1},[]]), + + {"~tp",[[]]} = logger:format_report([[],[],[]]), + + ok. + +filter_failed(_Config) -> + ok = logger:add_handler(h1,?MODULE,#{level=>info,filter_default=>log}), + + %% Logger filters + {error,{invalid_filter,_}} = + logger:add_logger_filter(lf,{fun(_) -> ok end,args}), + ok = logger:add_logger_filter(lf,{fun(_,_) -> a=b end,args}), + {ok,#{filters:=[_]}} = logger:get_logger_config(), + ok = logger:info(M1=?map_rep), + ok = check_logged(info,M1,#{}), + {error,{not_found,lf}} = logger:remove_logger_filter(lf), + + ok = logger:add_logger_filter(lf,{fun(_,_) -> faulty_return end,args}), + {ok,#{filters:=[_]}} = logger:get_logger_config(), + ok = logger:info(M2=?map_rep), + ok = check_logged(info,M2,#{}), + {error,{not_found,lf}} = logger:remove_logger_filter(lf), + + %% Handler filters + {error,{not_found,h0}} = + logger:add_handler_filter(h0,hf,{fun(_,_) -> ignore end,args}), + {error,{not_found,h0}} = logger:remove_handler_filter(h0,hf), + {error,{invalid_filter,_}} = + logger:add_handler_filter(h1,hf,{fun(_) -> ok end,args}), + ok = logger:add_handler_filter(h1,hf,{fun(_,_) -> a=b end,args}), + {ok,{?MODULE,#{filters:=[_]}}} = logger:get_handler_config(h1), + ok = logger:info(M3=?map_rep), + ok = check_logged(info,M3,#{}), + {error,{not_found,hf}} = logger:remove_handler_filter(h1,hf), + + ok = logger:add_handler_filter(h1,hf,{fun(_,_) -> faulty_return end,args}), + {ok,{?MODULE,#{filters:=[_]}}} = logger:get_handler_config(h1), + ok = logger:info(M4=?map_rep), + ok = check_logged(info,M4,#{}), + {error,{not_found,hf}} = logger:remove_handler_filter(h1,hf), + + ok. + +filter_failed(cleanup,_Config) -> + logger:remove_handler(h1), + ok. + +handler_failed(_Config) -> + {error,{invalid_id,1}} = logger:add_handler(1,?MODULE,#{}), + {error,{invalid_module,"nomodule"}} = logger:add_handler(h1,"nomodule",#{}), + {error,{invalid_handler_config,bad}} = logger:add_handler(h1,?MODULE,bad), + {error,{invalid_filters,false}} = + logger:add_handler(h1,?MODULE,#{filters=>false}), + {error,{invalid_filter_default,true}} = + logger:add_handler(h1,?MODULE,#{filter_default=>true}), + {error,{invalid_formatter,[]}} = + logger:add_handler(h1,?MODULE,#{formatter=>[]}), + ok = logger:add_handler(h1,nomodule,#{filter_default=>log}), + logger:info(?map_rep), + check_no_log(), + #{logger:=#{handlers:=Ids1}, + handlers:=H1} = logger:i(), + false = lists:member(h1,Ids1), + false = lists:keymember(h1,1,H1), + {error,{not_found,h1}} = logger:remove_handler(h1), + + ok = logger:add_handler(h2,?MODULE,#{filter_default=>log,crash=>true}), + {error,{already_exist,h2}} = logger:add_handler(h2,othermodule,#{}), + + logger:info(?map_rep), + check_no_log(), + #{logger:=#{handlers:=Ids2}, + handlers:=H2} = logger:i(), + false = lists:member(h2,Ids2), + false = lists:keymember(h2,1,H2), + {error,{not_found,h2}} = logger:remove_handler(h2), + + ok. + +handler_failed(cleanup,_Config) -> + logger:remove_handler(h1), + logger:remove_handler(h2), + ok. + +config_sanity_check(_Config) -> + %% Logger config + {error,{invalid_filter_default,bad}} = + logger:set_logger_config(filter_default,bad), + {error,{invalid_level,bad}} = logger:set_logger_config(level,bad), + {error,{invalid_handlers,bad}} = logger:set_logger_config(handlers,bad), + {error,{invalid_id,{bad,bad}}} = + logger:set_logger_config(handlers,[{bad,bad}]), + {error,{invalid_id,"bad"}} = logger:set_logger_config(handlers,["bad"]), + {error,{invalid_filters,bad}} = logger:set_logger_config(filters,bad), + {error,{invalid_filter,bad}} = logger:set_logger_config(filters,[bad]), + {error,{invalid_filter,{_,_}}} = + logger:set_logger_config(filters,[{id,bad}]), + {error,{invalid_filter,{_,{_,_}}}} = + logger:set_logger_config(filters,[{id,{bad,args}}]), + {error,{invalid_filter,{_,{_,_}}}} = + logger:set_logger_config(filters,[{id,{fun() -> ok end,args}}]), + {error,{invalid_logger_config,{bad,bad}}} = + logger:set_logger_config(bad,bad), + + %% Handler config + {error,{not_found,h1}} = logger:set_handler_config(h1,a,b), + ok = logger:add_handler(h1,?MODULE,#{}), + {error,{invalid_filter_default,bad}} = + logger:set_handler_config(h1,filter_default,bad), + {error,{invalid_level,bad}} = logger:set_handler_config(h1,level,bad), + {error,{invalid_filters,bad}} = logger:set_handler_config(h1,filters,bad), + {error,{invalid_filter,bad}} = logger:set_handler_config(h1,filters,[bad]), + {error,{invalid_filter,{_,_}}} = + logger:set_handler_config(h1,filters,[{id,bad}]), + {error,{invalid_filter,{_,{_,_}}}} = + logger:set_handler_config(h1,filters,[{id,{bad,args}}]), + {error,{invalid_filter,{_,{_,_}}}} = + logger:set_handler_config(h1,filters,[{id,{fun() -> ok end,args}}]), + {error,{invalid_formatter,bad}} = + logger:set_handler_config(h1,formatter,bad), + {error,{invalid_module,{bad}}} = + logger:set_handler_config(h1,formatter,{{bad},cfg}), + {error,{invalid_formatter_config,bad}} = + logger:set_handler_config(h1,formatter,{logger_formatter,bad}), + {error,{invalid_formatter_config,{bad,bad}}} = + logger:set_handler_config(h1,formatter,{logger_formatter,#{bad=>bad}}), + {error,{invalid_formatter_config,{template,bad}}} = + logger:set_handler_config(h1,formatter,{logger_formatter, + #{template=>bad}}), + {error,{invalid_formatter_template,[1]}} = + logger:set_handler_config(h1,formatter,{logger_formatter, + #{template=>[1]}}), + ok = logger:set_handler_config(h1,formatter,{logger_formatter, + #{template=>[]}}), + {error,{invalid_formatter_config,{single_line,bad}}} = + logger:set_handler_config(h1,formatter,{logger_formatter, + #{single_line=>bad}}), + ok = logger:set_handler_config(h1,formatter,{logger_formatter, + #{single_line=>true}}), + {error,{invalid_formatter_config,{legacy_header,bad}}} = + logger:set_handler_config(h1,formatter,{logger_formatter, + #{legacy_header=>bad}}), + ok = logger:set_handler_config(h1,formatter,{logger_formatter, + #{legacy_header=>true}}), + ok = logger:set_handler_config(h1,custom,custom), + ok. + +config_sanity_check(cleanup,_Config) -> + logger:remove_handler(h1), + ok. + +log_failed(_Config) -> + ok = logger:add_handler(h1,?MODULE,#{level=>info,filter_default=>log}), + {error,function_clause} = ?TRY(logger:log(bad,?map_rep)), + {error,function_clause} = ?TRY(logger:log(info,?map_rep,bad)), + {error,function_clause} = ?TRY(logger:log(info,fun() -> ?map_rep end,bad)), + {error,function_clause} = ?TRY(logger:log(info,fun() -> ?map_rep end,bad,#{})), + {error,function_clause} = ?TRY(logger:log(info,bad,bad,bad)), + {error,function_clause} = ?TRY(logger:log(info,bad,bad,#{})), + check_no_log(), + ok = logger:log(info,M1=?str,#{}), + check_logged(info,M1,#{}), + ok = logger:log(info,M2=?map_rep,#{}), + check_logged(info,M2,#{}), + ok = logger:log(info,M3=?keyval_rep,#{}), + check_logged(info,M3,#{}), + + %% Should we check report input more thoroughly? + ok = logger:log(info,M4=?keyval_rep++[other,stuff,in,list],#{}), + check_logged(info,M4,#{}), + + %% This might break a handler since it is assumed to be a format + %% string and args, so it depends how the handler protects itself + %% against something like io_lib:format("ok","ok") + ok = logger:log(info,"ok","ok",#{}), + check_logged(info,"ok","ok",#{}), + + ok. + +log_failed(cleanup,_Config) -> + logger:remove_handler(h1), + ok. + +emulator(_Config) -> + ok = logger:add_handler(h1,?MODULE,#{level=>info,filter_default=>log, + tc_proc=>self()}), + Msg = "Error in process ~p on node ~p with exit value:~n~p~n", + Error = {badmatch,4}, + Stack = [{module, function, 2, []}], + Pid = spawn(?MODULE, generate_error, [Error, Stack]), + check_logged(error, Msg, [Pid, node(), {Error, Stack}], + #{gl=>group_leader(), + error_logger=>#{tag=>error,emulator=>true}}), + ok. + +emulator(cleanup, _Config) -> + logger:remove_handler(h1), + ok. + +generate_error(Error, Stack) -> + erlang:raise(error, Error, Stack). + +via_logger_process(Config) -> + ok = logger:add_handler(h1,?MODULE,#{level=>info,filter_default=>log, + tc_proc=>self()}), + + %% Explicitly send a message to the logger process + %% This is used by code_server, erl_prim_loader, init, prim_file, ... + Msg = ?str, + logger ! {log,error,Msg,[],#{}}, + check_logged(error, Msg, [], #{}), + + case os:type() of + {win32,_} -> + %% Skip this part on windows - cant change file mode" + ok; + _ -> + %% This should trigger the same thing from erl_prim_loader + Dir = filename:join(?config(priv_dir,Config),"dummydir"), + ok = file:make_dir(Dir), + ok = file:change_mode(Dir,8#0222), + error = erl_prim_loader:list_dir(Dir), + check_logged(error, + #{report=>"File operation error: eacces. Target: " ++ + Dir ++". Function: list_dir. "}, + #{pid=>self(), + gl=>group_leader(), + error_logger=>#{tag=>error_report, + type=>std_error}}), + ok + end. + +via_logger_process(cleanup, Config) -> + Dir = filename:join(?config(priv_dir,Config),"dummydir"), + _ = file:change_mode(Dir,8#0664), + _ = file:del_dir(Dir), + logger:remove_handler(h1), + ok. + +other_node(_Config) -> + ok = logger:add_handler(h1,?MODULE,#{level=>info,filter_default=>log, + tc_proc=>self()}), + {ok,Node} = test_server:start_node(?FUNCTION_NAME,slave,[]), + rpc:call(Node,logger,error,[Msg=?str,#{}]), + check_logged(error,Msg,#{}), + ok. + +other_node(cleanup,_Config) -> + Nodes = nodes(), + [test_server:stop_node(Node) || Node <- Nodes], + logger:remove_handler(h1), + ok. + +compare_levels(_Config) -> + Levels = [emergency,alert,critical,error,warning,notice,info,debug], + ok = compare(Levels), + {error,badarg} = ?TRY(logger:compare_levels(bad,bad)), + {error,badarg} = ?TRY(logger:compare_levels({bad},info)), + {error,badarg} = ?TRY(logger:compare_levels(info,"bad")), + ok. + +compare([L|Rest]) -> + eq = logger:compare_levels(L,L), + [gt = logger:compare_levels(L,L1) || L1 <- Rest], + [lt = logger:compare_levels(L1,L) || L1 <- Rest], + compare(Rest); +compare([]) -> + ok. + +process_metadata(_Config) -> + undefined = logger:get_process_metadata(), + {error,badarg} = ?TRY(logger:set_process_metadata(bad)), + ok = logger:add_handler(h1,?MODULE,#{level=>info,filter_default=>log}), + Time = erlang:monotonic_time(microsecond), + ProcMeta = #{time=>Time,line=>0,custom=>proc}, + ok = logger:set_process_metadata(ProcMeta), + S1 = ?str, + ?LOG_INFO(S1,#{custom=>macro}), + check_logged(info,S1,#{time=>Time,line=>0,custom=>macro}), + + Time2 = erlang:monotonic_time(microsecond), + S2 = ?str, + ?LOG_INFO(S2,#{time=>Time2,line=>1,custom=>macro}), + check_logged(info,S2,#{time=>Time2,line=>1,custom=>macro}), + + logger:info(S3=?str,#{custom=>func}), + check_logged(info,S3,#{time=>Time,line=>0,custom=>func}), + + ProcMeta = logger:get_process_metadata(), + ok = logger:unset_process_metadata(), + undefined = logger:get_process_metadata(), + + ok. + +process_metadata(cleanup,_Config) -> + logger:remove_handler(h1), + ok. + +%%%----------------------------------------------------------------- +%%% Internal +check_logged(Level,Format,Args,Meta) -> + do_check_logged(Level,{Format,Args},Meta). + +check_logged(Level,Msg,Meta) when ?IS_REPORT(Msg) -> + do_check_logged(Level,{report,Msg},Meta); +check_logged(Level,Msg,Meta) when ?IS_STRING(Msg) -> + do_check_logged(Level,{string,Msg},Meta). + +do_check_logged(Level,Msg0,Meta0) -> + receive + {#{level:=Level,msg:=Msg,meta:=Meta},_} -> + check_msg(Msg0,Msg), + check_maps(Meta0,Meta,meta) + after 500 -> + ct:fail({timeout,no_log,process_info(self(),messages)}) + end. + +check_no_log() -> + receive + X -> ct:fail({got_unexpected_log,X}) + after 500 -> + ok + end. + +check_msg(Msg,Msg) -> + ok; +check_msg({report,Expected},{report,Got}) when is_map(Expected), is_map(Got) -> + check_maps(Expected,Got,msg); +check_msg(Expected,Got) -> + ct:fail({unexpected,msg,Expected,Got}). + +check_maps(Expected,Got,What) -> + case maps:merge(Got,Expected) of + Got -> + ok; + _ -> + ct:fail({unexpected,What,Expected,Got}) + end. + +%% Handler +adding_handler(_Id,Config) -> + maybe_send(add), + {ok,Config}. +removing_handler(_Id) -> + maybe_send(remove), + ok. +changing_config(_Id,_Old,#{call:=Fun}) -> + Fun(); +changing_config(_Id,_Old,#{fail:=true}) -> + {error,fail}; +changing_config(_Id,_Old,Config) -> + maybe_send(changing_config), + {ok,Config}. + +maybe_send(Msg) -> + case whereis(callback_receiver) of + undefined -> ok; + Pid -> Pid ! Msg + end. + +log(_Log,#{crash:=true}) -> + a=b; +log(Log,Config) -> + TcProc = maps:get(tc_proc,Config,self()), + TcProc ! {Log,Config}, + ok. + +test_api(Level) -> + logger:Level(#{Level=>rep}), + ok = check_logged(Level,#{Level=>rep},#{}), + logger:Level(#{Level=>rep},#{my=>meta}), + ok = check_logged(Level,#{Level=>rep},#{my=>meta}), + logger:Level("~w: ~w",[Level,fa]), + ok = check_logged(Level,"~w: ~w",[Level,fa],#{}), + logger:Level("~w: ~w ~w",[Level,fa,meta],#{my=>meta}), + ok = check_logged(Level,"~w: ~w ~w",[Level,fa,meta],#{my=>meta}), + logger:Level(fun(x) -> {"~w: ~w ~w",[Level,fun_to_fa,meta]} end,x, + #{my=>meta}), + ok = check_logged(Level,"~w: ~w ~w",[Level,fun_to_fa,meta],#{my=>meta}), + logger:Level(fun(x) -> #{Level=>fun_to_r,meta=>true} end,x, + #{my=>meta}), + ok = check_logged(Level,#{Level=>fun_to_r,meta=>true},#{my=>meta}), + logger:Level(fun(x) -> <<"fun_to_s">> end,x,#{}), + ok = check_logged(Level,<<"fun_to_s">>,#{}), + logger:Level(F1=fun(x) -> {fun_to_bad} end,x,#{}), + ok = check_logged(Level,"LAZY_FUN ERROR: ~tp; Returned: ~tp", + [{F1,x},{fun_to_bad}],#{}), + logger:Level(F2=fun(x) -> erlang:error(fun_that_crashes) end,x,#{}), + ok = check_logged(Level,"LAZY_FUN CRASH: ~tp; Reason: ~tp", + [{F2,x},{error,fun_that_crashes}],#{}), + ok. + +test_log_function(Level) -> + logger:log(Level,#{Level=>rep}), + ok = check_logged(Level,#{Level=>rep},#{}), + logger:log(Level,#{Level=>rep},#{my=>meta}), + ok = check_logged(Level,#{Level=>rep},#{my=>meta}), + logger:log(Level,"~w: ~w",[Level,fa]), + ok = check_logged(Level,"~w: ~w",[Level,fa],#{}), + logger:log(Level,"~w: ~w ~w",[Level,fa,meta],#{my=>meta}), + ok = check_logged(Level,"~w: ~w ~w",[Level,fa,meta],#{my=>meta}), + logger:log(Level,fun(x) -> {"~w: ~w ~w",[Level,fun_to_fa,meta]} end, + x, #{my=>meta}), + ok = check_logged(Level,"~w: ~w ~w",[Level,fun_to_fa,meta],#{my=>meta}), + logger:log(Level,fun(x) -> #{Level=>fun_to_r,meta=>true} end, + x, #{my=>meta}), + ok = check_logged(Level,#{Level=>fun_to_r,meta=>true},#{my=>meta}), + logger:log(Level,fun(x) -> <<"fun_to_s">> end,x,#{}), + ok = check_logged(Level,<<"fun_to_s">>,#{}), + logger:log(Level,F1=fun(x) -> {fun_to_bad} end,x,#{}), + ok = check_logged(Level,"LAZY_FUN ERROR: ~tp; Returned: ~tp", + [{F1,x},{fun_to_bad}],#{}), + logger:log(Level,F2=fun(x) -> erlang:error(fun_that_crashes) end,x,#{}), + ok = check_logged(Level,"LAZY_FUN CRASH: ~tp; Reason: ~tp", + [{F2,x},{error,fun_that_crashes}],#{}), + ok. + +test_macros(emergency=Level) -> + ?LOG_EMERGENCY(#{Level=>rep}), + ok = check_logged(Level,#{Level=>rep},?MY_LOC(1)), + ?LOG_EMERGENCY(#{Level=>rep},#{my=>meta}), + ok = check_logged(Level,#{Level=>rep},(?MY_LOC(1))#{my=>meta}), + ?LOG_EMERGENCY("~w: ~w",[Level,fa]), + ok = check_logged(Level,"~w: ~w",[Level,fa],?MY_LOC(1)), + ?LOG_EMERGENCY("~w: ~w ~w",[Level,fa,meta],#{my=>meta}), + ok = check_logged(Level,"~w: ~w ~w",[Level,fa,meta],(?MY_LOC(1))#{my=>meta}), + ?LOG_EMERGENCY(fun(x) -> {"~w: ~w ~w",[Level,fun_to_fa,meta]} end, + x, #{my=>meta}), + ok = check_logged(Level,"~w: ~w ~w",[Level,fun_to_fa,meta], + (?MY_LOC(3))#{my=>meta}), + ?LOG_EMERGENCY(fun(x) -> #{Level=>fun_to_r,meta=>true} end, x, #{my=>meta}), + ok = check_logged(Level,#{Level=>fun_to_r,meta=>true}, + (?MY_LOC(2))#{my=>meta}), + ?LOG_EMERGENCY(fun(x) -> <<"fun_to_s">> end,x,#{}), + ok = check_logged(Level,<<"fun_to_s">>,?MY_LOC(1)), + F1=fun(x) -> {fun_to_bad} end, + ?LOG_EMERGENCY(F1,x,#{}), + ok = check_logged(Level,"LAZY_FUN ERROR: ~tp; Returned: ~tp", + [{F1,x},{fun_to_bad}],#{}), + F2=fun(x) -> erlang:error(fun_that_crashes) end, + ?LOG_EMERGENCY(F2,x,#{}), + ok = check_logged(Level,"LAZY_FUN CRASH: ~tp; Reason: ~tp", + [{F2,x},{error,fun_that_crashes}],#{}), + ok. + +%%%----------------------------------------------------------------- +%%% Called by macro ?TRY(X) +my_try(Fun) -> + try Fun() catch C:R -> {C,R} end. diff --git a/lib/kernel/test/logger_bench_SUITE.erl b/lib/kernel/test/logger_bench_SUITE.erl new file mode 100644 index 0000000000..d47122ea9d --- /dev/null +++ b/lib/kernel/test/logger_bench_SUITE.erl @@ -0,0 +1,500 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(logger_bench_SUITE). + +-compile(export_all). + +%%%----------------------------------------------------------------- +%%% To include lager tests, add paths to lager and goldrush +%%% (goldrush is a dependency inside the lager repo) +%%% +%%% To print data to .csv files, add the following to a config file: +%%% {print_csv,[{console_handler,[{path,"/some/dir/"}]}]}. +%%% +%%%----------------------------------------------------------------- + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). +-include_lib("kernel/include/logger.hrl"). +-include_lib("kernel/src/logger_internal.hrl"). + +-define(msg,lists:flatten(string:pad("Log from "++atom_to_list(?FUNCTION_NAME)++ + ":"++integer_to_list(?LINE), + 80,trailing,$*))). +-define(meta,#{mfa=>{?MODULE,?FUNCTION_NAME,?FUNCTION_ARITY}, + pid=>self()}). + +-define(NO_COMPARE,[profile]). + +-define(TIMES,100000). + +suite() -> + [{timetrap,{seconds,120}}]. + +init_per_suite(Config) -> + DataDir = ?config(data_dir,Config), + have_lager() andalso make(DataDir), + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(Group, Config) -> + H = remove_all_handlers(), + do_init_per_group(Group), + [{handlers,H}|Config]. + +do_init_per_group(minimal_handler) -> + ok = logger:add_handler(?MODULE,?MODULE,#{level=>error,filter_default=>log}); +do_init_per_group(console_handler) -> + ok = logger:add_handler(?MODULE,logger_std_h, + #{filter_default=>stop, + filters=>?DEFAULT_HANDLER_FILTERS, + logger_std_h=>#{type=>standard_io, + toggle_sync_qlen => ?TIMES+1, + drop_new_reqs_qlen => ?TIMES+2, + flush_reqs_qlen => ?TIMES+3, + enable_burst_limit => false}}), + have_lager() andalso lager_helper:start(), + ok. + +end_per_group(Group, Config) -> + case ?config(saved_config,Config) of + {_,[{bench,Bench}]} -> + print_compare_chart(Group,Bench); + _ -> + ok + end, + add_all_handlers(?config(handlers,Config)), + do_end_per_group(Group). + +do_end_per_group(minimal_handler) -> + ok = logger:remove_handler(?MODULE); +do_end_per_group(console_handler) -> + ok = logger:remove_handler(?MODULE), + have_lager() andalso lager_helper:stop(), + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(Case, Config) -> + try apply(?MODULE,Case,[cleanup,Config]) + catch error:undef -> ok + end, + wait_for_handlers(), + ok. + +wait_for_handlers() -> + wait_for_handler(?MODULE), + wait_for_handler(lager_event). + +wait_for_handler(Handler) -> + case whereis(Handler) of + undefined -> + io:format("~p: noproc1",[Handler]), + ok; + Pid -> + case process_info(Pid,message_queue_len) of + {_,0} -> + io:format("~p: queue=~p",[Handler,0]), + ok; + {_,Q} -> + io:format("~p: queue=~p",[Handler,Q]), + timer:sleep(2000), + wait_for_handler(Handler); + undefined -> + io:format("~p: noproc2",[Handler]), + ok + end + end. + +groups() -> + [{minimal_handler,[],[log, + log_drop, + log_drop_by_handler, + macro, + macro_drop, + macro_drop_by_handler, + error_logger, + error_logger_drop, + error_logger_drop_by_handler + ]}, + {console_handler,[],[%profile, + log, + log_drop, + log_drop_by_handler, + %% log_handler_complete, + macro, + macro_drop, + macro_drop_by_handler, + %% macro_handler_complete, + error_logger, + error_logger_drop, + error_logger_drop_by_handler%% , + %% error_logger_handler_complete + ] ++ lager_cases()} + ]. + +lager_cases() -> + case have_lager() of + true -> + [lager_log, + lager_log_drop, + lager_log_drop_by_handler, + %% lager_log_handler_complete, + lager_parsetrans, + lager_parsetrans_drop, + lager_parsetrans_drop_by_handler%% , + %% lager_parsetrans_handler_complete + ]; + false -> + [] + end. + + +all() -> + [{group,minimal_handler}, + {group,console_handler} + ]. + +log(Config) -> + Times = ?TIMES, + run_benchmark(Config,?FUNCTION_NAME,fun do_log_func/2, [error,?msg], Times). + +log_drop(Config) -> + Times = ?TIMES*100, + ok = logger:set_logger_config(level,error), + run_benchmark(Config,?FUNCTION_NAME,fun do_log_func/2, [info,?msg], Times). + +log_drop(cleanup,_Config) -> + ok = logger:set_logger_config(level,info). + +log_drop_by_handler(Config) -> + Times = ?TIMES, + %% just ensure correct levels + ok = logger:set_logger_config(level,info), + ok = logger:set_handler_config(?MODULE,level,error), + run_benchmark(Config,?FUNCTION_NAME,fun do_log_func/2, [info,?msg], Times). + +log_handler_complete(Config) -> + ok = logger:set_handler_config(?MODULE,formatter, + {?MODULE,?DEFAULT_FORMAT_CONFIG}), + handler_complete(Config, ?FUNCTION_NAME, fun do_log_func/2, [error,?msg]). + +log_handler_complete(cleanup,_Config) -> + ok=logger:set_handler_config(?MODULE,formatter, + {?DEFAULT_FORMATTER,?DEFAULT_FORMAT_CONFIG}). + +macro(Config) -> + Times = ?TIMES, + run_benchmark(Config,?FUNCTION_NAME,fun do_log_macro/2,[error,?msg], Times). + +macro_drop(Config) -> + Times = ?TIMES*100, + ok = logger:set_logger_config(level,error), + run_benchmark(Config,?FUNCTION_NAME,fun do_log_macro/2,[info,?msg], Times). + +macro_drop(cleanup,_Config) -> + ok = logger:set_logger_config(level,info). + +macro_drop_by_handler(Config) -> + Times = ?TIMES, + %% just ensure correct levels + ok = logger:set_logger_config(level,info), + ok = logger:set_handler_config(?MODULE,level,error), + run_benchmark(Config,?FUNCTION_NAME,fun do_log_macro/2, [info,?msg], Times). + +macro_handler_complete(Config) -> + ok = logger:set_handler_config(?MODULE,formatter, + {?MODULE,?DEFAULT_FORMAT_CONFIG}), + handler_complete(Config, ?FUNCTION_NAME, fun do_log_macro/2, [error,?msg]). + +macro_handler_complete(cleanup,_Config) -> + ok=logger:set_handler_config(?MODULE,formatter, + {?DEFAULT_FORMATTER,?DEFAULT_FORMAT_CONFIG}). + +error_logger(Config) -> + Times = ?TIMES, + run_benchmark(Config,?FUNCTION_NAME,fun do_error_logger/2, [error,?msg], Times). + +error_logger_drop(Config) -> + Times = ?TIMES*100, + ok = logger:set_logger_config(level,error), + run_benchmark(Config,?FUNCTION_NAME,fun do_error_logger/2, [info,?msg], Times). + +error_logger_drop(cleanup,_Config) -> + ok = logger:set_logger_config(level,info). + +error_logger_drop_by_handler(Config) -> + Times = ?TIMES, + %% just ensure correct levels + ok = logger:set_logger_config(level,info), + ok = logger:set_handler_config(?MODULE,level,error), + run_benchmark(Config,?FUNCTION_NAME,fun do_log_func/2, [info,?msg], Times). + +error_logger_handler_complete(Config) -> + ok = logger:set_handler_config(?MODULE,formatter, + {?MODULE,?DEFAULT_FORMAT_CONFIG}), + handler_complete(Config, ?FUNCTION_NAME, fun do_error_logger/2, [error,?msg]). + +error_logger_handler_complete(cleanup,_Config) -> + ok=logger:set_handler_config(?MODULE,formatter, + {?DEFAULT_FORMATTER,?DEFAULT_FORMAT_CONFIG}). + +lager_log(Config) -> + Times = ?TIMES, + run_benchmark(Config,?FUNCTION_NAME,fun lager_helper:do_func/2, [error,?msg], Times). + +lager_log_drop(Config) -> + Times = ?TIMES*100, + run_benchmark(Config,?FUNCTION_NAME,fun lager_helper:do_func/2, [info,?msg], Times). + +lager_log_drop_by_handler(Config) -> + %% This concept does not exist, so doing same as lager_log_drop/1 + Times = ?TIMES, + run_benchmark(Config,?FUNCTION_NAME,fun lager_helper:do_func/2, [info,?msg], Times). + +lager_log_handler_complete(Config) -> + handler_complete(Config, ?FUNCTION_NAME, fun lager_helper:do_func/2, [error,?msg]). + +lager_parsetrans(Config) -> + Times = ?TIMES, + run_benchmark(Config,?FUNCTION_NAME,fun lager_helper:do_parsetrans/2, [error,?msg], Times). + +lager_parsetrans_drop(Config) -> + Times = ?TIMES*100, + run_benchmark(Config,?FUNCTION_NAME,fun lager_helper:do_parsetrans/2, [info,?msg], Times). + +lager_parsetrans_drop_by_handler(Config) -> + %% This concept does not exist, so doing same as lager_parsetrans_drop/1 + Times = ?TIMES, + run_benchmark(Config,?FUNCTION_NAME,fun lager_helper:do_parsetrans/2, [info,?msg], Times). + +lager_parsetrans_handler_complete(Config) -> + handler_complete(Config, ?FUNCTION_NAME, fun lager_helper:do_parsetrans/2, [error,?msg]). + + +profile(Config) -> + Times = ?TIMES, + %% fprof:apply(fun repeated_apply/3,[fun lager_helper:do_func/2,[error,?msg],Times]), + fprof:apply(fun repeated_apply/3,[fun do_log_func/2,[error,?msg],Times]), + ok = fprof:profile(), + ok = fprof:analyse(dest,"../fprof.analyse"), + ok. + +%%%----------------------------------------------------------------- +%%% Internal +%% Handler +log(_Log,_Config) -> + ok. + +format(Log=#{meta:=#{pid:=Pid}},Config) when is_pid(Pid) -> + String = ?DEFAULT_FORMATTER:format(Log,Config), + Pid ! done, + String; +format(Log=#{meta:=#{pid:=PidStr}},Config) when is_list(PidStr) -> + String = ?DEFAULT_FORMATTER:format(Log,Config), + list_to_pid(PidStr) ! done, + String. + +handler_complete(Config, TC, Fun, Args) -> + Times = ?TIMES, + Start = os:perf_counter(microsecond), + repeated_apply(Fun, Args, Times), + MSecs = wait_for_done(Start,Times), + calc_and_report(Config,TC,MSecs,Times). + +wait_for_done(Start,0) -> + os:perf_counter(microsecond) - Start; +wait_for_done(Start,N) -> + receive + done -> + wait_for_done(Start,N-1) + after 20000 -> + ct:fail("missing " ++ integer_to_list(N) ++ " replys") + end. + +%%%----------------------------------------------------------------- +%%% Benchmark stuff +run_benchmark(Config,Tag,Fun,Args,Times) -> + _ = erlang:apply(Fun, Args), % apply once to ensure level is cached + MSecs = measure_repeated_op(Fun, Args, Times), + %% fprof:profile(), + %% fprof:analyse(dest,"../"++atom_to_list(Tag)++".prof"), + calc_and_report(Config,Tag,MSecs,Times). + +measure_repeated_op(Fun, Args, Times) -> + Start = os:perf_counter(microsecond), + %% fprof:apply(fun repeated_apply/3, [Fun, Args, Times]), + repeated_apply(Fun, Args, Times), + os:perf_counter(microsecond) - Start. + +repeated_apply(_F, _Args, Times) when Times =< 0 -> + ok; +repeated_apply(F, Args, Times) -> + erlang:apply(F, Args), + repeated_apply(F, Args, Times - 1). + +calc_and_report(Config,Tag,MSecs,Times) -> + IOPS = trunc(Times * (1000000 / MSecs)), + ct_event:notify(#event{ name = benchmark_data, data = [{value,IOPS}] }), + ct:print("~p:~n~p IOPS, ~p us", [Tag, IOPS, MSecs]), + ct:comment("~p IOPS, ~p us", [IOPS, MSecs]), + Bench = case ?config(saved_config,Config) of + {_,[{bench,B}]} -> B; + undefined -> [] + end, + {save_config,[{bench,[{Tag,IOPS,MSecs}|Bench]}]}. + +remove_all_handlers() -> + #{handlers:=Hs} = logger:i(), + [logger:remove_handler(Id) || {Id,_,_} <- Hs], + Hs. + +add_all_handlers(Hs) -> + [logger:add_handler(Id,Mod,Config) || {Id,Mod,Config} <- Hs], + ok. + +%%%----------------------------------------------------------------- +%%% Call logger in different ways +do_log_func(Level,Msg) -> + logger:Level(Msg,[],?meta). + +do_log_macro(error,Msg) -> + ?LOG_ERROR(Msg,[]); +do_log_macro(info,Msg) -> + ?LOG_INFO(Msg,[]); +do_log_macro(debug,Msg) -> + ?LOG_DEBUG(Msg,[]). + +do_error_logger(error,Msg) -> + error_logger:error_msg(Msg,[]); +do_error_logger(info,Msg) -> + error_logger:info_msg(Msg,[]). + +%%%----------------------------------------------------------------- +%%% +print_compare_chart(Group,Bench) -> + io:format("~-20s~12s~12s~12s~12s", + ["Microseconds:","Log","Drop","HDrop","Complete"]), + io:format(user,"~-20s~12s~12s~12s~12s~n", + ["Microseconds:","Log","Drop","HDrop","Complete"]), + {Log,Drop,HDrop,Comp} = sort_bench(Bench,[],[],[],[]), + print_compare_chart(Log,Drop,HDrop,Comp), + io:format(user,"~n",[]), + maybe_print_csv_files(Group, + [{log,Log},{drop,Drop},{hdrop,HDrop},{comp,Comp}]). + +print_compare_chart([{What,LIOPS,LMSecs}|Log], + [{What,DIOPS,DMSecs}|Drop], + [{What,HIOPS,HMSecs}|HDrop], + [{What,CIOPS,CMSecs}|Comp]) -> + io:format("~-20w~12w~12w~12w~12w",[What,LMSecs,DMSecs,HMSecs,CMSecs]), + io:format(user,"~-20w~12w~12w~12w~12w~n",[What,LMSecs,DMSecs,HMSecs,CMSecs]), + print_compare_chart(Log,Drop,HDrop,Comp); +print_compare_chart([{What,LIOPS,LMSecs}|Log], + [{What,DIOPS,DMSecs}|Drop], + [{What,HIOPS,HMSecs}|HDrop], + []=Comp) -> + io:format("~-20w~12w~12w~12w",[What,LMSecs,DMSecs,HMSecs]), + io:format(user,"~-20w~12w~12w~12w~n",[What,LMSecs,DMSecs,HMSecs]), + print_compare_chart(Log,Drop,HDrop,Comp); +print_compare_chart([],[],[],[]) -> + ok; +print_compare_chart(Log,Drop,HDrop,Comp) -> + ct:fail({Log,Drop,HDrop,Comp}). + +sort_bench([{TC,IOPS,MSecs}|Bench],Log,Drop,HDrop,Comp) -> + case lists:member(TC,?NO_COMPARE) of + true -> + sort_bench(Bench,Log,Drop,HDrop,Comp); + false -> + TCStr = atom_to_list(TC), + {What,Type} = + case re:run(TCStr,"(.*)_(drop.*)", + [{capture,all_but_first,list}]) of + {match,[WhatStr,TypeStr]} -> + {list_to_atom(WhatStr),list_to_atom(TypeStr)}; + nomatch -> + case re:run(TCStr,"(.*)_(handler_complete.*)", + [{capture,all_but_first,list}]) of + {match,[WhatStr,TypeStr]} -> + {list_to_atom(WhatStr),list_to_atom(TypeStr)}; + nomatch -> + {TC,log} + end + end, + case Type of + log -> + sort_bench(Bench,[{What,IOPS,MSecs}|Log],Drop,HDrop,Comp); + drop -> + sort_bench(Bench,Log,[{What,IOPS,MSecs}|Drop],HDrop,Comp); + drop_by_handler -> + sort_bench(Bench,Log,Drop,[{What,IOPS,MSecs}|HDrop],Comp); + handler_complete -> + sort_bench(Bench,Log,Drop,HDrop,[{What,IOPS,MSecs}|Comp]) + end + end; +sort_bench([],Log,Drop,HDrop,Comp) -> + {lists:keysort(1,Log), + lists:keysort(1,Drop), + lists:keysort(1,HDrop), + lists:keysort(1,Comp)}. + +maybe_print_csv_files(Group,Data) -> + case ct:get_config({print_csv,Group}) of + undefined -> + ok; + Cfg -> + Path = proplists:get_value(path,Cfg,".."), + Files = [begin + File = filename:join(Path,F)++".csv", + case filelib:is_regular(File) of + true -> + {ok,Fd} = file:open(File,[append]), + Fd; + false -> + {ok,Fd} = file:open(File,[write]), + ok = file:write(Fd, + "error_logger,lager_log," + "lager_parsetrans,logger_log," + "logger_macro\n"), + Fd + end + end || {F,_} <- Data], + [print_csv_file(F,D) || {F,D} <- lists:zip(Files,Data)], + [file:close(Fd) || Fd <- Files], + ok + end. + +print_csv_file(Fd,{_,Data}) -> + AllIOPS = [integer_to_list(IOPS) || {_,IOPS,_} <- Data], + ok = file:write(Fd,lists:join(",",AllIOPS)++"\n"). + +have_lager() -> + code:ensure_loaded(lager) == {module,lager}. + +make(Dir) -> + {ok,Cwd} = file:get_cwd(), + ok = file:set_cwd(Dir), + up_to_date = make:all([load]), + ok = file:set_cwd(Cwd), + code:add_path(Dir). diff --git a/lib/kernel/test/logger_bench_SUITE_data/Emakefile b/lib/kernel/test/logger_bench_SUITE_data/Emakefile new file mode 100644 index 0000000000..85c82bdaab --- /dev/null +++ b/lib/kernel/test/logger_bench_SUITE_data/Emakefile @@ -0,0 +1 @@ +{['lager_helper'],[{outdir,"."},debug_info,{i,"/home/uabshan/Work/git/otp/lib/kernel/src"},{i,"/home/uabshan/Work/git/otp/lib/kernel/include"}]}. diff --git a/lib/kernel/test/logger_bench_SUITE_data/lager_helper.erl b/lib/kernel/test/logger_bench_SUITE_data/lager_helper.erl new file mode 100644 index 0000000000..296ced4276 --- /dev/null +++ b/lib/kernel/test/logger_bench_SUITE_data/lager_helper.erl @@ -0,0 +1,73 @@ +-module(lager_helper). + +-compile(export_all). +-compile({parse_transform,lager_transform}). + +-include_lib("kernel/src/logger_internal.hrl"). + +start() -> + application:load(lager), + application:set_env(lager, error_logger_redirect, false), + application:set_env(lager, async_threshold, 100010), + application:set_env(lager, async_threshold_window, 100), + application:set_env(lager,handlers,[{?MODULE,[{level,error}]}]), + lager:start(). + +stop() -> + application:stop(lager). + +do_func(Level,Msg) -> + lager:log(Level,[{pid,self()}],Msg,[]). + +do_parsetrans(error,Msg) -> + lager:error(Msg,[]); +do_parsetrans(info,Msg) -> + lager:info(Msg,[]). + +%%%----------------------------------------------------------------- +%%% Dummy handler for lager +-record(state, {level :: {'mask', integer()}, + formatter :: atom(), + format_config :: any()}). +init(Opts) -> + Level = proplists:get_value(level,Opts,info), + Formatter = proplists:get_value(formatter,Opts,logger_bench_SUITE), + FormatConfig = proplists:get_value(format_config,Opts,?DEFAULT_FORMAT_CONFIG), + {ok,#state{level=lager_util:config_to_mask(Level), + formatter=Formatter, + format_config=FormatConfig}}. + +handle_call(get_loglevel, #state{level=Level} = State) -> + {ok, Level, State}; +handle_call({set_loglevel, Level}, State) -> + try lager_util:config_to_mask(Level) of + Levels -> + {ok, ok, State#state{level=Levels}} + catch + _:_ -> + {ok, {error, bad_log_level}, State} + end; +handle_call(_Request, State) -> + {ok, ok, State}. + +handle_event({log, Message}, + #state{level=L,formatter=Formatter,format_config=FormatConfig} = State) -> + case lager_util:is_loggable(Message, L, ?MODULE) of + true -> + Metadata = + case maps:from_list(lager_msg:metadata(Message)) of + Meta = #{pid:=Pid} when is_pid(Pid) -> + Meta; + Meta = #{pid:=PidStr} when is_list(PidStr) -> + Meta + end, + Log = #{level=>lager_msg:severity(Message), + msg=>{report,lager_msg:message(Message)}, + meta=>Metadata}, + io:put_chars(user, Formatter:format(Log,FormatConfig)), + {ok, State}; + false -> + {ok, State} + end; +handle_event(_Event, State) -> + {ok, State}. diff --git a/lib/kernel/test/logger_disk_log_h_SUITE.erl b/lib/kernel/test/logger_disk_log_h_SUITE.erl new file mode 100644 index 0000000000..c7c6137380 --- /dev/null +++ b/lib/kernel/test/logger_disk_log_h_SUITE.erl @@ -0,0 +1,1417 @@ +-module(logger_disk_log_h_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("kernel/include/logger.hrl"). +-include_lib("kernel/src/logger_internal.hrl"). +-include_lib("kernel/src/logger_h_common.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). +-include_lib("kernel/include/file.hrl"). + +-define(check_no_log, [] = test_server:messages_get()). + +-define(check(Expected), + receive {log,Expected} -> + [] = test_server:messages_get() + after 1000 -> + ct:fail({report_not_received, + {line,?LINE}, + {got,test_server:messages_get()}}) + end). + +-define(msg,"Log from "++atom_to_list(?FUNCTION_NAME)++ + ":"++integer_to_list(?LINE)). +-define(bin(Msg), list_to_binary(Msg++"\n")). +-define(log_no(File,N), lists:concat([File,".",N])). +-define(domain,#{domain=>[?MODULE]}). + +-define(SYNC_REP_INT, if is_atom(?FILESYNC_REPEAT_INTERVAL) -> 5500; + true -> ?FILESYNC_REPEAT_INTERVAL + 500 + end). + +suite() -> + [{timetrap,{seconds,30}}]. + +init_per_suite(Config) -> + timer:start(), % to avoid progress report + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(TestHooksCase, Config) when + TestHooksCase == write_failure; + TestHooksCase == sync_failure -> + if ?TEST_HOOKS_TAB == undefined -> + {skip,"Define the TEST_HOOKS macro to run this test"}; + true -> + ct:print("********** ~w **********", [TestHooksCase]), + Config + end; +init_per_testcase(TestCase, Config) -> + ct:print("********** ~w **********", [TestCase]), + Config. + +end_per_testcase(Case, Config) -> + try apply(?MODULE,Case,[cleanup,Config]) + catch error:undef -> ok + end, + ok. + +groups() -> + []. + +all() -> + [start_stop_handler, + create_log, + open_existing_log, + disk_log_opts, + default_formatter, + logging, + errors, + formatter_fail, + config_fail, + bad_input, + info_and_reset, + reconfig, + disk_log_sync, + disk_log_full, + disk_log_wrap, + disk_log_events, + write_failure, + sync_failure, + op_switch_to_sync, + op_switch_to_drop, + op_switch_to_flush, + limit_burst_disabled, + limit_burst_enabled_one, + limit_burst_enabled_period, + kill_disabled, + qlen_kill_new, + %% qlen_kill_std, + mem_kill_new, + %% mem_kill_std, + restart_after, + handler_requests_under_load + ]. + +start_stop_handler(_Config) -> + ok = logger:add_handler(?MODULE, logger_disk_log_h, #{}), + {error,{already_exist,?MODULE}} = + logger:add_handler(?MODULE, logger_disk_log_h, #{}), + true = is_pid(whereis(?MODULE)), + ok = logger:remove_handler(?MODULE), + timer:sleep(500), + undefined = whereis(?MODULE). +start_stop_handler(cleanup, _Config) -> + logger:remove_handler(?MODULE). + +create_log(Config) -> + PrivDir = ?config(priv_dir,Config), + %% test new handler + Name1 = list_to_atom(lists:concat([?FUNCTION_NAME,"_A"])), + LogFile1 = filename:join(PrivDir, Name1), + ok = start_and_add(Name1, #{filter_default=>stop, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), + formatter=>{?MODULE,self()}}, + #{file=>LogFile1}), + logger:info("hello", ?domain), + logger_disk_log_h:disk_log_sync(Name1), + ct:pal("Checking contents of ~p", [?log_no(LogFile1,1)]), + try_read_file(?log_no(LogFile1,1), {ok,<<"hello\n">>}, 5000), + + %% test second handler + Name2 = list_to_atom(lists:concat([?FUNCTION_NAME,"_B"])), + DLName = lists:concat([?FUNCTION_NAME,"_B_log"]), + LogFile2 = filename:join(PrivDir, DLName), + ok = start_and_add(Name2, #{filter_default=>stop, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), + formatter=>{?MODULE,self()}}, + #{file=>LogFile2}), + logger:info("dummy", ?domain), + logger_disk_log_h:disk_log_sync(Name2), + ct:pal("Checking contents of ~p", [?log_no(LogFile2,1)]), + try_read_file(?log_no(LogFile2,1), {ok,<<"dummy\n">>}, 5000), + + remove_and_stop(Name1), + remove_and_stop(Name2), + try_read_file(?log_no(LogFile1,1), {ok,<<"hello\ndummy\n">>}, 1), + try_read_file(?log_no(LogFile2,1), {ok,<<"dummy\n">>}, 5000), + ok. + +open_existing_log(Config) -> + PrivDir = ?config(priv_dir,Config), + %% test new handler + HName = ?FUNCTION_NAME, + DLName = lists:concat([?FUNCTION_NAME,"_log"]), + LogFile1 = filename:join(PrivDir, DLName), + ok = start_and_add(HName, #{filter_default=>stop, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), + formatter=>{?MODULE,self()}}, + #{file=>LogFile1}), + logger:info("one", ?domain), + logger_disk_log_h:disk_log_sync(HName), + ct:pal("Checking contents of ~p", [?log_no(LogFile1,1)]), + try_read_file(?log_no(LogFile1,1), {ok,<<"one\n">>}, 5000), + logger:info("two", ?domain), + ok = remove_and_stop(HName), + try_read_file(?log_no(LogFile1,1), {ok,<<"one\ntwo\n">>}, 5000), + + logger:info("two and a half", ?domain), + + ok = start_and_add(HName, #{filter_default=>stop, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), + formatter=>{?MODULE,self()}}, + #{file=>LogFile1}), + logger:info("three", ?domain), + logger_disk_log_h:disk_log_sync(HName), + try_read_file(?log_no(LogFile1,1), {ok,<<"one\ntwo\nthree\n">>}, 5000), + remove_and_stop(HName), + try_read_file(?log_no(LogFile1,1), {ok,<<"one\ntwo\nthree\n">>}, 5000). + +disk_log_opts(Config) -> + Get = fun(Key, PL) -> proplists:get_value(Key, PL) end, + PrivDir = ?config(priv_dir,Config), + WName = list_to_atom(lists:concat([?FUNCTION_NAME,"_W"])), + WFile = lists:concat([?FUNCTION_NAME,"_W_log"]), + Size = length("12345"), + ConfigW = #{filter_default=>stop, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), + formatter => {?MODULE,no_nl}}, + WFileFull = filename:join(PrivDir, WFile), + DLOptsW = #{file => WFileFull, + type => wrap, + max_no_bytes => Size, + max_no_files => 2}, + ok = start_and_add(WName, ConfigW, DLOptsW), + WInfo1 = disk_log:info(WName), + ct:log("Fullname = ~s", [WFileFull]), + {WFileFull,wrap,{Size,2},1} = {Get(file,WInfo1),Get(type,WInfo1), + Get(size,WInfo1),Get(current_file,WInfo1)}, + logger:info("123", ?domain), + logger_disk_log_h:disk_log_sync(WName), + timer:sleep(500), + 1 = Get(current_file, disk_log:info(WName)), + + logger:info("45", ?domain), + logger_disk_log_h:disk_log_sync(WName), + timer:sleep(500), + 1 = Get(current_file, disk_log:info(WName)), + + logger:info("6", ?domain), + logger_disk_log_h:disk_log_sync(WName), + timer:sleep(500), + 2 = Get(current_file, disk_log:info(WName)), + + logger:info("7890", ?domain), + logger_disk_log_h:disk_log_sync(WName), + timer:sleep(500), + 2 = Get(current_file, disk_log:info(WName)), + + HName1 = list_to_atom(lists:concat([?FUNCTION_NAME,"_H1"])), + HFile1 = lists:concat([?FUNCTION_NAME,"_H1_log"]), + ConfigH = #{filter_default=>stop, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), + formatter => {?MODULE,no_nl}}, + HFile1Full = filename:join(PrivDir, HFile1), + DLOptsH1 = #{file => HFile1Full, + type => halt}, + ok = start_and_add(HName1, ConfigH, DLOptsH1), + HInfo1 = disk_log:info(HName1), + ct:log("Fullname = ~s", [HFile1Full]), + {HFile1Full,halt,infinity} = {Get(file,HInfo1),Get(type,HInfo1), + Get(size,HInfo1)}, + logger:info("12345", ?domain), + logger_disk_log_h:disk_log_sync(HName1), + timer:sleep(500), + 1 = Get(no_written_items, disk_log:info(HName1)), + + HName2 = list_to_atom(lists:concat([?FUNCTION_NAME,"_H2"])), + HFile2 = lists:concat([?FUNCTION_NAME,"_H2_log"]), + HFile2Full = filename:join(PrivDir, HFile2), + DLOptsH2 = DLOptsH1#{file => HFile2Full, + max_no_bytes => 1000}, + ok = start_and_add(HName2, ConfigH, DLOptsH2), + HInfo3 = disk_log:info(HName2), + ct:log("Fullname = ~s", [HFile2Full]), + {HFile2Full,halt,1000} = {Get(file,HInfo3),Get(type,HInfo3), + Get(size,HInfo3)}, + + remove_and_stop(WName), + remove_and_stop(HName1), + remove_and_stop(HName2), + ok. + +default_formatter(Config) -> + PrivDir = ?config(priv_dir,Config), + LogFile = filename:join(PrivDir,atom_to_list(?FUNCTION_NAME)), + HConfig = #{disk_log_opts => #{file=>LogFile}, + filter_default=>log}, + ct:pal("Log: ~p", [LogFile]), + ok = logger:add_handler(?MODULE, logger_disk_log_h, HConfig), + ok = logger:set_handler_config(?MODULE,formatter, + {?DEFAULT_FORMATTER,?DEFAULT_FORMAT_CONFIG}), + LogName = lists:concat([LogFile, ".1"]), + logger:info("dummy"), + wait_until_written(LogName), + {ok,Bin} = file:read_file(LogName), + match = re:run(Bin, "=INFO REPORT====.*\ndummy", [{capture,none}]), + ok. +default_formatter(cleanup, _Config) -> + logger:remove_handler(?MODULE). + +logging(Config) -> + PrivDir = ?config(priv_dir,Config), + %% test new handler + Name = list_to_atom(lists:concat([?FUNCTION_NAME,"_1"])), + LogFile = filename:join(PrivDir, Name), + ok = start_and_add(Name, #{filter_default=>log, + formatter=>{?MODULE,self()}}, + #{file => LogFile}), + MsgFormatter = fun(Term) -> {io_lib:format("Term:~p",[Term]),[]} end, + logger:info([{x,y}], #{report_cb => MsgFormatter}), + logger:info([{x,y}], #{}), + ct:pal("Checking contents of ~p", [?log_no(LogFile,1)]), + try_read_file(?log_no(LogFile,1), {ok,<<"Term:[{x,y}]\n x: y\n">>}, 5000). + +logging(cleanup, _Config) -> + Name = list_to_atom(lists:concat([?FUNCTION_NAME,"_1"])), + remove_and_stop(Name). + +errors(Config) -> + PrivDir = ?config(priv_dir,Config), + Name1 = list_to_atom(lists:concat([?FUNCTION_NAME,"_1"])), + LogFile1 = filename:join(PrivDir,Name1), + HConfig = #{disk_log_opts=>#{file=>LogFile1}, + filter_default=>log, + formatter=>{?MODULE,self()}}, + ok = logger:add_handler(Name1, logger_disk_log_h, HConfig), + {error,{already_exist,Name1}} = + logger:add_handler(Name1, logger_disk_log_h, #{}), + + %%! TODO: + %%! Check how bad log_opts are handled! + + {error,{illegal_config_change,_,_}} = + logger:set_handler_config(Name1, + disk_log_opts, + #{file=>LogFile1, + type=>halt}), + {error,{illegal_config_change,_,_}} = + logger:set_handler_config(Name1,id,new), + + ok = logger:remove_handler(Name1), + {error,{not_found,Name1}} = logger:remove_handler(Name1), + ok. + +errors(cleanup, _Config) -> + Name1 = list_to_atom(lists:concat([?FUNCTION_NAME,"_1"])), + _ = logger:remove_handler(Name1). + +formatter_fail(Config) -> + PrivDir = ?config(priv_dir,Config), + Name = ?FUNCTION_NAME, + LogFile = filename:join(PrivDir,Name), + ct:pal("Log = ~p", [LogFile]), + HConfig = #{disk_log_opts => #{file=>LogFile}, + filter_default=>stop, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE])}, + %% no formatter! + logger:add_handler(Name, logger_disk_log_h, HConfig), + Pid = whereis(Name), + true = is_pid(Pid), + {ok,#{handlers:=H}} = logger:get_logger_config(), + true = lists:member(Name,H), + + %% Formatter is added automatically + {ok,{_,#{formatter:={logger_formatter,_}}}} = + logger:get_handler_config(Name), + logger:info(M1=?msg,?domain), + Got1 = try_match_file(?log_no(LogFile,1),"=INFO REPORT====.*\n"++M1,5000), + + ok = logger:set_handler_config(Name,formatter,{nonexistingmodule,#{}}), + logger:info(M2=?msg,?domain), + Got2 = try_match_file(?log_no(LogFile,1), + Got1++"=INFO REPORT====.*\nFORMATTER CRASH: .*"++M2, + 5000), + + ok = logger:set_handler_config(Name,formatter,{?MODULE,crash}), + logger:info(M3=?msg,?domain), + Got3 = try_match_file(?log_no(LogFile,1), + Got2++"=INFO REPORT====.*\nFORMATTER CRASH: .*"++M3, + 5000), + + ok = logger:set_handler_config(Name,formatter,{?MODULE,bad_return}), + logger:info(?msg,?domain), + try_match_file(?log_no(LogFile,1), + Got3++"FORMATTER ERROR: bad_return_value", + 5000), + + %% Check that handler is still alive and was never dead + Pid = whereis(Name), + {ok,#{handlers:=H}} = logger:get_logger_config(), + ok. + +formatter_fail(cleanup,_Config) -> + _ = logger:remove_handler(?FUNCTION_NAME), + ok. + +config_fail(_Config) -> + {error,{handler_not_added,{invalid_config,logger_disk_log_h,{bad,bad}}}} = + logger:add_handler(?MODULE,logger_disk_log_h, + #{logger_disk_log_h => #{bad => bad}, + filter_default=>log, + formatter=>{?MODULE,self()}}), + {error,{handler_not_added,{invalid_levels,{42,42,_}}}} = + logger:add_handler(?MODULE,logger_disk_log_h, + #{logger_disk_log_h => #{toggle_sync_qlen=>42, + drop_new_reqs_qlen=>42}}), + + ok = logger:add_handler(?MODULE,logger_disk_log_h, + #{filter_default=>log, + formatter=>{?MODULE,self()}}), + %% can't change the disk log options for a log already in use + {error,{illegal_config_change,_,_}} = + logger:set_handler_config(?MODULE,disk_log_opts, + #{max_no_files=>2}), + %% can't change name of an existing handler + {error,{illegal_config_change,_,_}} = + logger:set_handler_config(?MODULE,id,bad), + %% incorrect values of OP params + {error,{invalid_levels,_}} = + logger:set_handler_config(?MODULE,logger_disk_log_h, + #{toggle_sync_qlen=>100, + flush_reqs_qlen=>99}), + %% invalid name of config parameter + {error,{invalid_config,logger_disk_log_h,{filesync_rep_int,2000}}} = + logger:set_handler_config(?MODULE, logger_disk_log_h, + #{filesync_rep_int => 2000}), + ok. +config_fail(cleanup,_Config) -> + logger:remove_handler(?MODULE). + +bad_input(_Config) -> + {error,{badarg,{disk_log_sync,["BadType"]}}} = + logger_disk_log_h:disk_log_sync("BadType"), + {error,{badarg,{info,["BadType"]}}} = logger_disk_log_h:info("BadType"), + {error,{badarg,{reset,["BadType"]}}} = logger_disk_log_h:reset("BadType"). + +info_and_reset(_Config) -> + ok = logger:add_handler(?MODULE,logger_disk_log_h, + #{filter_default=>log, + formatter=>{?MODULE,self()}}), + #{id := ?MODULE} = logger_disk_log_h:info(?MODULE), + ok = logger_disk_log_h:reset(?MODULE). +info_and_reset(cleanup,_Config) -> + logger:remove_handler(?MODULE). + +reconfig(Config) -> + Dir = ?config(priv_dir,Config), + ok = logger:add_handler(?MODULE, + logger_disk_log_h, + #{filter_default=>log, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), + formatter=>{?MODULE,self()}}), + #{id := ?MODULE, + toggle_sync_qlen := ?TOGGLE_SYNC_QLEN, + drop_new_reqs_qlen := ?DROP_NEW_REQS_QLEN, + flush_reqs_qlen := ?FLUSH_REQS_QLEN, + enable_burst_limit := ?ENABLE_BURST_LIMIT, + burst_limit_size := ?BURST_LIMIT_SIZE, + burst_window_time := ?BURST_WINDOW_TIME, + enable_kill_overloaded := ?ENABLE_KILL_OVERLOADED, + handler_overloaded_qlen := ?HANDLER_OVERLOADED_QLEN, + handler_overloaded_mem := ?HANDLER_OVERLOADED_MEM, + handler_restart_after := ?HANDLER_RESTART_AFTER, + filesync_repeat_interval := ?FILESYNC_REPEAT_INTERVAL, + log_opts := #{type := ?DISK_LOG_TYPE, + max_no_files := ?DISK_LOG_MAX_NO_FILES, + max_no_bytes := ?DISK_LOG_MAX_NO_BYTES, + file := _DiskLogFile}} = + logger_disk_log_h:info(?MODULE), + + ok = logger:set_handler_config(?MODULE, logger_disk_log_h, + #{toggle_sync_qlen => 1, + drop_new_reqs_qlen => 2, + flush_reqs_qlen => 3, + enable_burst_limit => false, + burst_limit_size => 10, + burst_window_time => 10, + enable_kill_overloaded => true, + handler_overloaded_qlen => 100000, + handler_overloaded_mem => 10000000, + handler_restart_after => never, + filesync_repeat_interval => no_repeat}), + #{id := ?MODULE, + toggle_sync_qlen := 1, + drop_new_reqs_qlen := 2, + flush_reqs_qlen := 3, + enable_burst_limit := false, + burst_limit_size := 10, + burst_window_time := 10, + enable_kill_overloaded := true, + handler_overloaded_qlen := 100000, + handler_overloaded_mem := 10000000, + handler_restart_after := never, + filesync_repeat_interval := no_repeat} = + logger_disk_log_h:info(?MODULE), + + ok = logger:remove_handler(?MODULE), + + File = filename:join(Dir, "logfile"), + ok = logger:add_handler(?MODULE, + logger_disk_log_h, + #{filter_default=>log, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), + formatter=>{?MODULE,self()}, + disk_log_opts=> + #{type => halt, + max_no_files => 1, + max_no_bytes => 1024, + file => File}}), + #{log_opts := #{type := halt, + max_no_files := 1, + max_no_bytes := 1024, + file := File}} = + logger_disk_log_h:info(?MODULE), + ok. + +reconfig(cleanup, _Config) -> + logger:remove_handler(?MODULE). + +disk_log_sync(Config) -> + Dir = ?config(priv_dir,Config), + File = filename:join(Dir, ?FUNCTION_NAME), + Log = lists:concat([File,".1"]), + ok = logger:add_handler(?MODULE, + logger_disk_log_h, + #{disk_log_opts => #{file => File}, + filter_default=>log, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), + formatter=>{?MODULE,nl}}), + + start_tracer([{?MODULE,format,2}, + {disk_log,blog,2}, + {disk_log,sync,1}], + [{formatter,"first"}, + {disk_log,blog}, + {disk_log,sync}]), + + logger:info("first", ?domain), + %% wait for automatic disk_log_sync + check_tracer(?FILESYNC_REPEAT_INTERVAL*2), + + start_tracer([{?MODULE,format,2}, + {disk_log,blog,2}, + {disk_log,sync,1}], + [{formatter,"second"}, + {formatter,"third"}, + {disk_log,blog}, + {disk_log,blog}, + {disk_log,sync}]), + %% two log requests in fast succession will make the handler skip + %% an automatic disk log sync + logger:info("second", ?domain), + logger:info("third", ?domain), + %% do explicit disk_log_sync + logger_disk_log_h:disk_log_sync(?MODULE), + check_tracer(100), + + %% check that if there's no repeated disk_log_sync active, + %% a disk_log_sync is still performed when handler goes idle + logger:set_handler_config(?MODULE, logger_disk_log_h, + #{filesync_repeat_interval => no_repeat}), + no_repeat = maps:get(filesync_repeat_interval, + logger_disk_log_h:info(?MODULE)), + + start_tracer([{?MODULE,format,2}, + {disk_log,blog,2}, + {disk_log,sync,1}], + [{formatter,"fourth"}, + {disk_log,blog}, + {formatter,"fifth"}, + {disk_log,blog}, + {disk_log,sync}]), + + logger:info("fourth", ?domain), + timer:sleep(?IDLE_DETECT_TIME_MSEC*2), + logger:info("fifth", ?domain), + %% wait for automatic disk_log_sync + check_tracer(?IDLE_DETECT_TIME_MSEC*2), + + try_read_file(Log, {ok,<<"first\nsecond\nthird\nfourth\nfifth\n">>}, 1000), + + %% switch repeated disk_log_sync on and verify that the looping works + SyncInt = 1000, + WaitT = 4500, + OneSync = {logger_disk_log_h,handle_cast,repeated_disk_log_sync}, + %% receive 1 initial repeated_disk_log_sync, then 1 per sec + start_tracer([{logger_disk_log_h,handle_cast,2}], + [OneSync || _ <- lists:seq(1, 1 + trunc(WaitT/SyncInt))]), + + logger:set_handler_config(?MODULE, logger_disk_log_h, + #{filesync_repeat_interval => SyncInt}), + SyncInt = maps:get(filesync_repeat_interval, + logger_disk_log_h:info(?MODULE)), + timer:sleep(WaitT), + logger:set_handler_config(?MODULE, logger_disk_log_h, + #{filesync_repeat_interval => no_repeat}), + check_tracer(100), + ok. +disk_log_sync(cleanup,_Config) -> + logger:remove_handler(?MODULE). + +disk_log_wrap(Config) -> + Get = fun(Key, PL) -> proplists:get_value(Key, PL) end, + Dir = ?config(priv_dir,Config), + File = filename:join(Dir, ?FUNCTION_NAME), + ct:pal("Log = ~p", [File]), + MaxFiles = 3, + MaxBytes = 5, + ok = logger:add_handler(?MODULE, + logger_disk_log_h, + #{filter_default=>log, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), + formatter=>{?MODULE,self()}, + disk_log_opts=> + #{type => wrap, + max_no_files => MaxFiles, + max_no_bytes => MaxBytes, + file => File}}), + Info = disk_log:info(?MODULE), + {File,wrap,{MaxBytes,MaxFiles},1} = + {Get(file,Info),Get(type,Info),Get(size,Info),Get(current_file,Info)}, + Tester = self(), + TraceFun = fun({trace,_,call,{Mod,Func,Details}}, Pid) -> + Pid ! {trace,Mod,Func,Details}, + Pid + end, + {ok,_} = dbg:tracer(process, {TraceFun, Tester}), + {ok,_} = dbg:p(whereis(?MODULE), [c]), + {ok,_} = dbg:tp(logger_disk_log_h, handle_info, 2, []), + + Text = [34 + rand:uniform(126-34) || _ <- lists:seq(1,MaxBytes)], + ct:pal("String = ~p (~w)", [Text, erts_debug:size(Text)]), + %% fill first file + lists:foreach(fun(N) -> + Log = lists:concat([File,".",N]), + logger:info(Text, ?domain), + wait_until_written(Log), + ct:pal("N = ~w", + [N = Get(current_file, + disk_log:info(?MODULE))]) + end, lists:seq(1,MaxFiles)), + + %% wait for trace messages + timer:sleep(1000), + dbg:stop_clear(), + Received = lists:flatmap(fun({trace,_M,handle_info, + [{disk_log,_Node,_Name,What},_]}) -> + [{trace,What}]; + ({log,_}) -> + [] + end, test_server:messages_get()), + ct:pal("Trace =~n~p", [Received]), + Received = [{trace,{wrap,0}} || _ <- lists:seq(1,MaxFiles-1)], + ok. + +disk_log_wrap(cleanup,_Config) -> + logger:remove_handler(?MODULE). + +disk_log_full(Config) -> + Dir = ?config(priv_dir,Config), + File = filename:join(Dir, ?FUNCTION_NAME), + ct:pal("Log = ~p", [File]), + MaxBytes = 50, + ok = logger:add_handler(?MODULE, + logger_disk_log_h, + #{filter_default=>log, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), + formatter=>{?MODULE,self()}, + disk_log_opts=> + #{type => halt, + max_no_files => 1, + max_no_bytes => MaxBytes, + file => File}}), + + Tester = self(), + TraceFun = fun({trace,_,call,{Mod,Func,Details}}, Pid) -> + Pid ! {trace,Mod,Func,Details}, + Pid + end, + {ok,_} = dbg:tracer(process, {TraceFun, Tester}), + {ok,_} = dbg:p(whereis(?MODULE), [c]), + {ok,_} = dbg:tp(logger_disk_log_h, handle_info, 2, []), + + NoOfChars = 5, + Text = [34 + rand:uniform(126-34) || _ <- lists:seq(1,NoOfChars)], + [logger:info(Text, ?domain) || _ <- lists:seq(1,trunc(MaxBytes/NoOfChars)+1)], + + %% wait for trace messages + timer:sleep(2000), + dbg:stop_clear(), + Received = lists:flatmap(fun({trace,_M,handle_info, + [{disk_log,_Node,_Name,What},_]}) -> + [{trace,What}]; + ({log,_}) -> + [] + end, test_server:messages_get()), + ct:pal("Trace =~n~p", [Received]), + [{trace,full}, + {trace,{error_status,{error,{full,_}}}}] = Received, + ok. +disk_log_full(cleanup, _Config) -> + logger:remove_handler(?MODULE). + +disk_log_events(Config) -> + Node = node(), + Log = ?MODULE, + ok = logger:add_handler(?MODULE, + logger_disk_log_h, + #{filter_default=>log, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), + formatter=>{?MODULE,self()}}), + + %% Events copied from disk_log API + Events = + [{disk_log, Node, Log, {wrap, 0}}, + {disk_log, Node, Log, {truncated, 0}}, + {disk_log, Node, Log, {read_only, 42}}, + {disk_log, Node, Log, {blocked_log, 42}}, + {disk_log, Node, Log, {format_external, 42}}, + {disk_log, Node, Log, full}, + {disk_log, Node, Log, {error_status, ok}}], + + Tester = self(), + TraceFun = fun({trace,_,call,{Mod,Func,Details}}, Pid) -> + Pid ! {trace,Mod,Func,Details}, + Pid + end, + {ok,_} = dbg:tracer(process, {TraceFun, Tester}), + {ok,_} = dbg:p(whereis(?MODULE), [c]), + {ok,_} = dbg:tp(logger_disk_log_h, handle_info, 2, []), + + [whereis(?MODULE) ! E || E <- Events], + %% wait for trace messages + timer:sleep(2000), + dbg:stop_clear(), + Received = lists:map(fun({trace,_M,handle_info, + [Got,_]}) -> Got + end, test_server:messages_get()), + ct:pal("Trace =~n~p", [Received]), + NoOfEvents = length(Events), + NoOfEvents = length(Received), + lists:foreach(fun(Event) -> + true = lists:member(Event, Received) + end, Received), + ok. +disk_log_events(cleanup, _Config) -> + logger:remove_handler(?MODULE). + +write_failure(Config) -> + Dir = ?config(priv_dir, Config), + File = filename:join(Dir, ?FUNCTION_NAME), + Log = lists:concat([File,".1"]), + ct:pal("Log = ~p", [Log]), + + Node = start_h_on_new_node(Config, ?FUNCTION_NAME, File), + false = (undefined == rpc:call(Node, ets, whereis, [?TEST_HOOKS_TAB])), + rpc:call(Node, ets, insert, [?TEST_HOOKS_TAB,{tester,self()}]), + rpc:call(Node, ?MODULE, set_internal_log, [?MODULE,internal_log]), + rpc:call(Node, ?MODULE, set_result, [disk_log_blog,ok]), + HState = rpc:call(Node, logger_disk_log_h, info, [?STANDARD_HANDLER]), + ct:pal("LogOpts = ~p", [LogOpts = maps:get(log_opts, HState)]), + + ok = log_on_remote_node(Node, "Logged1"), + rpc:call(Node, logger_disk_log_h, disk_log_sync, [?STANDARD_HANDLER]), + ?check_no_log, + try_read_file(Log, {ok,<<"Logged1\n">>}, ?SYNC_REP_INT), + + rpc:call(Node, ?MODULE, set_result, [disk_log_blog,{error,no_such_log}]), + ok = log_on_remote_node(Node, "Cause simple error printout"), + + ?check({error,{?STANDARD_HANDLER,log,LogOpts,{error,no_such_log}}}), + + ok = log_on_remote_node(Node, "No second error printout"), + ?check_no_log, + + rpc:call(Node, ?MODULE, set_result, [disk_log_blog, + {error,{full,?STANDARD_HANDLER}}]), + ok = log_on_remote_node(Node, "Cause simple error printout"), + ?check({error,{?STANDARD_HANDLER,log,LogOpts, + {error,{full,?STANDARD_HANDLER}}}}), + + rpc:call(Node, ?MODULE, set_result, [disk_log_blog,ok]), + ok = log_on_remote_node(Node, "Logged2"), + rpc:call(Node, logger_disk_log_h, disk_log_sync, [?STANDARD_HANDLER]), + ?check_no_log, + try_read_file(Log, {ok,<<"Logged1\nLogged2\n">>}, ?SYNC_REP_INT), + ok. +write_failure(cleanup, _Config) -> + Nodes = nodes(), + [test_server:stop_node(Node) || Node <- Nodes]. + + +sync_failure(Config) -> + Dir = ?config(priv_dir, Config), + FileName = lists:concat([?MODULE,"_",?FUNCTION_NAME]), + File = filename:join(Dir, FileName), + Log = lists:concat([File,".1"]), + + Node = start_h_on_new_node(Config, ?FUNCTION_NAME, File), + false = (undefined == rpc:call(Node, ets, whereis, [?TEST_HOOKS_TAB])), + rpc:call(Node, ets, insert, [?TEST_HOOKS_TAB,{tester,self()}]), + rpc:call(Node, ?MODULE, set_internal_log, [?MODULE,internal_log]), + rpc:call(Node, ?MODULE, set_result, [disk_log_sync,ok]), + HState = rpc:call(Node, logger_disk_log_h, info, [?STANDARD_HANDLER]), + LogOpts = maps:get(log_opts, HState), + + SyncInt = 500, + ok = rpc:call(Node, logger, set_handler_config, + [?STANDARD_HANDLER, logger_disk_log_h, + #{filesync_repeat_interval => SyncInt}]), + Info = rpc:call(Node, logger_disk_log_h, info, [?STANDARD_HANDLER]), + SyncInt = maps:get(filesync_repeat_interval, Info), + + ok = log_on_remote_node(Node, "Logged1"), + ?check_no_log, + + rpc:call(Node, ?MODULE, set_result, [disk_log_sync,{error,no_such_log}]), + ok = log_on_remote_node(Node, "Cause simple error printout"), + + ?check({error,{?STANDARD_HANDLER,sync,LogOpts,{error,no_such_log}}}), + + ok = log_on_remote_node(Node, "No second error printout"), + ?check_no_log, + + rpc:call(Node, ?MODULE, set_result, + [disk_log_sync,{error,{blocked_log,?STANDARD_HANDLER}}]), + ok = log_on_remote_node(Node, "Cause simple error printout"), + ?check({error,{?STANDARD_HANDLER,sync,LogOpts, + {error,{blocked_log,?STANDARD_HANDLER}}}}), + + rpc:call(Node, ?MODULE, set_result, [disk_log_sync,ok]), + ok = log_on_remote_node(Node, "Logged2"), + ?check_no_log, + ok. +sync_failure(cleanup, _Config) -> + Nodes = nodes(), + [test_server:stop_node(Node) || Node <- Nodes]. + +start_h_on_new_node(_Config, Func, File) -> + Pa = filename:dirname(code:which(?MODULE)), + Dest = + case os:type() of + {win32,_} -> + lists:concat([" {disk_log,\\\"",File,"\\\"}"]); + _ -> + lists:concat([" \'{disk_log,\"",File,"\"}\'"]) + end, + Args = lists:concat([" -kernel ",logger_dest,Dest," -pa ",Pa]), + NodeName = lists:concat([?MODULE,"_",Func]), + ct:pal("Starting ~s with ~tp", [NodeName,Args]), + {ok,Node} = test_server:start_node(NodeName, peer, [{args, Args}]), + Pid = rpc:call(Node,erlang,whereis,[?STANDARD_HANDLER]), + true = is_pid(Pid), + ok = rpc:call(Node,logger,set_handler_config,[?STANDARD_HANDLER,formatter, + {?MODULE,nl}]), + Node. + +log_on_remote_node(Node,Msg) -> + _ = spawn_link(Node, + fun() -> erlang:group_leader(whereis(user),self()), + logger:info(Msg) + end), + ok. + +%% functions for test hook macros to be called by rpc +set_internal_log(Mod, Func) -> + ?set_internal_log({Mod,Func}). +set_result(Op, Result) -> + ?set_result(Op, Result). +set_defaults() -> + ?set_defaults(). + +%% internal log function that sends the term to the test case process +internal_log(Type, Term) -> + [{tester,Tester}] = ets:lookup(?TEST_HOOKS_TAB, tester), + Tester ! {log,{Type,Term}}, + logger:internal_log(Type, Term), + ok. + + +%%%----------------------------------------------------------------- +%%% Overload protection tests + +op_switch_to_sync(Config) -> + {Log,HConfig,DLHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + NewHConfig = + HConfig#{logger_disk_log_h => DLHConfig#{toggle_sync_qlen => 3, + drop_new_reqs_qlen => 501, + flush_reqs_qlen => 2000, + enable_burst_limit => false}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + NumOfReqs = 500, + send_burst({n,NumOfReqs}, seq, {chars,79}, info), + NumOfReqs = count_lines(Log), + ok = file:delete(Log). +op_switch_to_sync(cleanup, _Config) -> + ok = stop_handler(?MODULE). + +op_switch_to_drop(Config) -> + {Log,HConfig,DLHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + + NewHConfig = + HConfig#{logger_disk_log_h => DLHConfig#{toggle_sync_qlen => 2, + drop_new_reqs_qlen => 3, + flush_reqs_qlen => 600, + enable_burst_limit => false}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + NumOfReqs = 500, + send_burst({n,NumOfReqs}, seq, {chars,79}, info), + Logged = count_lines(Log), + ct:pal("Number of messages dropped = ~w (~w)", + [NumOfReqs-Logged,NumOfReqs]), + true = (Logged < NumOfReqs), + ok = file:delete(Log). +op_switch_to_drop(cleanup, _Config) -> + ok = stop_handler(?MODULE). + +op_switch_to_flush() -> + [{timetrap,{seconds,60}}]. +op_switch_to_flush(Config) -> + {Log,HConfig,DLHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + + %% it's important that both async and sync requests have been queued + %% when the flush happens (verify with coverage of flush_log_requests/2) + + NewHConfig = + HConfig#{logger_disk_log_h => DLHConfig#{toggle_sync_qlen => 2, + drop_new_reqs_qlen => 99, + flush_reqs_qlen => 100, + enable_burst_limit => false}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + NumOfReqs = 1000, + Procs = 500, + send_burst({n,NumOfReqs}, {spawn,Procs,0}, {chars,79}, info), + Logged = count_lines(Log), + ct:pal("Number of messages flushed/dropped = ~w (~w)", + [(NumOfReqs*Procs)-Logged,NumOfReqs*Procs]), + true = (Logged < (NumOfReqs*Procs)), + ok = file:delete(Log). +op_switch_to_flush(cleanup, _Config) -> + ok = stop_handler(?MODULE). + + +limit_burst_disabled(Config) -> + {Log,HConfig,DLHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + NewHConfig = + HConfig#{logger_disk_log_h => DLHConfig#{enable_burst_limit => false, + burst_limit_size => 10, + burst_window_time => 2000, + drop_new_reqs_qlen => 200, + flush_reqs_qlen => 300}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + NumOfReqs = 100, + send_burst({n,NumOfReqs}, seq, {chars,79}, info), + Logged = count_lines(Log), + ct:pal("Number of messages logged = ~w", [Logged]), + ok = file:delete(Log), + NumOfReqs = Logged. +limit_burst_disabled(cleanup, _Config) -> + ok = stop_handler(?MODULE). + +limit_burst_enabled_one(Config) -> + {Log,HConfig,DLHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + ReqLimit = 10, + NewHConfig = + HConfig#{logger_disk_log_h => DLHConfig#{enable_burst_limit => true, + burst_limit_size => ReqLimit, + burst_window_time => 2000, + drop_new_reqs_qlen => 200, + flush_reqs_qlen => 300}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + NumOfReqs = 100, + send_burst({n,NumOfReqs}, seq, {chars,79}, info), + Logged = count_lines(Log), + ct:pal("Number of messages logged = ~w", [Logged]), + ok = file:delete(Log), + ReqLimit = Logged. +limit_burst_enabled_one(cleanup, _Config) -> + ok = stop_handler(?MODULE). + +limit_burst_enabled_period(Config) -> + {Log,HConfig,DLHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + ReqLimit = 10, + BurstTWin = 1000, + NewHConfig = + HConfig#{logger_disk_log_h => DLHConfig#{enable_burst_limit => true, + burst_limit_size => ReqLimit, + burst_window_time => BurstTWin, + drop_new_reqs_qlen => 20000, + flush_reqs_qlen => 20001}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + + Windows = 3, + Sent = send_burst({t,BurstTWin*Windows}, seq, {chars,79}, info), + Logged = count_lines(Log), + ct:pal("Number of messages sent = ~w~nNumber of messages logged = ~w", + [Sent,Logged]), + ok = file:delete(Log), + true = (Logged > (ReqLimit*Windows)) andalso + (Logged < (ReqLimit*(Windows+2))). +limit_burst_enabled_period(cleanup, _Config) -> + ok = stop_handler(?MODULE). + +kill_disabled(Config) -> + {Log,HConfig,DLHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + NewHConfig = + HConfig#{logger_disk_log_h=>DLHConfig#{enable_kill_overloaded=>false, + handler_overloaded_qlen=>10, + handler_overloaded_mem=>100}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + NumOfReqs = 100, + send_burst({n,NumOfReqs}, seq, {chars,79}, info), + Logged = count_lines(Log), + ct:pal("Number of messages logged = ~w", [Logged]), + ok = file:delete(Log), + true = is_pid(whereis(?MODULE)), + ok. +kill_disabled(cleanup, _Config) -> + ok = stop_handler(?MODULE). + +qlen_kill_new(Config) -> + {Log,HConfig,DLHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + Pid0 = whereis(?MODULE), + {_,Mem0} = process_info(Pid0, memory), + RestartAfter = 2000, + NewHConfig = + HConfig#{logger_disk_log_h => + DLHConfig#{enable_kill_overloaded=>true, + handler_overloaded_qlen=>10, + handler_overloaded_mem=>Mem0+50000, + handler_restart_after=>RestartAfter}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + MRef = erlang:monitor(process, Pid0), + NumOfReqs = 100, + Procs = 2, + send_burst({n,NumOfReqs}, {spawn,Procs,0}, {chars,79}, info), + %% send_burst({n,NumOfReqs}, seq, {chars,79}, info), + receive + {'DOWN', MRef, _, _, Info} -> + case Info of + {shutdown,{overloaded,?MODULE,QLen,Mem}} -> + ct:pal("Terminated with qlen = ~w, mem = ~w", [QLen,Mem]); + killed -> + ct:pal("Slow shutdown, handler process was killed!", []) + end, + timer:sleep(RestartAfter + 1000), + true = is_pid(whereis(?MODULE)), + ok + after + 5000 -> + Info = logger_disk_log_h:info(?MODULE), + ct:pal("Handler state = ~p", [Info]), + ct:fail("Handler not dead! It should not have survived this!") + end. +qlen_kill_new(cleanup, _Config) -> + ok = stop_handler(?MODULE). + +mem_kill_new(Config) -> + {Log,HConfig,DLHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + Pid0 = whereis(?MODULE), + {_,Mem0} = process_info(Pid0, memory), + RestartAfter = 2000, + NewHConfig = + HConfig#{logger_disk_log_h => + DLHConfig#{enable_kill_overloaded=>true, + handler_overloaded_qlen=>50000, + handler_overloaded_mem=>Mem0+500, + handler_restart_after=>RestartAfter}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + MRef = erlang:monitor(process, Pid0), + NumOfReqs = 100, + Procs = 2, + send_burst({n,NumOfReqs}, {spawn,Procs,0}, {chars,79}, info), + %% send_burst({n,NumOfReqs}, seq, {chars,79}, info), + receive + {'DOWN', MRef, _, _, Info} -> + case Info of + {shutdown,{overloaded,?MODULE,QLen,Mem}} -> + ct:pal("Terminated with qlen = ~w, mem = ~w", [QLen,Mem]); + killed -> + ct:pal("Slow shutdown, handler process was killed!", []) + end, + timer:sleep(RestartAfter * 2), + true = is_pid(whereis(?MODULE)), + ok + after + 5000 -> + Info = logger_disk_log_h:info(?MODULE), + ct:pal("Handler state = ~p", [Info]), + ct:fail("Handler not dead! It should not have survived this!") + end. +mem_kill_new(cleanup, _Config) -> + ok = stop_handler(?MODULE). + +restart_after(Config) -> + {Log,HConfig,DLHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + NewHConfig1 = + HConfig#{logger_disk_log_h=>DLHConfig#{enable_kill_overloaded=>true, + handler_overloaded_qlen=>10, + handler_restart_after=>never}}, + ok = logger:set_handler_config(?MODULE, NewHConfig1), + MRef1 = erlang:monitor(process, whereis(?MODULE)), + %% kill handler + send_burst({n,100}, {spawn,2,0}, {chars,79}, info), + receive + {'DOWN', MRef1, _, _, _Info1} -> + timer:sleep(?HANDLER_RESTART_AFTER + 1000), + undefined = whereis(?MODULE), + ok + after + 5000 -> + ct:fail("Handler not dead! It should not have survived this!") + end, + + {Log,_,_} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + RestartAfter = 2000, + NewHConfig2 = + HConfig#{logger_disk_log_h=>DLHConfig#{enable_kill_overloaded=>true, + handler_overloaded_qlen=>10, + handler_restart_after=>RestartAfter}}, + ok = logger:set_handler_config(?MODULE, NewHConfig2), + Pid0 = whereis(?MODULE), + MRef2 = erlang:monitor(process, Pid0), + %% kill handler + send_burst({n,100}, {spawn,2,0}, {chars,79}, info), + receive + {'DOWN', MRef2, _, _, _Info2} -> + timer:sleep(RestartAfter + 1000), + Pid1 = whereis(?MODULE), + true = is_pid(Pid1), + false = (Pid1 == Pid0), + ok + after + 5000 -> + ct:fail("Handler not dead! It should not have survived this!") + end, + ok. +restart_after(cleanup, _Config) -> + ok = stop_handler(?MODULE). + +%% send handler requests (filesync, info, reset, change_config) +%% during high load to verify that sync, dropping and flushing is +%% handled correctly. +handler_requests_under_load() -> + [{timetrap,{seconds,60}}]. +handler_requests_under_load(Config) -> + {Log,HConfig,DLHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + NewHConfig = + HConfig#{logger_disk_log_h => DLHConfig#{toggle_sync_qlen => 2, + drop_new_reqs_qlen => 1000, + flush_reqs_qlen => 2000, + enable_burst_limit => false}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + Pid = spawn_link(fun() -> send_requests(?MODULE, 1, [{disk_log_sync,[]}, + {info,[]}, + {reset,[]}, + {change_config,[]}]) + end), + Procs = 100, + Sent = Procs * send_burst({n,5000}, {spawn,Procs,10}, {chars,79}, info), + Pid ! {self(),finish}, + ReqResult = receive {Pid,Result} -> Result end, + Logged = count_lines(Log), + ct:pal("Number of messages sent = ~w~nNumber of messages logged = ~w", + [Sent,Logged]), + FindError = fun(Res) -> + [E || E <- Res, + is_tuple(E) andalso (element(1,E) == error)] + end, + Errors = [{Req,FindError(Res)} || {Req,Res} <- ReqResult], + NoOfReqs = lists:foldl(fun({_,Res}, N) -> N + length(Res) end, 0, ReqResult), + ct:pal("~w requests made. Errors: ~n~p", [NoOfReqs,Errors]), + ok = file:delete(Log). +handler_requests_under_load(cleanup, Config) -> + ok = stop_handler(?MODULE). + +send_requests(HName, TO, Reqs = [{Req,Res}|Rs]) -> + receive + {From,finish} -> + From ! {self(),Reqs} + after + TO -> + Result = + case Req of + change_config -> + logger:set_handler_config(HName, logger_disk_log_h, + #{enable_kill_overloaded => + false}); + Func -> + logger_disk_log_h:Func(HName) + end, + send_requests(HName, TO, Rs ++ [{Req,[Result|Res]}]) + end. + +%%%----------------------------------------------------------------- +%%% +start_handler(Name, FuncName, Config) -> + Dir = ?config(priv_dir,Config), + File = filename:join(Dir, FuncName), + ct:pal("Logging to ~tp", [File]), + ok = logger:add_handler(Name, + logger_disk_log_h, + #{disk_log_opts=>#{file => File, + max_no_files => 1, + max_no_bytes => 100000000}, + filter_default=>log, + filters=>?DEFAULT_HANDLER_FILTERS([Name]), + formatter=>{?MODULE,op}}), + {ok,{_,HConfig = #{logger_disk_log_h := DLHConfig}}} = + logger:get_handler_config(Name), + {lists:concat([File,".1"]),HConfig,DLHConfig}. + +stop_handler(Name) -> + ok = logger:remove_handler(Name), + ct:pal("Handler ~p stopped!", [Name]). + +send_burst(NorT, Type, {chars,Sz}, Class) -> + Text = [34 + rand:uniform(126-34) || _ <- lists:seq(1,Sz)], + case NorT of + {n,N} -> + %% process_flag(priority, high), + send_n_burst(N, Type, Text, Class), + %% process_flag(priority, normal), + N; + {t,T} -> + ct:pal("Sending messages sequentially for ~w ms", [T]), + T0 = erlang:monotonic_time(millisecond), + send_t_burst(T0, T, Text, Class, 0) + end. + +send_n_burst(0, _, _Text, _Class) -> + ok; +send_n_burst(N, seq, Text, Class) -> + ok = logger:Class(Text, ?domain), + send_n_burst(N-1, seq, Text, Class); +send_n_burst(N, {spawn,Ps,TO}, Text, Class) -> + ct:pal("~w processes each sending ~w messages", [Ps,N]), + PerProc = fun() -> + send_n_burst(N, seq, Text, Class) + end, + MRefs = [begin if TO == 0 -> ok; true -> timer:sleep(TO) end, + monitor(process,spawn_link(PerProc)) end || + _ <- lists:seq(1,Ps)], + lists:foreach(fun(MRef) -> + receive + {'DOWN', MRef, _, _, _} -> + ok + end + end, MRefs), + ct:pal("Message burst sent", []), + ok. + +send_t_burst(T0, T, Text, Class, N) -> + T1 = erlang:monotonic_time(millisecond), + if (T1-T0) > T -> + N; + true -> + ok = logger:Class(Text, ?domain), + send_t_burst(T0, T, Text, Class, N+1) + end. + +%%%----------------------------------------------------------------- +%%% Formatter callback +%%% Using this to send the formatted string back to the test case +%%% process - so it can check for logged events. +format(_,bad_return) -> + bad_return; +format(_,crash) -> + erlang:error(formatter_crashed); +format(#{msg:={report,R},meta:=#{report_cb:=Fun}}=Log,Config) -> + format(Log#{msg=>Fun(R)},Config); +format(#{msg:={string,String0}},no_nl) -> + String = unicode:characters_to_list(String0), + String; +format(#{msg:={string,String0}},nl) -> + String = unicode:characters_to_list(String0), + String++"\n"; +format(#{msg:={string,String0}},op) -> + String = unicode:characters_to_list(String0), + String++"\n"; +format(#{msg:={report,#{label:={supervisor,progress}}}},op) -> + ""; +format(#{msg:={report,#{label:={gen_server,terminate}}}},op) -> + ""; +format(#{msg:={report,#{label:={proc_lib,crash}}}},op) -> + ""; +format(#{msg:={F,A}},Pid) when is_list(F), is_list(A) -> + String = lists:flatten(io_lib:format(F,A)), + Pid ! {log,String}, + String++"\n"; +format(#{msg:={string,String0}},Pid) -> + String = unicode:characters_to_list(String0), + Pid ! {log,String}, + String++"\n"; +format(Msg,Tag) -> + Error = {unexpected_format,Msg,Tag}, + erlang:display(Error), + exit(Error). + +remove(Handler, LogName) -> + logger_disk_log_h:remove(Handler, LogName), + HState = #{log_names := Logs} = logger_disk_log_h:info(), + false = maps:is_key(LogName, HState), + false = lists:member(LogName, Logs), + false = logger_config:exist(logger, LogName), + {error,no_such_log} = disk_log:info(LogName), + ok. + +start_and_add(Name, Config, LogOpts) -> + ct:pal("Adding handler ~w with: ~p", + [Name,Config#{disk_log_opts=>LogOpts}]), + ok = logger:add_handler(Name, logger_disk_log_h, + Config#{disk_log_opts=>LogOpts}), + Pid = whereis(Name), + true = is_pid(Pid), + Name = proplists:get_value(name, disk_log:info(Name)), + ok. + +remove_and_stop(Handler) -> + ok = logger:remove_handler(Handler), + timer:sleep(500), + undefined = whereis(Handler), + ok. + +try_read_file(FileName, Expected, Time) -> + try_read_file(FileName, Expected, Time, undefined). + +try_read_file(FileName, Expected, Time, _) when Time > 0 -> + case file:read_file(FileName) of + Expected -> + ok; + Error = {error,_Reason} -> + erlang:error(Error); + SomethingElse -> + ct:pal("try_read_file read unexpected: ~p~n", [SomethingElse]), + timer:sleep(500), + try_read_file(FileName, Expected, Time-500, SomethingElse) + end; + +try_read_file(_, _, _, Incorrect) -> + ct:pal("try_read_file got incorrect pattern: ~p~n", [Incorrect]), + erlang:error({error,not_matching_pattern,Incorrect}). + +try_match_file(FileName, Pattern, Time) -> + try_match_file(FileName, Pattern, Time, <<>>). + +try_match_file(FileName, Pattern, Time, _) when Time > 0 -> + case file:read_file(FileName) of + {ok, Bin} -> + case re:run(Bin,Pattern,[{capture,none}]) of + match -> + unicode:characters_to_list(Bin); + _ -> + timer:sleep(100), + try_match_file(FileName, Pattern, Time-100, Bin) + end; + Error -> + erlang:error(Error) + end; +try_match_file(_,Pattern,_,Incorrect) -> + ct:pal("try_match_file did not match pattern: ~p~nGot: ~p~n", + [Pattern,Incorrect]), + erlang:error({error,not_matching_pattern,Pattern,Incorrect}). + +count_lines(File) -> + wait_until_written(File), + count_lines1(File). + +wait_until_written(File) -> + wait_until_written(File, -1). + +wait_until_written(File, Sz) -> + timer:sleep(2000), + case file:read_file_info(File) of + {ok,#file_info{size = Sz}} -> + timer:sleep(1000), + case file:read_file_info(File) of + {ok,#file_info{size = Sz1}} -> + ok; + {ok,#file_info{size = Sz2}} -> + wait_until_written(File, Sz2) + end; + {ok,#file_info{size = Sz1}} -> + wait_until_written(File, Sz1) + end. + +count_lines1(File) -> + Counter = fun Cnt(Dev,LC) -> + case file:read_line(Dev) of + eof -> LC; + _ -> Cnt(Dev,LC+1) + end + end, + {_,Dev} = file:open(File, [read]), + Lines = Counter(Dev, 0), + file:close(Dev), + Lines. + +start_tracer(Trace,Expected) -> + Pid = self(), + dbg:tracer(process,{fun tracer/2,{Pid,Expected}}), + dbg:p(whereis(?MODULE),[c]), + dbg:p(Pid,[c]), + tpl(Trace), + ok. + +tpl([{M,F,A}|Trace]) -> + {ok,Match} = dbg:tpl(M,F,A,[]), + case lists:keyfind(matched,1,Match) of + {_,_,1} -> + ok; + _ -> + dbg:stop_clear(), + throw({skip,"Can't trace "++atom_to_list(M)++":"++ + atom_to_list(F)++"/"++integer_to_list(A)}) + end, + tpl(Trace); +tpl([]) -> + ok. + +tracer({trace,_,call,{?MODULE,format,[#{msg:={string,Msg}}|_]}}, {Pid,[{formatter,Msg}|Expected]}) -> + maybe_tracer_done(Pid,Expected,{formatter,Msg}); +tracer({trace,_,call,{logger_disk_log_h,handle_cast,[{Op,_}|_]}}, {Pid,[{Mod,Func,Op}|Expected]}) -> + maybe_tracer_done(Pid,Expected,{Mod,Func,Op}); +tracer({trace,_,call,{Mod,Func,_}}, {Pid,[{Mod,Func}|Expected]}) -> + maybe_tracer_done(Pid,Expected,{Mod,Func}); +tracer({trace,_,call,Call}, {Pid,Expected}) -> + Pid ! {tracer_got_unexpected,Call,Expected}, + {Pid,Expected}. + +maybe_tracer_done(Pid,[],Got) -> + ct:log("Tracer got: ~p~n",[Got]), + Pid ! tracer_done; +maybe_tracer_done(Pid,Expected,Got) -> + ct:log("Tracer got: ~p~n",[Got]), + {Pid,Expected}. + +check_tracer(T) -> + receive + tracer_done -> + dbg:stop_clear(), + ok; + {tracer_got_unexpected,Got,Expected} -> + dbg:stop_clear(), + ct:fail({tracer_got_unexpected,Got,Expected}) + after T -> + ct:fail({timeout,tracer}) + end. diff --git a/lib/kernel/test/logger_env_var_SUITE.erl b/lib/kernel/test/logger_env_var_SUITE.erl new file mode 100644 index 0000000000..c2d3364701 --- /dev/null +++ b/lib/kernel/test/logger_env_var_SUITE.erl @@ -0,0 +1,451 @@ +% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(logger_env_var_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("kernel/include/logger.hrl"). +-include_lib("kernel/src/logger_internal.hrl"). + +-define(all_vars,[{kernel,logger_dest}, + {kernel,logger_level}, + {kernel,logger_log_progress}, + {kernel,logger_sasl_compatible}, + {kernel,error_logger}]). + +suite() -> + [{timetrap,{seconds,30}}]. + +init_per_suite(Config) -> + Env = [{App,Key,application:get_env(App,Key)} || {App,Key} <- ?all_vars], + Removed = cleanup(), + [{env,Env},{logger,Removed}|Config]. + +end_per_suite(Config) -> + [application:set_env(App,Key,Val) || + {App,Key,Val} <- ?config(env,Config), + Val =/= undefined], + Hs = ?config(logger,Config), + [ok = logger:add_handler(Id,Mod,C) || {Id,Mod,C} <- Hs], + ok. + +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(Case, Config) -> + try apply(?MODULE,Case,[cleanup,Config]) + catch error:undef -> ok + end, + cleanup(), + ok. + +groups() -> + []. + +all() -> + [default, + default_sasl_compatible, + dest_tty, + dest_tty_sasl_compatible, + dest_false, + dest_false_progress, + dest_false_sasl_compatible, + dest_silent, + dest_silent_sasl_compatible, + dest_file_old, + dest_file, + dest_disk_log, + %% disk_log_vars, % or test this in logger_disk_log_SUITE? + sasl_compatible_false, + sasl_compatible_false_no_progress, + sasl_compatible, + bad_dest%% , + %% bad_level, + %% bad_sasl_compatibility, + %% bad_progress + ]. + +default(Config) -> + {ok,{_Log,Hs}} = setup(Config,?FUNCTION_NAME, + undefined, + undefined, % dest + undefined, % level + undefined, % sasl comp (default=false) + undefined), % progress (default=false) + {logger_std_h,logger_std_h,StdC} = lists:keyfind(logger_std_h,1,Hs), + true = is_pid(whereis(logger_std_h)), + info = maps:get(level,StdC), + StdFilters = maps:get(filters,StdC), + {domain,{_,{log,prefix_of,[beam,erlang,otp,sasl]}}} = + lists:keyfind(domain,1,StdFilters), + true = lists:keymember(stop_progress,1,StdFilters), + false = lists:keymember(logger_simple,1,Hs), + false = lists:keymember(sasl_h,1,Hs), + false = is_pid(whereis(sasl_h)), + ok. + +default_sasl_compatible(Config) -> + {ok,{_Log,Hs}} = setup(Config,?FUNCTION_NAME, + undefined, + undefined, % dest + undefined, % level + true, % sasl comp (default=false) + undefined), % progress (default=false) + {logger_std_h,logger_std_h,StdC} = lists:keyfind(logger_std_h,1,Hs), + true = is_pid(whereis(logger_std_h)), + info = maps:get(level,StdC), + StdFilters = maps:get(filters,StdC), + {domain,{_,{log,prefix_of,[beam,erlang,otp]}}} = + lists:keyfind(domain,1,StdFilters), + false = lists:keymember(stop_progress,1,StdFilters), + false = lists:keymember(logger_simple,1,Hs), + true = lists:keymember(sasl_h,1,Hs), + true = is_pid(whereis(sasl_h)), + ok. + +dest_tty(Config) -> + {ok,{_Log,Hs}} = setup(Config,?FUNCTION_NAME, + logger_dest, + tty, % dest + undefined, % level + undefined, % sasl comp (default=false) + undefined), % progress (default=false) + {logger_std_h,logger_std_h,StdC} = lists:keyfind(logger_std_h,1,Hs), + true = is_pid(whereis(logger_std_h)), + info = maps:get(level,StdC), + StdFilters = maps:get(filters,StdC), + {domain,{_,{log,prefix_of,[beam,erlang,otp,sasl]}}} = + lists:keyfind(domain,1,StdFilters), + true = lists:keymember(stop_progress,1,StdFilters), + false = lists:keymember(logger_simple,1,Hs), + false = lists:keymember(sasl_h,1,Hs), + false = is_pid(whereis(sasl_h)), + ok. + +dest_tty_sasl_compatible(Config) -> + {ok,{_Log,Hs}} = setup(Config,?FUNCTION_NAME, + logger_dest, + tty, % dest + undefined, % level + true, % sasl comp (default=false) + undefined), % progress (default=false) + {logger_std_h,logger_std_h,StdC} = lists:keyfind(logger_std_h,1,Hs), + true = is_pid(whereis(logger_std_h)), + info = maps:get(level,StdC), + StdFilters = maps:get(filters,StdC), + {domain,{_,{log,prefix_of,[beam,erlang,otp]}}} = + lists:keyfind(domain,1,StdFilters), + false = lists:keymember(stop_progress,1,StdFilters), + false = lists:keymember(logger_simple,1,Hs), + true = lists:keymember(sasl_h,1,Hs), + true = is_pid(whereis(sasl_h)), + ok. + +dest_false(Config) -> + {ok,{_Log,Hs}} = setup(Config,?FUNCTION_NAME, + logger_dest, + false, % dest + notice, % level + undefined, % sasl comp (default=false) + undefined), % progress (default=false) + false = lists:keymember(logger_std_h,1,Hs), + {logger_simple,logger_simple,SimpleC} = lists:keyfind(logger_simple,1,Hs), + notice = maps:get(level,SimpleC), + SimpleFilters = maps:get(filters,SimpleC), + {domain,{_,{log,prefix_of,[beam,erlang,otp,sasl]}}} = + lists:keyfind(domain,1,SimpleFilters), + true = lists:keymember(stop_progress,1,SimpleFilters), + false = lists:keymember(sasl_h,1,Hs), + ok. + +dest_false_progress(Config) -> + {ok,{_Log,Hs}} = setup(Config,?FUNCTION_NAME, + logger_dest, + false, % dest + notice, % level + undefined, % sasl comp (default=false) + true), % progress (default=false) + false = lists:keymember(logger_std_h,1,Hs), + {logger_simple,logger_simple,SimpleC} = lists:keyfind(logger_simple,1,Hs), + notice = maps:get(level,SimpleC), + SimpleFilters = maps:get(filters,SimpleC), + {domain,{_,{log,prefix_of,[beam,erlang,otp,sasl]}}} = + lists:keyfind(domain,1,SimpleFilters), + false = lists:keymember(stop_progress,1,SimpleFilters), + false = lists:keymember(sasl_h,1,Hs), + ok. + +dest_false_sasl_compatible(Config) -> + {ok,{_Log,Hs}} = setup(Config,?FUNCTION_NAME, + logger_dest, + false, % dest + notice, % level + true, % sasl comp (default=false) + undefined), % progress (default=false) + false = lists:keymember(logger_std_h,1,Hs), + {logger_simple,logger_simple,SimpleC} = lists:keyfind(logger_simple,1,Hs), + notice = maps:get(level,SimpleC), + SimpleFilters = maps:get(filters,SimpleC), + {domain,{_,{log,prefix_of,[beam,erlang,otp]}}} = + lists:keyfind(domain,1,SimpleFilters), + false = lists:keymember(stop_progress,1,SimpleFilters), + true = lists:keymember(sasl_h,1,Hs), + true = is_pid(whereis(sasl_h)), + ok. + +dest_silent(Config) -> + {ok,{_Log,Hs}} = setup(Config,?FUNCTION_NAME, + logger_dest, + silent, % dest + undefined, % level + undefined, % sasl comp (default=false) + undefined), % progress (default=false) + false = lists:keymember(logger_std_h,1,Hs), + false = lists:keymember(logger_simple,1,Hs), + false = lists:keymember(sasl_h,1,Hs), + ok. + +dest_silent_sasl_compatible(Config) -> + {ok,{_Log,Hs}} = setup(Config,?FUNCTION_NAME, + logger_dest, + silent, % dest + undefined, % level + true, % sasl comp (default=false) + undefined), % progress (default=false) + false = lists:keymember(logger_std_h,1,Hs), + false = lists:keymember(logger_simple,1,Hs), + true = lists:keymember(sasl_h,1,Hs), + true = is_pid(whereis(sasl_h)), + ok. + + +dest_file_old(Config) -> + {ok,{Log,_Hs}} = setup(Config,?FUNCTION_NAME, + error_logger, + file, % dest + undefined, % level + undefined, % sasl comp (default=false) + undefined), % progress (default=false) + check_log(Log, + file, % dest + 0), % progress in std logger + ok. + + +dest_file(Config) -> + {ok,{Log,_Hs}} = setup(Config,?FUNCTION_NAME, + logger_dest, + file, % dest + undefined, % level + undefined, % sasl comp (default=false) + undefined), % progress (default=false) + check_log(Log, + file, % dest + 0), % progress in std logger + ok. + + +dest_disk_log(Config) -> + {ok,{Log,_Hs}} = setup(Config,?FUNCTION_NAME, + logger_dest, + disk_log, % dest + undefined, % level + undefined, % sasl comp (default=false) + undefined), % progress (default=false) + check_log(Log, + disk_log, % dest + 0), % progress in std logger + ok. + + +sasl_compatible_false(Config) -> + {ok,{Log,_Hs}} = setup(Config,?FUNCTION_NAME, + logger_dest, + file, % dest + undefined, % level + false, % sasl comp + true), % progress + check_log(Log, + file, % dest + 4), % progress in std logger + ok. + +sasl_compatible_false_no_progress(Config) -> + {ok,{Log,_Hs}} = setup(Config,?FUNCTION_NAME, + logger_dest, + file, % dest + undefined, % level + false, % sasl comp + false), % progress + check_log(Log, + file, % dest + 0), % progress in std logger + ok. + +sasl_compatible(Config) -> + {ok,{Log,_Hs}} = setup(Config,?FUNCTION_NAME, + logger_dest, + file, % dest + undefined, % level + true, % sasl comp + undefined), % progress + check_log(Log, + file, % dest + 0), % progress in std logger + ok. + +bad_dest(Config) -> + {error,{bad_config,{kernel,{logger_dest,baddest}}}} = + setup(Config,?FUNCTION_NAME, + logger_dest, + baddest, + undefined, + undefined, + undefined). + +bad_level(Config) -> + error = + setup(Config,?FUNCTION_NAME, + logger_dest, + tty, + badlevel, + undefined, + undefined). + +bad_sasl_compatibility(Config) -> + error = + setup(Config,?FUNCTION_NAME, + logger_dest, + tty, + info, + badcomp, + undefined). + +bad_progress(Config) -> + error = + setup(Config,?FUNCTION_NAME, + logger_dest, + tty, + info, + undefined, + badprogress). + +%%%----------------------------------------------------------------- +%%% Internal +setup(Config,Func,DestVar,Dest,Level,SaslComp,Progress) -> + ok = logger:add_handler(logger_simple,logger_simple, + #{filter_default=>log, + logger_simple=>#{buffer=>true}}), + Dir = ?config(priv_dir,Config), + File = lists:concat([?MODULE,"_",Func,".log"]), + Log = filename:join(Dir,File), + case Dest of + undefined -> + ok; + F when F==file; F==disk_log -> + application:set_env(kernel,DestVar,{Dest,Log}); + _ -> + application:set_env(kernel,DestVar,Dest) + end, + case Level of + undefined -> + ok; + _ -> + application:set_env(kernel,logger_level,Level) + end, + case SaslComp of + undefined -> + ok; + _ -> + application:set_env(kernel,logger_sasl_compatible,SaslComp) + end, + case Progress of + undefined -> + ok; + _ -> + application:set_env(kernel,logger_log_progress,Progress) + end, + case logger:setup_standard_handler() of + ok -> + application:start(sasl), + StdH = case Dest of + NoH when NoH==false; NoH==silent -> false; + _ -> true + end, + StdH = is_pid(whereis(?STANDARD_HANDLER)), + SaslH = if SaslComp -> true; + true -> false + end, + SaslH = is_pid(whereis(sasl_h)), + {ok,{Log,maps:get(handlers,logger:i())}}; + Error -> + Error + end. + +check_log(Log,Dest,NumProgress) -> + ok = logger:alert("dummy1"), + ok = logger:debug("dummy1"), + + %% Check that there are progress reports (supervisor and + %% application_controller) and an error report (the call above) in + %% the log. There should not be any info reports yet. + {ok,Bin1} = sync_and_read(Dest,Log), + ct:log("Log content:~n~s",[Bin1]), + match(Bin1,<<"PROGRESS REPORT">>,NumProgress), + match(Bin1,<<"ALERT REPORT">>,1), + match(Bin1,<<"INFO REPORT">>,0), + match(Bin1,<<"DEBUG REPORT">>,0), + + %% Then stop sasl and see that the info report from + %% application_controller is there + ok = application:stop(sasl), + {ok,Bin2} = sync_and_read(Dest,Log), + ct:log("Log content:~n~s",[Bin2]), + match(Bin2,<<"INFO REPORT">>,1), + match(Bin1,<<"DEBUG REPORT">>,0), + ok. + +match(Bin,Pattern,0) -> + nomatch = re:run(Bin,Pattern,[{capture,none}]); +match(Bin,Pattern,N) -> + {match,M} = re:run(Bin,Pattern,[{capture,all},global]), + N = length(M). + +sync_and_read(disk_log,Log) -> + logger_disk_log_h:disk_log_sync(?STANDARD_HANDLER), + file:read_file(Log ++ ".1"); +sync_and_read(file,Log) -> + logger_std_h:filesync(?STANDARD_HANDLER), + file:read_file(Log). + +cleanup() -> + application:stop(sasl), + [application:unset_env(App,Key) || {App,Key} <- ?all_vars], + #{handlers:=Hs0} = logger:i(), + Hs = lists:keydelete(cth_log_redirect,1,Hs0), + [ok = logger:remove_handler(Id) || {Id,_,_} <- Hs], + Hs. diff --git a/lib/kernel/test/logger_filters_SUITE.erl b/lib/kernel/test/logger_filters_SUITE.erl new file mode 100644 index 0000000000..21f14bbc02 --- /dev/null +++ b/lib/kernel/test/logger_filters_SUITE.erl @@ -0,0 +1,214 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(logger_filters_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-define(ndlog, + #{level=>info,msg=>{"Line: ~p",[?LINE]},meta=>#{}}). +-define(dlog(Domain), + #{level=>info,msg=>{"Line: ~p",[?LINE]},meta=>#{domain=>Domain}}). +-define(llog(Level), + #{level=>Level,msg=>{"Line: ~p",[?LINE]},meta=>#{}}). +-define(plog, + #{level=>info, + msg=>{report,#{label=>{?MODULE,progress}}}, + meta=>#{line=>?LINE}}). +-define(rlog(Node), + #{level=>info, + msg=>{"Line: ~p",[?LINE]}, + meta=>#{gl=>rpc:call(Node,erlang,whereis,[user])}}). + +-define(TRY(X), my_try(fun() -> X end)). + +suite() -> + [{timetrap,{seconds,30}}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(Case, Config) -> + try apply(?MODULE,Case,[cleanup,Config]) + catch error:undef -> ok + end, + ok. + +groups() -> + []. + +all() -> + [domain, + level, + progress, + remote_gl]. + +domain(_Config) -> + L1 = logger_filters:domain(L1=?dlog([]),{log,prefix_of,[]}), + stop = logger_filters:domain(?dlog([]),{stop,prefix_of,[]}), + L2 = logger_filters:domain(L2=?dlog([]),{log,starts_with,[]}), + stop = logger_filters:domain(?dlog([]),{stop,starts_with,[]}), + L3 = logger_filters:domain(L3=?dlog([]),{log,equals,[]}), + stop = logger_filters:domain(?dlog([]),{stop,equals,[]}), + ignore = logger_filters:domain(?dlog([]),{log,no_domain,[]}), + ignore = logger_filters:domain(?dlog([]),{stop,no_domain,[]}), + + L4 = logger_filters:domain(L4=?dlog([a]),{log,prefix_of,[a,b]}), + stop = logger_filters:domain(?dlog([a]),{stop,prefix_of,[a,b]}), + ignore = logger_filters:domain(?dlog([a]),{log,starts_with,[a,b]}), + ignore = logger_filters:domain(?dlog([a]),{stop,starts_with,[a,b]}), + ignore = logger_filters:domain(?dlog([a]),{log,equals,[a,b]}), + ignore = logger_filters:domain(?dlog([a]),{stop,equals,[a,b]}), + ignore = logger_filters:domain(?dlog([a]),{log,no_domain,[a,b]}), + ignore = logger_filters:domain(?dlog([a]),{stop,no_domain,[a,b]}), + + ignore = logger_filters:domain(?dlog([a,b]),{log,prefix_of,[a]}), + ignore = logger_filters:domain(?dlog([a,b]),{stop,prefix_of,[a]}), + L5 = logger_filters:domain(L5=?dlog([a,b]),{log,starts_with,[a]}), + stop = logger_filters:domain(?dlog([a,b]),{stop,starts_with,[a]}), + ignore = logger_filters:domain(?dlog([a,b]),{log,equals,[a]}), + ignore = logger_filters:domain(?dlog([a,b]),{stop,equals,[a]}), + ignore = logger_filters:domain(?dlog([a,b]),{log,no_domain,[a]}), + ignore = logger_filters:domain(?dlog([a,b]),{stop,no_domain,[a]}), + + ignore = logger_filters:domain(?ndlog,{log,prefix_of,[a]}), + ignore = logger_filters:domain(?ndlog,{stop,prefix_of,[a]}), + ignore = logger_filters:domain(?ndlog,{log,starts_with,[a]}), + ignore = logger_filters:domain(?ndlog,{stop,starts_with,[a]}), + ignore = logger_filters:domain(?ndlog,{log,equals,[a]}), + ignore = logger_filters:domain(?ndlog,{stop,equals,[a]}), + L6 = logger_filters:domain(L6=?ndlog,{log,no_domain,[a]}), + stop = logger_filters:domain(?ndlog,{stop,no_domain,[a]}), + + L7 = logger_filters:domain(L7=?dlog([a,b,c,d]),{log,prefix_of,[a,b,c,d]}), + stop = logger_filters:domain(?dlog([a,b,c,d]),{stop,prefix_of,[a,b,c,d]}), + L8 = logger_filters:domain(L8=?dlog([a,b,c,d]),{log,starts_with,[a,b,c,d]}), + stop = logger_filters:domain(?dlog([a,b,c,d]),{stop,starts_with,[a,b,c,d]}), + L9 = logger_filters:domain(L9=?dlog([a,b,c,d]),{log,equals,[a,b,c,d]}), + stop = logger_filters:domain(?dlog([a,b,c,d]),{stop,equals,[a,b,c,d]}), + ignore = logger_filters:domain(?dlog([a,b,c,d]),{log,no_domain,[a,b,c,d]}), + ignore = logger_filters:domain(?dlog([a,b,c,d]),{stop,no_domain,[a,b,c,d]}), + + %% A domain field in meta which is not a list is allowed by the + %% filter, but it will never match. + ignore = logger_filters:domain(?dlog(dummy),{log,prefix_of,[a,b,c,d]}), + ignore = logger_filters:domain(?dlog(dummy),{stop,prefix_of,[a,b,c,d]}), + ignore = logger_filters:domain(?dlog(dummy),{log,starts_with,[a,b,c,d]}), + ignore = logger_filters:domain(?dlog(dummy),{stop,starts_with,[a,b,c,d]}), + ignore = logger_filters:domain(?dlog(dummy),{log,equals,[a,b,c,d]}), + ignore = logger_filters:domain(?dlog(dummy),{stop,equals,[a,b,c,d]}), + ignore = logger_filters:domain(?dlog(dummy),{log,no_domain,[a,b,c,d]}), + ignore = logger_filters:domain(?dlog(dummy),{stop,no_domain,[a,b,c,d]}), + + {error,badarg} = ?TRY(logger_filters:domain(?ndlog,bad)), + {error,badarg} = ?TRY(logger_filters:domain(?ndlog,{bad,prefix_of,[]})), + {error,badarg} = ?TRY(logger_filters:domain(?ndlog,{log,bad,[]})), + {error,badarg} = ?TRY(logger_filters:domain(?ndlog,{log,prefix_of,bad})), + + ok. + +level(_Config) -> + ignore = logger_filters:level(?llog(info),{log,lt,info}), + ignore = logger_filters:level(?llog(info),{stop,lt,info}), + ignore = logger_filters:level(?llog(info),{log,gt,info}), + ignore = logger_filters:level(?llog(info),{stop,gt,info}), + L1 = logger_filters:level(L1=?llog(info),{log,lteq,info}), + stop = logger_filters:level(?llog(info),{stop,lteq,info}), + L2 = logger_filters:level(L2=?llog(info),{log,gteq,info}), + stop = logger_filters:level(?llog(info),{stop,gteq,info}), + L3 = logger_filters:level(L3=?llog(info),{log,eq,info}), + stop = logger_filters:level(?llog(info),{stop,eq,info}), + ignore = logger_filters:level(?llog(info),{log,neq,info}), + ignore = logger_filters:level(?llog(info),{stop,neq,info}), + + ignore = logger_filters:level(?llog(error),{log,lt,info}), + ignore = logger_filters:level(?llog(error),{stop,lt,info}), + L4 = logger_filters:level(L4=?llog(error),{log,gt,info}), + stop = logger_filters:level(?llog(error),{stop,gt,info}), + ignore = logger_filters:level(?llog(error),{log,lteq,info}), + ignore = logger_filters:level(?llog(error),{stop,lteq,info}), + L5 = logger_filters:level(L5=?llog(error),{log,gteq,info}), + stop = logger_filters:level(?llog(error),{stop,gteq,info}), + ignore = logger_filters:level(?llog(error),{log,eq,info}), + ignore = logger_filters:level(?llog(error),{stop,eq,info}), + L6 = logger_filters:level(L6=?llog(error),{log,neq,info}), + stop = logger_filters:level(?llog(error),{stop,neq,info}), + + L7 = logger_filters:level(L7=?llog(info),{log,lt,error}), + stop = logger_filters:level(?llog(info),{stop,lt,error}), + ignore = logger_filters:level(?llog(info),{log,gt,error}), + ignore = logger_filters:level(?llog(info),{stop,gt,error}), + L8 = logger_filters:level(L8=?llog(info),{log,lteq,error}), + stop = logger_filters:level(?llog(info),{stop,lteq,error}), + ignore = logger_filters:level(?llog(info),{log,gteq,error}), + ignore = logger_filters:level(?llog(info),{stop,gteq,error}), + ignore = logger_filters:level(?llog(info),{log,eq,error}), + ignore = logger_filters:level(?llog(info),{stop,eq,error}), + L9 = logger_filters:level(L9=?llog(info),{log,neq,error}), + stop = logger_filters:level(?llog(info),{stop,neq,error}), + + {error,badarg} = ?TRY(logger_filters:level(?llog(info),bad)), + {error,badarg} = ?TRY(logger_filters:level(?llog(info),{bad,eq,info})), + {error,badarg} = ?TRY(logger_filters:level(?llog(info),{log,bad,info})), + {error,badarg} = ?TRY(logger_filters:level(?llog(info),{log,eq,bad})), + + ok. + +progress(_Config) -> + L1 = logger_filters:progress(L1=?plog,log), + stop = logger_filters:progress(?plog,stop), + ignore = logger_filters:progress(?ndlog,log), + ignore = logger_filters:progress(?ndlog,stop), + + {error,badarg} = ?TRY(logger_filters:progress(?plog,bad)), + + ok. + +remote_gl(_Config) -> + {ok,Node} = test_server:start_node(?FUNCTION_NAME,slave,[]), + L1 = logger_filters:remote_gl(L1=?rlog(Node),log), + stop = logger_filters:remote_gl(?rlog(Node),stop), + ignore = logger_filters:remote_gl(?ndlog,log), + ignore = logger_filters:remote_gl(?ndlog,stop), + + {error,badarg} = ?TRY(logger_filters:remote_gl(?rlog(Node),bad)), + ok. + +remote_gl(cleanup,_Config) -> + [test_server:stop_node(N) || N<-nodes()]. + +%%%----------------------------------------------------------------- +%%% Called by macro ?TRY(X) +my_try(Fun) -> + try Fun() catch C:R -> {C,R} end. diff --git a/lib/kernel/test/logger_formatter_SUITE.erl b/lib/kernel/test/logger_formatter_SUITE.erl new file mode 100644 index 0000000000..ac1abba629 --- /dev/null +++ b/lib/kernel/test/logger_formatter_SUITE.erl @@ -0,0 +1,558 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(logger_formatter_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-define(TRY(X), my_try(fun() -> X end)). + +suite() -> + [{timetrap,{seconds,30}}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(Case, Config) -> + try apply(?MODULE,Case,[cleanup,Config]) + catch error:undef -> ok + end, + ok. + +groups() -> + []. + +all() -> + [default, + legacy_header, + single_line, + template, + format_msg, + report_cb, + max_size, + depth, + chars_limit, + format_mfa, + format_time, + level_or_msg_in_meta, + faulty_log, + faulty_config, + faulty_msg]. + +default(_Config) -> + String1 = format(info,{"~p",[term]},#{},#{}), + ct:log(String1), + [_Date,_Time,"info:\nterm\n"] = string:lexemes(String1," "), + + Time = timestamp(), + ExpectedTimestamp = default_time_format(Time), + String2 = format(info,{"~p",[term]},#{time=>Time},#{}), + ct:log(String2), + " info:\nterm\n" = string:prefix(String2,ExpectedTimestamp), + ok. + +legacy_header(_Config) -> + Time = timestamp(), + String1 = format(info,{"~p",[term]},#{time=>Time},#{legacy_header=>true}), + ct:log(String1), + "=INFO REPORT==== "++Rest = String1, + [Timestamp,"\nterm\n"] = string:lexemes(Rest," ="), + [D,M,Y,H,Min,S,Micro] = string:lexemes(Timestamp,"-:."), + integer(D,31), + integer(Y,2018,infinity), + integer(H,23), + integer(Min,59), + integer(S,59), + integer(Micro,999999), + true = lists:member(M,["Jan","Feb","Mar","Apr","May","Jun", + "Jul","Aug","Sep","Oct","Nov","Dec"]), + + String2 = format(info,{"~p",[term]},#{time=>Time},#{legacy_header=>false}), + ct:log(String2), + ExpectedTimestamp = default_time_format(Time), + " info:\nterm\n" = string:prefix(String2,ExpectedTimestamp), + + String3 = format(info,{"~p",[term]},#{time=>Time},#{legacy_header=>bad}), + ct:log(String3), + String3 = String2, + + String4 = format(info,{"~p",[term]},#{time=>Time}, + #{legacy_header=>true, + single_line=>true}), % <---ignored + ct:log(String4), + String4 = String1, + + String5 = format(info,{"~p",[term]},#{}, % <--- no time + #{legacy_header=>true}), + ct:log(String5), + "=INFO REPORT==== "++_ = String5, + ok. + +single_line(_Config) -> + Time = timestamp(), + ExpectedTimestamp = default_time_format(Time), + String1 = format(info,{"~p",[term]},#{time=>Time},#{single_line=>true}), + ct:log(String1), + " info: term\n" = string:prefix(String1,ExpectedTimestamp), + + String2 = format(info,{"~p",[term]},#{time=>Time},#{single_line=>false}), + ct:log(String2), + " info:\nterm\n" = string:prefix(String2,ExpectedTimestamp), + + String2 = format(info,{"~p",[term]},#{time=>Time},#{single_line=>bad}), + ok. + +template(_Config) -> + Time = timestamp(), + + Template1 = [msg], + String1 = format(info,{"~p",[term]},#{time=>Time},#{template=>Template1}), + ct:log(String1), + "term" = String1, + + Template2 = [msg,unknown], + String2 = format(info,{"~p",[term]},#{time=>Time},#{template=>Template2}), + ct:log(String2), + "term" = String2, + + Template3 = ["string"], + String3 = format(info,{"~p",[term]},#{time=>Time},#{template=>Template3}), + ct:log(String3), + "string" = String3, + + Template4 = ["string\nnewline"], + String4 = format(info,{"~p",[term]},#{time=>Time},#{template=>Template4, + single_line=>true}), + ct:log(String4), + "string\nnewline" = String4, + + Template5 = [], + String5 = format(info,{"~p",[term]},#{time=>Time},#{template=>Template5}), + ct:log(String5), + "" = String5, + + Ref6 = erlang:make_ref(), + Meta6 = #{atom=>some_atom, + integer=>632, + list=>[list,"string",4321,#{},{tuple}], + mfa=>{mod,func,0}, + pid=>self(), + ref=>Ref6, + string=>"some string", + time=>Time, + tuple=>{1,atom,"list"}, + nested=>#{subkey=>subvalue}}, + Template6 = lists:join(";",maps:keys(maps:remove(nested,Meta6)) ++ + [{nested,subkey}]), + String6 = format(info,{"~p",[term]},Meta6,#{template=>Template6, + single_line=>true}), + ct:log(String6), + SelfStr = pid_to_list(self()), + RefStr6 = ref_to_list(Ref6), + ListStr = "[list,\"string\",4321,#{},{tuple}]", + ExpectedTime6 = default_time_format(Time), + ["some_atom", + "632", + ListStr, + "mod:func/0", + SelfStr, + RefStr6, + "some string", + ExpectedTime6, + "{1,atom,\"list\"}", + "subvalue"] = string:lexemes(String6,";"), + + Meta7 = #{time=>Time, + nested=>#{key1=>#{subkey1=>value1}, + key2=>value2}}, + Template7 = lists:join(";",[nested, + {nested,key1}, + {nested,key1,subkey1}, + {nested,key2}, + {nested,key2,subkey2}, + {nested,key3}, + {nested,key3,subkey3}]), + String7 = format(info,{"~p",[term]},Meta7,#{template=>Template7, + single_line=>true}), + ct:log(String7), + [MultipleKeysStr, + "#{subkey1 => value1}", + "value1", + "value2", + "", + "", + ""] = string:split(String7,";",all), + %% Order of keys is not fixed + case MultipleKeysStr of + "#{key2 => value2,key1 => #{subkey1 => value1}}" -> ok; + "#{key1 => #{subkey1 => value1},key2 => value2}" -> ok; + _ -> ct:fail({full_nested_map_unexpected,MultipleKeysStr}) + end, + ok. + +format_msg(_Config) -> + Template = [msg], + + String1 = format(info,{"~p",[term]},#{},#{template=>Template}), + ct:log(String1), + "term" = String1, + + String2 = format(info,{"list",[term]},#{},#{template=>Template}), + ct:log(String2), + "FORMAT ERROR: \"list\" - [term]" = String2, + + String3 = format(info,{report,term},#{},#{template=>Template}), + ct:log(String3), + "term" = String3, + + String4 = format(info,{report,term}, + #{report_cb=>fun(_)-> {"formatted",[]} end}, + #{template=>Template}), + ct:log(String4), + "formatted" = String4, + + String5 = format(info,{report,term}, + #{report_cb=>fun(_)-> faulty_return end}, + #{template=>Template}), + ct:log(String5), + "REPORT_CB ERROR: term; Returned: faulty_return" = String5, + + String6 = format(info,{report,term}, + #{report_cb=>fun(_)-> erlang:error(fun_crashed) end}, + #{template=>Template}), + ct:log(String6), + "REPORT_CB CRASH: term; Reason: {error,fun_crashed}" = String6, + + %% strings are not formatted + String7 = format(info,{string,"string"}, + #{report_cb=>fun(_)-> {"formatted",[]} end}, + #{template=>Template}), + ct:log(String7), + "string" = String7, + + String8 = format(info,{string,['not',printable,list]}, + #{report_cb=>fun(_)-> {"formatted",[]} end}, + #{template=>Template}), + ct:log(String8), + "INVALID STRING: ['not',printable,list]" = String8, + + String9 = format(info,{string,"string"},#{},#{template=>Template}), + ct:log(String9), + "string" = String9, + + ok. + +report_cb(_Config) -> + Template = [msg], + MetaFun = fun(_) -> {"meta_rcb",[]} end, + ConfigFun = fun(_) -> {"config_rcb",[]} end, + "term" = format(info,{report,term},#{},#{template=>Template}), + "meta_rcb" = + format(info,{report,term},#{report_cb=>MetaFun},#{template=>Template}), + "config_rcb" = + format(info,{report,term},#{},#{template=>Template, + report_cb=>ConfigFun}), + "config_rcb" = + format(info,{report,term},#{report_cb=>MetaFun},#{template=>Template, + report_cb=>ConfigFun}), + ok. + +max_size(_Config) -> + Template = [msg], + "12345678901234567890" = + format(info,{"12345678901234567890",[]},#{},#{template=>Template}), + application:set_env(kernel,logger_max_size,11), + "12345678901234567890" = % min value is 50, so this is not limited + format(info,{"12345678901234567890",[]},#{},#{template=>Template}), + "12345678901234567890123456789012345678901234567..." = % 50 + format(info, + {"123456789012345678901234567890123456789012345678901234567890", + []}, + #{}, + #{template=>Template}), + application:set_env(kernel,logger_max_size,53), + "12345678901234567890123456789012345678901234567890..." = %53 + format(info, + {"123456789012345678901234567890123456789012345678901234567890", + []}, + #{}, + #{template=>Template}), + "123456789012..." = + format(info,{"12345678901234567890",[]},#{},#{template=>Template, + max_size=>15}), + "12345678901234567890" = + format(info,{"12345678901234567890",[]},#{},#{template=>Template, + max_size=>unlimited}), + %% Check that one newline at the end of the line is kept (if it exists) + "12345678901...\n" = + format(info,{"12345678901234567890\n",[]},#{},#{template=>Template, + max_size=>15}), + "12345678901...\n" = + format(info,{"12345678901234567890",[]},#{},#{template=>[msg,"\n"], + max_size=>15}), + ok. +max_size(cleanup,_Config) -> + application:unset_env(kernel,logger_max_size), + ok. + +depth(_Config) -> + Template = [msg], + "[1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0]" = + format(info, + {"~p",[[1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0]]}, + #{}, + #{template=>Template}), + application:set_env(kernel,error_logger_format_depth,11), + "[1,2,3,4,5,6,7,8,9,0|...]" = + format(info, + {"~p",[[1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0]]}, + #{}, + #{template=>Template}), + application:set_env(kernel,logger_format_depth,12), + "[1,2,3,4,5,6,7,8,9,0,1|...]" = + format(info, + {"~p",[[1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0]]}, + #{}, + #{template=>Template}), + "[1,2,3,4,5,6,7,8,9,0,1,2|...]" = + format(info, + {"~p",[[1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0]]}, + #{}, + #{template=>Template, + depth=>13}), + "[1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0]" = + format(info, + {"~p",[[1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0]]}, + #{}, + #{template=>Template, + depth=>unlimited}), + ok. +depth(cleanup,_Config) -> + application:unset_env(kernel,logger_format_depth), + ok. + +chars_limit(_Config) -> + FA = {"LoL: ~p~nL: ~p~nMap: ~p~n", + [lists:duplicate(10,lists:seq(1,100)), + lists:seq(1,100), + maps:from_list(lists:zip(lists:seq(1,100), + lists:duplicate(100,value)))]}, + Meta = #{time=>"2018-04-26 9:15:40.449879"}, + Template = [time," - ", msg, "\n"], + FC = #{template=>Template, + depth=>unlimited, + max_size=>unlimited, + chars_limit=>unlimited, + single_line=>true}, + CL1 = 80, + String1 = format(info,FA,Meta,FC#{chars_limit=>CL1}), + L1 = string:length(String1), + ct:log("String1: ~p~nLength1: ~p~n",[lists:flatten(String1),L1]), + true = L1 > CL1, + true = L1 < CL1 + 10, + + String2 = format(info,FA,Meta,FC#{chars_limit=>CL1,depth=>10}), + L2 = string:length(String2), + ct:log("String2: ~p~nLength2: ~p~n",[lists:flatten(String2),L2]), + String2 = String1, + + CL3 = 200, + String3 = format(info,FA,Meta,FC#{chars_limit=>CL3}), + L3 = string:length(String3), + ct:log("String3: ~p~nLength3: ~p~n",[lists:flatten(String3),L3]), + true = L3 > CL3, + true = L3 < CL3 + 10, + + String4 = format(info,FA,Meta,FC#{chars_limit=>CL3,depth=>10}), + L4 = string:length(String4), + ct:log("String4: ~p~nLength4: ~p~n",[lists:flatten(String4),L4]), + true = L4 > CL3, + true = L4 < CL3 + 10, + + %% Test that max_size truncates the string which is limited by + %% depth and chars_limit + MS5 = 150, + String5 = format(info,FA,Meta,FC#{chars_limit=>CL3,depth=>10,max_size=>MS5}), + L5 = string:length(String5), + ct:log("String5: ~p~nLength5: ~p~n",[String5,L5]), + L5 = MS5, + true = lists:prefix(lists:sublist(String5,L5-4),String4), + + ok. + +format_mfa(_Config) -> + Template = [mfa], + + Meta1 = #{mfa=>{mod,func,0}}, + String1 = format(info,{"~p",[term]},Meta1,#{template=>Template}), + ct:log(String1), + "mod:func/0" = String1, + + Meta2 = #{mfa=>{mod,func,[]}}, + String2 = format(info,{"~p",[term]},Meta2,#{template=>Template}), + ct:log(String2), + "mod:func/0" = String2, + + Meta3 = #{mfa=>"mod:func/0"}, + String3 = format(info,{"~p",[term]},Meta3,#{template=>Template}), + ct:log(String3), + "mod:func/0" = String3, + + Meta4 = #{mfa=>othermfa}, + String4 = format(info,{"~p",[term]},Meta4,#{template=>Template}), + ct:log(String4), + "othermfa" = String4, + + ok. + +format_time(_Config) -> + Time1 = timestamp(), + ExpectedTimestamp1 = default_time_format(Time1), + String1 = format(info,{"~p",[term]},#{time=>Time1},#{}), + ct:log(String1), + " info:\nterm\n" = string:prefix(String1,ExpectedTimestamp1), + + Time2 = timestamp(), + ExpectedTimestamp2 = default_time_format(Time2,true), + String2 = format(info,{"~p",[term]},#{time=>Time2},#{utc=>true}), + ct:log(String2), + " info:\nterm\n" = string:prefix(String2,ExpectedTimestamp2), + + application:set_env(kernel,logger_utc,true), + Time3 = timestamp(), + ExpectedTimestamp3 = default_time_format(Time3,true), + String3 = format(info,{"~p",[term]},#{time=>Time3},#{}), + ct:log(String3), + " info:\nterm\n" = string:prefix(String3,ExpectedTimestamp3), + + ok. + +format_time(cleanup,_Config) -> + application:unset_env(kernel,logger_utc), + ok. + +level_or_msg_in_meta(_Config) -> + %% The template contains atoms to pick out values from meta, + %% or level/msg to add these from the log event. What if you have + %% a key named 'level' or 'msg' in meta and want to display + %% its value? + %% For now we simply ignore Meta on this and display the + %% actual level and msg from the log event. + + Meta = #{level=>mylevel, + msg=>"metamsg"}, + Template = [level,";",msg], + String = format(info,{"~p",[term]},Meta,#{template=>Template}), + ct:log(String), + "info;term" = String, % so mylevel and "metamsg" are ignored + + ok. + +faulty_log(_Config) -> + %% Unexpected log (should be type logger:log()) - print error + {error, + function_clause, + {logger_formatter,format,[_,_],_}} = + ?TRY(logger_formatter:format(unexp_log,#{})), + ok. + +faulty_config(_Config) -> + {error, + function_clause, + {logger_formatter,format,[_,_],_}} = + ?TRY(logger_formatter:format(#{level=>info, + msg=>{"~p",[term]}, + meta=>#{time=>timestamp()}}, + unexp_config)), + ok. + +faulty_msg(_Config) -> + {error, + function_clause, + {logger_formatter,_,_,_}} = + ?TRY(logger_formatter:format(#{level=>info, + msg=>term, + meta=>#{time=>timestamp()}}, + #{})), + ok. + +%%%----------------------------------------------------------------- +%%% Internal +format(Level,Msg,Meta,Config) -> + format(#{level=>Level,msg=>Msg,meta=>add_time(Meta)},Config). + +format(Log,Config) -> + lists:flatten(logger_formatter:format(Log,Config)). + +default_time_format(Timestamp) -> + default_time_format(Timestamp,false). + +default_time_format(Timestamp0,Utc) when is_integer(Timestamp0) -> + Timestamp=Timestamp0+erlang:time_offset(microsecond), + %% calendar:system_time_to_rfc3339(Time,[{unit,microsecond}]). + Micro = Timestamp rem 1000000, + Sec = Timestamp div 1000000, + UniversalTime = erlang:posixtime_to_universaltime(Sec), + {Date,Time} = + if Utc -> UniversalTime; + true -> erlang:universaltime_to_localtime(UniversalTime) + end, + default_time_format(Date,Time,Micro). + +default_time_format({Y,M,D},{H,Min,S},Micro) -> + lists:flatten( + io_lib:format("~4w-~2..0w-~2..0w ~2w:~2..0w:~2..0w.~6..0w", + [Y,M,D,H,Min,S,Micro])). + +integer(Str) -> + is_integer(list_to_integer(Str)). +integer(Str,Max) -> + integer(Str,0,Max). +integer(Str,Min,Max) -> + Int = list_to_integer(Str), + Int >= Min andalso Int =<Max. + +%%%----------------------------------------------------------------- +%%% Called by macro ?TRY(X) +my_try(Fun) -> + try Fun() catch C:R:S -> {C,R,hd(S)} end. + +timestamp() -> + erlang:monotonic_time(microsecond). + +%% necessary? +add_time(#{time:=_}=Meta) -> + Meta; +add_time(Meta) -> + Meta#{time=>timestamp()}. diff --git a/lib/kernel/test/logger_legacy_SUITE.erl b/lib/kernel/test/logger_legacy_SUITE.erl new file mode 100644 index 0000000000..b59f5f7758 --- /dev/null +++ b/lib/kernel/test/logger_legacy_SUITE.erl @@ -0,0 +1,282 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(logger_legacy_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("kernel/include/logger.hrl"). +-include_lib("kernel/src/logger_internal.hrl"). + +%%%----------------------------------------------------------------- +%%% This test suite test that log events from within OTP can be +%%% delivered to legacy error_logger event handlers on the same format +%%% as before 'logger' was introduced. +%%% +%%% Before changing the expected format of any of the log events in +%%% this suite, please make sure that the backwards incompatibility it +%%% introduces is ok. +%%% ----------------------------------------------------------------- + +-define(check(Expected), + receive Expected -> + [] = test_server:messages_get() + after 1000 -> + ct:fail({report_not_received, + {line,?LINE}, + {got,test_server:messages_get()}}) + end). +-define(check_no_flush(Expected), + receive Expected -> + ok + after 1000 -> + ct:fail({report_not_received, + {line,?LINE}, + {got,test_server:messages_get()}}) + end). + +suite() -> + [{timetrap,{seconds,30}}]. + +init_per_suite(Config) -> + logger:add_handler(error_logger,error_logger, + #{level=>info,filter_default=>stop}), + Config. + +end_per_suite(_Config) -> + logger:remove_handler(error_logger), + ok. + +init_per_group(std, Config) -> + ok = logger:set_handler_config( + error_logger,filters, + [{domain,{fun logger_filters:domain/2, + {log,prefix_of,[beam,erlang,otp]}}}]), + Config; +init_per_group(sasl, Config) -> + ok = logger:set_handler_config( + error_logger,filters, + [{domain,{fun logger_filters:domain/2, + {log,prefix_of,[beam,erlang,otp,sasl]}}}]), + + %% cth_log_redirect checks if sasl is started before displaying + %% any sasl reports - so just to see the real sasl reports in tc + %% log: + application:start(sasl), + Config; +init_per_group(_Group, Config) -> + Config. + +end_per_group(sasl, _Config) -> + application:stop(sasl), + ok; +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + error_logger:add_report_handler(?MODULE,{event_handler,self()}), + Config. + +end_per_testcase(Case, Config) -> + %% Using gen_event directly here, instead of + %% error_logger:delete_report_handler. This is to avoid + %% automatically stopping the error_logger process due to removing + %% the last handler. + gen_event:delete_handler(error_logger,?MODULE,[]), + try apply(?MODULE,Case,[cleanup,Config]) + catch error:undef -> ok + end, + ok. + +groups() -> + [{std,[],[gen_server, + gen_event, + gen_fsm, + gen_statem]}, + {sasl,[],[sasl_reports, + supervisor_handle_info]}]. + +all() -> + [{group,std}, + {group,sasl}]. + +gen_server(_Config) -> + {ok,Pid} = gen_server:start(?MODULE,gen_server,[]), + Msg = fun() -> a=b end, + Pid ! Msg, + ?check({warning_msg,"** Undefined handle_info in ~p"++_,[?MODULE,Msg]}), + ok = gen_server:cast(Pid,Msg), + ?check({error,"** Generic server ~tp terminating"++_, + [Pid,{'$gen_cast',Msg},gen_server,{{badmatch,b},_}]}). + +gen_event(_Config) -> + {ok,Pid} = gen_event:start(), + ok = gen_event:add_handler(Pid,?MODULE,gen_event), + Msg = fun() -> a=b end, + Pid ! Msg, + ?check({warning_msg,"** Undefined handle_info in ~tp"++_,[?MODULE,Msg]}), + gen_event:notify(Pid,Msg), + ?check({error,"** gen_event handler ~p crashed."++_, + [?MODULE,Pid,Msg,gen_event,{{badmatch,b},_}]}). + +gen_fsm(_Config) -> + {ok,Pid} = gen_fsm:start(?MODULE,gen_fsm,[]), + Msg = fun() -> a=b end, + Pid ! Msg, + ?check({warning_msg,"** Undefined handle_info in ~p"++_,[?MODULE,Msg]}), + gen_fsm:send_all_state_event(Pid,Msg), + ?check({error,"** State machine ~tp terminating"++_, + [Pid,Msg,mystate,gen_fsm,{{badmatch,b},_}]}). + +gen_statem(_Config) -> + {ok,Pid} = gen_statem:start(?MODULE,gen_statem,[]), + Msg = fun() -> a=b end, + Pid ! Msg, + ?check({error,"** State machine ~tp terminating"++_, + [Pid,{info,Msg},{mystate,gen_statem},error,{badmatch,b}|_]}). + +sasl_reports(Config) -> + App = {application,?MODULE,[{description, ""}, + {vsn, "1.0"}, + {modules, [?MODULE]}, + {registered, []}, + {applications, []}, + {mod, {?MODULE, []}}]}, + AppStr = io_lib:format("~p.",[App]), + Dir = ?config(priv_dir,Config), + AppFile = filename:join(Dir,?MODULE_STRING++".app"), + ok = file:write_file(AppFile,AppStr), + true = code:add_patha(Dir), + ok = application:start(?MODULE), + SupName = sup_name(), + Pid = whereis(SupName), + [{ch,ChPid,_,_}] = supervisor:which_children(Pid), + Node = node(), + ?check_no_flush({info_report,progress,[{application,?MODULE}, + {started_at,Node}]}), + ?check({info_report,progress,[{supervisor,{local,SupName}}, + {started,[{pid,ChPid}|_]}]}), + ok = gen_server:cast(ChPid, fun() -> + spawn_link(fun() -> receive x->ok end end) + end), + Msg = fun() -> a=b end, + ok = gen_server:cast(ChPid,Msg), + ?check_no_flush({error,"** Generic server ~tp terminating"++_, + [ChPid,{'$gen_cast',Msg},gen_server,{{badmatch,b},_}]}), + ?check_no_flush({error_report,crash_report, + [[{initial_call,_}, + {pid,ChPid}, + {registered_name,[]}, + {error_info,{error,{badmatch,b},_}}, + {ancestors,_}, + {message_queue_len,_}, + {messages,_}, + {links,[Pid,Neighbour]}, + {dictionary,_}, + {trap_exit,_}, + {status,_}, + {heap_size,_}, + {stack_size,_}, + {reductions,_}], + [{neighbour,[{pid,Neighbour}, + {registered_name,_}, + {initial_call,_}, + {current_function,_}, + {ancestors,_}, + {message_queue_len,_}, + {links,[ChPid]}, + {trap_exit,_}, + {status,_}, + {heap_size,_}, + {stack_size,_}, + {reductions,_}, + {current_stacktrace,_}]}]]}), + ?check_no_flush({error_report,supervisor_report, + [{supervisor,{local,SupName}}, + {errorContext,child_terminated}, + {reason,{{badmatch,b},_}}, + {offender,[{pid,ChPid}|_]}]}), + ?check({info_report,progress,[{supervisor,{local,SupName}}, + {started,_}]}), + + ok = application:stop(?MODULE), + ?check({info_report,std_info,[{application,?MODULE}, + {exited,stopped}, + {type,temporary}]}). + +sasl_reports(cleanup,_Config) -> + application:stop(?MODULE). + +supervisor_handle_info(_Config) -> + {ok,Pid} = supervisor:start_link({local,sup_name()},?MODULE,supervisor), + ?check({info_report,progress,[{supervisor,_},{started,_}]}), + Pid ! msg, + ?check({error,"Supervisor received unexpected message: ~tp~n",[msg]}). + +supervisor_handle_info(cleanup,_Config) -> + Pid = whereis(sup_name()), + unlink(Pid), + exit(Pid,shutdown). + +%%%----------------------------------------------------------------- +%%% Callbacks for error_logger event handler, gen_server, gen_statem, +%%% gen_fsm, gen_event, supervisor and application. +start(_,_) -> + supervisor:start_link({local,sup_name()},?MODULE,supervisor). + +init(supervisor) -> + {ok,{#{},[#{id=>ch,start=>{gen_server,start_link,[?MODULE,gen_server,[]]}}]}}; +init(StateMachine) when StateMachine==gen_statem; StateMachine==gen_fsm -> + {ok,mystate,StateMachine}; +init(State) -> + {ok,State}. + +%% error_logger event handler +handle_event({Tag,_Gl,{_Pid,Type,Report}},{_,Pid}=State) -> + Pid ! {Tag,Type,Report}, + {ok,State}; +%% other gen_event +handle_event(Fun,State) when is_function(Fun) -> + Fun(), + {next_state,State}. + +%% gen_fsm +handle_event(Fun,State,Data) when is_function(Fun) -> + Fun(), + {next_state,State,Data}. + +%% gen_statem +handle_event(info,Fun,State,Data) when is_function(Fun) -> + Fun(), + {next_state,State,Data}. + +%% gen_server +handle_cast(Fun,State) when is_function(Fun) -> + Fun(), + {noreply,State}. + +%% gen_statem +callback_mode() -> + handle_event_function. + +%%%----------------------------------------------------------------- +%%% Internal +sup_name() -> + list_to_atom(?MODULE_STRING++"_sup"). diff --git a/lib/kernel/test/logger_simple_SUITE.erl b/lib/kernel/test/logger_simple_SUITE.erl new file mode 100644 index 0000000000..5d8d32492d --- /dev/null +++ b/lib/kernel/test/logger_simple_SUITE.erl @@ -0,0 +1,247 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(logger_simple_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("kernel/include/logger.hrl"). +-include_lib("kernel/src/logger_internal.hrl"). + +-define(check_no_log,[] = test_server:messages_get()). +-define(check(Expected), + receive {log,Expected} -> + [] = test_server:messages_get() + after 1000 -> + ct:fail({report_not_received, + {line,?LINE}, + {expected,Expected}, + {got,test_server:messages_get()}}) + end). + +-define(str,"Log from "++atom_to_list(?FUNCTION_NAME)++ + ":"++integer_to_list(?LINE)). +-define(map_rep,#{function=>?FUNCTION_NAME, line=>?LINE}). +-define(keyval_rep,[{function,?FUNCTION_NAME}, {line,?LINE}]). + +suite() -> + [{timetrap,{seconds,30}}]. + +init_per_suite(Config) -> + #{handlers:=Hs0} = logger:i(), + Hs = lists:keydelete(cth_log_redirect,1,Hs0), + [ok = logger:remove_handler(Id) || {Id,_,_} <- Hs], + Env = [{App,Key,application:get_env(App,Key)} || + {App,Key} <- [{kernel,logger_dest}, + {kernel,logger_level}]], + [{env,Env},{logger,Hs}|Config]. + +end_per_suite(Config) -> + [application:set_env(App,Key,Val) || {App,Key,Val} <- ?config(env,Config)], + Hs = ?config(logger,Config), + [ok = logger:add_handler(Id,Mod,C) || {Id,Mod,C} <- Hs], + ok. + +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(Case, Config) -> + try apply(?MODULE,Case,[cleanup,Config]) + catch error:undef -> ok + end, + ok. + +groups() -> + []. + +all() -> + [start_stop, + get_buffer, + replace_file, + replace_disk_log + ]. + +start_stop(_Config) -> + undefined = whereis(logger_simple), + register(logger_simple,self()), + {error,_} = logger:add_handler(logger_simple, + logger_simple, + #{filter_default=>log}), + unregister(logger_simple), + ok = logger:add_handler(logger_simple,logger_simple,#{filter_default=>log}), + Pid = whereis(logger_simple), + true = is_pid(Pid), + ok = logger:remove_handler(logger_simple), + false = is_pid(whereis(logger_simple)), + ok. +start_stop(cleanup,_Config) -> + logger:remove_handler(logger_simple). + +get_buffer(_Config) -> + %% Start simple without buffer + ok = logger:add_handler(logger_simple,logger_simple, + #{filter_default=>log}), + logger:emergency(?str), + logger:alert(?str,[]), + logger:error(?map_rep), + logger:info(?keyval_rep), + {ok,[]} = logger_simple:get_buffer(), % no buffer + ok = logger:remove_handler(logger_simple), + + %% Start with buffer + ok = logger:add_handler(logger_simple,logger_simple, + #{filter_default=>log, + logger_simple=>#{buffer=>true}}), + logger:emergency(M1=?str), + logger:alert(M2=?str,[]), + logger:error(M3=?map_rep), + logger:info(M4=?keyval_rep), + logger:info(M41=?keyval_rep++[not_key_val]), + error_logger:error_report(some_type,M5=?map_rep), + error_logger:warning_report("some_type",M6=?map_rep), + logger:critical(M7=?str,[A7=?keyval_rep]), + logger:notice(M8=["fake",string,"line:",?LINE]), + {ok,Buffered1} = logger_simple:get_buffer(), + [#{level:=emergency,msg:={string,M1}}, + #{level:=alert,msg:={M2,[]}}, + #{level:=error,msg:={report,M3}}, + #{level:=info,msg:={report,M4}}, + #{level:=info,msg:={report,M41}}, + #{level:=error,msg:={report,#{label:={error_logger,error_report}, + report:=M5}}}, + #{level:=warning,msg:={report,#{label:={error_logger,warning_report}, + report:=M6}}}, + #{level:=critical,msg:={M7,[A7]}}, + #{level:=notice,msg:={string,M8}}] = Buffered1, + + %% Keep logging - should not buffer any more + logger:emergency(?str), + logger:alert(?str,[]), + logger:error(?map_rep), + logger:info(?keyval_rep), + {ok,[]} = logger_simple:get_buffer(), + ok = logger:remove_handler(logger_simple), + + %% Fill buffer and drop + ok = logger:add_handler(logger_simple,logger_simple, + #{filter_default=>log, + logger_simple=>#{buffer=>true}}), + logger:emergency(M9=?str), + M10=?str, + [logger:info(M10) || _ <- lists:seq(1,8)], + logger:error(M11=?str), + logger:error(?str), + logger:error(?str), + {ok,Buffered3} = logger_simple:get_buffer(), + 11 = length(Buffered3), + [#{level:=emergency,msg:={string,M9}}, + #{level:=info,msg:={string,M10}}, + #{level:=info,msg:={string,M10}}, + #{level:=info,msg:={string,M10}}, + #{level:=info,msg:={string,M10}}, + #{level:=info,msg:={string,M10}}, + #{level:=info,msg:={string,M10}}, + #{level:=info,msg:={string,M10}}, + #{level:=info,msg:={string,M10}}, + #{level:=error,msg:={string,M11}}, + #{level:=info,msg:={"Simple handler buffer full, dropped ~w messages",[2]}}] + = Buffered3, + ok. +get_buffer(cleanup,_Config) -> + logger:remove_handler(logger_simple). + +replace_file(Config) -> + ok = logger:add_handler(logger_simple,logger_simple, + #{filter_default=>log, + logger_simple=>#{buffer=>true}}), + logger:emergency(M1=?str), + logger:alert(M2=?str,[]), + logger:error(?map_rep), + logger:info(?keyval_rep), + undefined = whereis(?STANDARD_HANDLER), + PrivDir = ?config(priv_dir,Config), + File = filename:join(PrivDir,atom_to_list(?FUNCTION_NAME)++".log"), + + application:set_env(kernel,logger_dest,{file,File}), + application:set_env(kernel,logger_level,info), + + ok = logger:setup_standard_handler(), + true = is_pid(whereis(?STANDARD_HANDLER)), + ok = logger_std_h:filesync(?STANDARD_HANDLER), + {ok,Bin} = file:read_file(File), + Lines = [unicode:characters_to_list(L) || + L <- binary:split(Bin,<<"\n">>,[global,trim])], + ["=EMERGENCY REPORT===="++_, + M1, + "=ALERT REPORT===="++_, + M2, + "=ERROR REPORT===="++_, + _, + _, + "=INFO REPORT===="++_, + _, + _] = Lines, + ok. +replace_file(cleanup,_Config) -> + logger:remove_handler(?STANDARD_HANDLER), + logger:remove_handler(logger_simple). + +replace_disk_log(Config) -> + ok = logger:add_handler(logger_simple,logger_simple, + #{filter_default=>log, + logger_simple=>#{buffer=>true}}), + logger:emergency(M1=?str), + logger:alert(M2=?str,[]), + logger:error(?map_rep), + logger:info(?keyval_rep), + undefined = whereis(?STANDARD_HANDLER), + PrivDir = ?config(priv_dir,Config), + File = filename:join(PrivDir,atom_to_list(?FUNCTION_NAME)), + + application:set_env(kernel,logger_dest,{disk_log,File}), + application:set_env(kernel,logger_level,info), + + ok = logger:setup_standard_handler(), + true = is_pid(whereis(?STANDARD_HANDLER)), + ok = logger_disk_log_h:disk_log_sync(?STANDARD_HANDLER), + {ok,Bin} = file:read_file(File++".1"), + Lines = [unicode:characters_to_list(L) || + L <- binary:split(Bin,<<"\n">>,[global,trim])], + ["=EMERGENCY REPORT===="++_, + M1, + "=ALERT REPORT===="++_, + M2, + "=ERROR REPORT===="++_, + _, + _, + "=INFO REPORT===="++_, + _, + _|_] = Lines, % the tail might be an info report about opening the disk log + ok. +replace_disk_log(cleanup,_Config) -> + logger:remove_handler(?STANDARD_HANDLER), + logger:remove_handler(logger_simple). + diff --git a/lib/kernel/test/logger_std_h_SUITE.erl b/lib/kernel/test/logger_std_h_SUITE.erl new file mode 100644 index 0000000000..e940e0a026 --- /dev/null +++ b/lib/kernel/test/logger_std_h_SUITE.erl @@ -0,0 +1,1396 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(logger_std_h_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("kernel/include/logger.hrl"). +-include_lib("kernel/src/logger_internal.hrl"). +-include_lib("kernel/src/logger_h_common.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). +-include_lib("kernel/include/file.hrl"). + +-define(check_no_log, [] = test_server:messages_get()). +-define(check(Expected), + receive + {log,Expected} -> + [] = test_server:messages_get() + after 5000 -> + ct:fail({report_not_received, + {line,?LINE}, + {expected,Expected}, + {got,test_server:messages_get()}}) + end). + +-define(msg,"Log from "++atom_to_list(?FUNCTION_NAME)++ + ":"++integer_to_list(?LINE)). +-define(bin(Msg), list_to_binary(Msg++"\n")). +-define(domain,#{domain=>[?MODULE]}). + +-define(FILESYNC_REP_INT, if is_atom(?FILESYNC_REPEAT_INTERVAL) -> 5500; + true -> ?FILESYNC_REPEAT_INTERVAL + 500 + end). + +suite() -> + [{timetrap,{seconds,30}}]. + +init_per_suite(Config) -> + timer:start(), % to avoid progress report + {ok,{?STANDARD_HANDLER,#{formatter:=OrigFormatter}}} = + logger:get_handler_config(?STANDARD_HANDLER), + [{formatter,OrigFormatter}|Config]. + +end_per_suite(Config) -> + {OrigMod,OrigConf} = proplists:get_value(formatter,Config), + logger:set_handler_config(?STANDARD_HANDLER,formatter,{OrigMod,OrigConf}), + ok. + +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(TestHooksCase, Config) when + TestHooksCase == write_failure; + TestHooksCase == sync_failure -> + if ?TEST_HOOKS_TAB == undefined -> + {skip,"Define the TEST_HOOKS macro to run this test"}; + true -> + ct:print("********** ~w **********", [TestHooksCase]), + Config + end; +init_per_testcase(TestCase, Config) -> + ct:print("********** ~w **********", [TestCase]), + Config. + +end_per_testcase(Case, Config) -> + try apply(?MODULE,Case,[cleanup,Config]) + catch error:undef -> ok + end, + ok. + +groups() -> + []. + +all() -> + [add_remove_instance_tty, + add_remove_instance_standard_io, + add_remove_instance_standard_error, + add_remove_instance_file1, + add_remove_instance_file2, + default_formatter, + errors, + formatter_fail, + config_fail, + crash_std_h_to_file, + crash_std_h_to_disk_log, + bad_input, + info_and_reset, + reconfig, + file_opts, + filesync, + write_failure, + sync_failure, + op_switch_to_sync_file, + op_switch_to_sync_tty, + op_switch_to_drop_file, + op_switch_to_drop_tty, + op_switch_to_flush_file, + op_switch_to_flush_tty, + limit_burst_disabled, + limit_burst_enabled_one, + limit_burst_enabled_period, + kill_disabled, + qlen_kill_new, + qlen_kill_std, + mem_kill_new, + mem_kill_std, + restart_after, + handler_requests_under_load + ]. + +add_remove_instance_tty(_Config) -> + {error,{handler_not_added,{invalid_config,logger_std_h,{type,tty}}}} = + logger:add_handler(?MODULE,logger_std_h, + #{logger_std_h => #{type => tty}, + filter_default=>log, + formatter=>{?MODULE,self()}}), + ok. + +add_remove_instance_standard_io(_Config) -> + add_remove_instance_nofile(standard_io). +add_remove_instance_standard_io(cleanup,_Config) -> + logger_std_h_remove(). + +add_remove_instance_standard_error(_Config) -> + add_remove_instance_nofile(standard_error). +add_remove_instance_standard_error(cleanup,_Config) -> + logger_std_h_remove(). + +add_remove_instance_file1(Config) -> + Dir = ?config(priv_dir,Config), + Log = filename:join(Dir,"stdlog1.txt"), + Type = {file,Log}, + add_remove_instance_file(Log, Type). +add_remove_instance_file1(cleanup,_Config) -> + logger_std_h_remove(). + +add_remove_instance_file2(Config) -> + Dir = ?config(priv_dir,Config), + Log = filename:join(Dir,"stdlog2.txt"), + Type = {file,Log,[raw,append]}, + add_remove_instance_file(Log, Type). +add_remove_instance_file2(cleanup,_Config) -> + logger_std_h_remove(). + +add_remove_instance_file(Log, Type) -> + ok = logger:add_handler(?MODULE, + logger_std_h, + #{logger_std_h => #{type => Type}, + filter_default=>stop, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), + formatter=>{?MODULE,self()}}), + Pid = whereis(?MODULE), + true = is_pid(Pid), + logger:info(M1=?msg,?domain), + ?check(M1), + B1 = ?bin(M1), + try_read_file(Log, {ok,B1}, ?FILESYNC_REP_INT), + ok = logger:remove_handler(?MODULE), + timer:sleep(500), + undefined = whereis(?MODULE), + logger:info(?msg,?domain), + ?check_no_log, + try_read_file(Log, {ok,B1}, ?FILESYNC_REP_INT), + ok. + +default_formatter(_Config) -> + ok = logger:set_handler_config(?STANDARD_HANDLER,formatter, + {?DEFAULT_FORMATTER,?DEFAULT_FORMAT_CONFIG}), + ct:capture_start(), + logger:info(M1=?msg), + timer:sleep(100), + ct:capture_stop(), + [Msg] = ct:capture_get(), + match = re:run(Msg,"=INFO REPORT====.*\n"++M1,[{capture,none}]), + ok. + +errors(Config) -> + Dir = ?config(priv_dir,Config), + Log = filename:join(Dir,?FUNCTION_NAME), + + ok = logger:add_handler(?MODULE,logger_std_h,#{}), + {error,{already_exist,?MODULE}} = + logger:add_handler(?MODULE,logger_std_h,#{}), + + {error,{not_found,no_such_name}} = logger:remove_handler(no_such_name), + + ok = logger:remove_handler(?MODULE), + {error,{not_found,?MODULE}} = logger:remove_handler(?MODULE), + + {error, + {handler_not_added, + {invalid_config,logger_std_h,{type,faulty_type}}}} = + logger:add_handler(?MODULE,logger_std_h, + #{logger_std_h => #{type => faulty_type}}), + + NoDir = lists:concat(["/",?MODULE,"_dir"]), + {error, + {handler_not_added,{{open_failed,NoDir,eacces},_}}} = + logger:add_handler(myh2,logger_std_h, + #{logger_std_h=>#{type=>{file,NoDir}}}), + + {error, + {handler_not_added,{{open_failed,Log,_},_}}} = + logger:add_handler(myh3,logger_std_h, + #{logger_std_h=>#{type=>{file,Log,[bad_file_opt]}}}), + + ok = logger:info(?msg). + +errors(cleanup,_Config) -> + logger:remove_handler(?MODULE). + +formatter_fail(Config) -> + Dir = ?config(priv_dir,Config), + Log = filename:join(Dir,?FUNCTION_NAME), + + %% no formatter + ok = logger:add_handler(?MODULE, + logger_std_h, + #{logger_std_h => #{type => {file,Log}}, + filter_default=>stop, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE])}), + Pid = whereis(?MODULE), + true = is_pid(Pid), + {ok,#{handlers:=H}} = logger:get_logger_config(), + true = lists:member(?MODULE,H), + + %% Formatter is added automatically + {ok,{_,#{formatter:={logger_formatter,_}}}} = + logger:get_handler_config(?MODULE), + logger:info(M1=?msg,?domain), + Got1 = try_match_file(Log,"=INFO REPORT====.*\n"++M1,5000), + + ok = logger:set_handler_config(?MODULE,formatter,{nonexistingmodule,#{}}), + logger:info(M2=?msg,?domain), + Got2 = try_match_file(Log, + Got1++"=INFO REPORT====.*\nFORMATTER CRASH: .*"++M2, + 5000), + + ok = logger:set_handler_config(?MODULE,formatter,{?MODULE,crash}), + logger:info(M3=?msg,?domain), + Got3 = try_match_file(Log, + Got2++"=INFO REPORT====.*\nFORMATTER CRASH: .*"++M3, + 5000), + + ok = logger:set_handler_config(?MODULE,formatter,{?MODULE,bad_return}), + logger:info(?msg,?domain), + try_match_file(Log, + Got3++"FORMATTER ERROR: bad_return_value", + 5000), + + %% Check that handler is still alive and was never dead + Pid = whereis(?MODULE), + {ok,#{handlers:=H}} = logger:get_logger_config(), + + ok. + +formatter_fail(cleanup,_Config) -> + logger:remove_handler(?MODULE). + +config_fail(_Config) -> + {error,{handler_not_added,{invalid_config,logger_std_h,{bad,bad}}}} = + logger:add_handler(?MODULE,logger_std_h, + #{logger_std_h => #{bad => bad}, + filter_default=>log, + formatter=>{?MODULE,self()}}), + {error,{handler_not_added,{invalid_config,logger_std_h, + {restart_type,bad}}}} = + logger:add_handler(?MODULE,logger_std_h, + #{logger_std_h => #{restart_type => bad}, + filter_default=>log, + formatter=>{?MODULE,self()}}), + {error,{handler_not_added,{invalid_levels,{42,42,_}}}} = + logger:add_handler(?MODULE,logger_std_h, + #{logger_std_h => #{toggle_sync_qlen=>42, + drop_new_reqs_qlen=>42}}), + + ok = logger:add_handler(?MODULE,logger_std_h, + #{filter_default=>log, + formatter=>{?MODULE,self()}}), + {error,{illegal_config_change,_,_}} = + logger:set_handler_config(?MODULE,logger_std_h, + #{type=>{file,"file"}}), + {error,{illegal_config_change,_,_}} = + logger:set_handler_config(?MODULE,id,bad), + {error,{invalid_levels,_}} = + logger:set_handler_config(?MODULE,logger_std_h, + #{toggle_sync_qlen=>100, + flush_reqs_qlen=>99}), + {error,{invalid_config,logger_std_h,{filesync_rep_int,2000}}} = + logger:set_handler_config(?MODULE, logger_std_h, + #{filesync_rep_int => 2000}), + ok. + +config_fail(cleanup,_Config) -> + logger:remove_handler(?MODULE). + +crash_std_h_to_file(Config) -> + crash_std_h(Config,?FUNCTION_NAME,logger_dest,file). +crash_std_h_to_file(cleanup,_Config) -> + crash_std_h(cleanup). + +crash_std_h_to_disk_log(Config) -> + crash_std_h(Config,?FUNCTION_NAME,logger_dest,disk_log). +crash_std_h_to_disk_log(cleanup,_Config) -> + crash_std_h(cleanup). + +crash_std_h(Config,Func,Var,Type) -> + Dir = ?config(priv_dir,Config), + File = lists:concat([?MODULE,"_",Func,".log"]), + Log = filename:join(Dir,File), + Pa = filename:dirname(code:which(?MODULE)), + TypeAndLog = + case os:type() of + {win32,_} -> + lists:concat([" {",Type,",\\\"",Log,"\\\"}"]); + _ -> + lists:concat([" \'{",Type,",\"",Log,"\"}\'"]) + end, + Args = lists:concat([" -kernel ",Var,TypeAndLog," -pa ",Pa]), + Name = lists:concat([?MODULE,"_",Func]), + ct:pal("Starting ~p with ~tp", [Name,Args]), + %% Start a node which prints kernel logs to the destination specified by Type + {ok,Node} = test_server:start_node(Name, peer, [{args, Args}]), + Pid = rpc:call(Node,erlang,whereis,[?STANDARD_HANDLER]), + ok = rpc:call(Node,logger,set_handler_config,[?STANDARD_HANDLER,formatter, + {?MODULE,self()}]), + ok = log_on_remote_node(Node,"dummy1"), + ?check("dummy1"), + {ok,Bin1} = sync_and_read(Node,Type,Log), + <<"dummy1\n">> = binary:part(Bin1,{byte_size(Bin1),-7}), + + %% Kill the logger_std_h process + exit(Pid, kill), + + %% Wait a bit, then check that it is gone + timer:sleep(2000), + undefined = rpc:call(Node,erlang,whereis,[?STANDARD_HANDLER]), + + %% Check that file is not empty + {ok,Bin2} = sync_and_read(Node,Type,Log), + <<"dummy1\n">> = binary:part(Bin2,{byte_size(Bin2),-7}), + ok. + +%% Can not use rpc:call here, since the code would execute on a +%% process with group_leader on this (the calling) node, and thus +%% logger would send the log event to the logger process here instead +%% of logging it itself. +log_on_remote_node(Node,Msg) -> + _ = spawn_link(Node, + fun() -> erlang:group_leader(whereis(user),self()), + logger:info(Msg) + end), + ok. + + +crash_std_h(cleanup) -> + Nodes = nodes(), + [test_server:stop_node(Node) || Node <- Nodes]. + +sync_and_read(Node,disk_log,Log) -> + rpc:call(Node,logger_disk_log_h,disk_log_sync,[?STANDARD_HANDLER]), + case file:read_file(Log ++ ".1") of + {ok,<<>>} -> + timer:sleep(5000), + file:read_file(Log ++ ".1"); + Ok -> + Ok + end; +sync_and_read(Node,file,Log) -> + rpc:call(Node,logger_std_h,filesync,[?STANDARD_HANDLER]), + case file:read_file(Log) of + {ok,<<>>} -> + timer:sleep(5000), + file:read_file(Log); + Ok -> + Ok + end. + +bad_input(_Config) -> + {error,{badarg,{filesync,["BadType"]}}} = logger_std_h:filesync("BadType"), + {error,{badarg,{info,["BadType"]}}} = logger_std_h:info("BadType"), + {error,{badarg,{reset,["BadType"]}}} = logger_std_h:reset("BadType"). + + +info_and_reset(_Config) -> + #{id := ?STANDARD_HANDLER} = logger_std_h:info(?STANDARD_HANDLER), + ok = logger_std_h:reset(?STANDARD_HANDLER). + +reconfig(Config) -> + Dir = ?config(priv_dir,Config), + ok = logger:add_handler(?MODULE, + logger_std_h, + #{logger_std_h => #{type => standard_io}, + filter_default=>log, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), + formatter=>{?MODULE,self()}}), + #{id := ?MODULE, + type := standard_io, + file_ctrl_pid := FileCtrlPid, + toggle_sync_qlen := ?TOGGLE_SYNC_QLEN, + drop_new_reqs_qlen := ?DROP_NEW_REQS_QLEN, + flush_reqs_qlen := ?FLUSH_REQS_QLEN, + enable_burst_limit := ?ENABLE_BURST_LIMIT, + burst_limit_size := ?BURST_LIMIT_SIZE, + burst_window_time := ?BURST_WINDOW_TIME, + enable_kill_overloaded := ?ENABLE_KILL_OVERLOADED, + handler_overloaded_qlen := ?HANDLER_OVERLOADED_QLEN, + handler_overloaded_mem := ?HANDLER_OVERLOADED_MEM, + handler_restart_after := ?HANDLER_RESTART_AFTER, + filesync_repeat_interval := ?FILESYNC_REPEAT_INTERVAL} = + logger_std_h:info(?MODULE), + + ok = logger:set_handler_config(?MODULE, logger_std_h, + #{toggle_sync_qlen => 1, + drop_new_reqs_qlen => 2, + flush_reqs_qlen => 3, + enable_burst_limit => false, + burst_limit_size => 10, + burst_window_time => 10, + enable_kill_overloaded => true, + handler_overloaded_qlen => 100000, + handler_overloaded_mem => 10000000, + handler_restart_after => never, + filesync_repeat_interval => no_repeat}), + #{id := ?MODULE, + type := standard_io, + file_ctrl_pid := FileCtrlPid, + toggle_sync_qlen := 1, + drop_new_reqs_qlen := 2, + flush_reqs_qlen := 3, + enable_burst_limit := false, + burst_limit_size := 10, + burst_window_time := 10, + enable_kill_overloaded := true, + handler_overloaded_qlen := 100000, + handler_overloaded_mem := 10000000, + handler_restart_after := never, + filesync_repeat_interval := no_repeat} = logger_std_h:info(?MODULE), + ok. + +reconfig(cleanup, _Config) -> + logger:remove_handler(?MODULE). + + +file_opts(Config) -> + Dir = ?config(priv_dir,Config), + Log = filename:join(Dir, lists:concat([?FUNCTION_NAME,".log"])), + BadFileOpts = [raw], + BadType = {file,Log,BadFileOpts}, + {error,{handler_not_added,{{open_failed,Log,enoent},_}}} = + logger:add_handler(?MODULE, logger_std_h, + #{logger_std_h => #{type => BadType}}), + + OkFileOpts = [raw,append], + OkType = {file,Log,OkFileOpts}, + ok = logger:add_handler(?MODULE, + logger_std_h, + #{logger_std_h => #{type => OkType}, + filter_default=>log, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), + formatter=>{?MODULE,self()}}), + + #{type := OkType} = logger_std_h:info(?MODULE), + logger:info(M1=?msg,?domain), + ?check(M1), + B1 = ?bin(M1), + try_read_file(Log, {ok,B1}, ?FILESYNC_REP_INT), + ok. +file_opts(cleanup, _Config) -> + logger:remove_handler(?MODULE). + + +filesync(Config) -> + Dir = ?config(priv_dir,Config), + Log = filename:join(Dir, lists:concat([?FUNCTION_NAME,".log"])), + Type = {file,Log}, + ok = logger:add_handler(?MODULE, + logger_std_h, + #{logger_std_h => #{type => Type}, + filter_default=>log, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), + formatter=>{?MODULE,self()}}), + Tester = self(), + TraceFun = fun({trace,_,call,{Mod,Func,Details}}, Pid) -> + Pid ! {trace,Mod,Func,Details}, + Pid; + ({trace,TPid,'receive',Received}, Pid) -> + Pid ! {trace,TPid,Received}, + Pid + end, + {ok,_} = dbg:tracer(process, {TraceFun, Tester}), + FileCtrlPid = maps:get(file_ctrl_pid , logger_std_h:info(?MODULE)), + {ok,_} = dbg:p(FileCtrlPid, [c]), + {ok,_} = dbg:tpl(logger_std_h, write_to_dev, 5, []), + {ok,_} = dbg:tpl(logger_std_h, sync_dev, 4, []), + {ok,_} = dbg:tp(file, datasync, 1, []), + + logger:info("first", ?domain), + %% wait for automatic filesync + timer:sleep(?FILESYNC_REP_INT), + Expected1 = [{log,"first"}, {trace,logger_std_h,write_to_dev}, + {trace,logger_std_h,sync_dev}, {trace,file,datasync}], + + logger:info("second", ?domain), + %% do explicit filesync + logger_std_h:filesync(?MODULE), + %% a second filesync should be ignored + logger_std_h:filesync(?MODULE), + Expected2 = [{log,"second"}, {trace,logger_std_h,write_to_dev}, + {trace,logger_std_h,sync_dev}, {trace,file,datasync}], + + %% check that if there's no repeated filesync active, + %% a filesync is still performed when handler goes idle + logger:set_handler_config(?MODULE, logger_std_h, + #{filesync_repeat_interval => no_repeat}), + no_repeat = maps:get(filesync_repeat_interval, logger_std_h:info(?MODULE)), + logger:info("third", ?domain), + timer:sleep(?IDLE_DETECT_TIME_MSEC*2), + logger:info("fourth", ?domain), + %% wait for automatic filesync + timer:sleep(?IDLE_DETECT_TIME_MSEC*2), + Expected3 = [{log,"third"}, {trace,logger_std_h,write_to_dev}, + {log,"fourth"}, {trace,logger_std_h,write_to_dev}, + {trace,logger_std_h,sync_dev}, {trace,file,datasync}], + + dbg:stop_clear(), + + %% verify that filesync has been performed as expected + Received1 = lists:map(fun({trace,M,F,_}) -> {trace,M,F}; + (Other) -> Other + end, test_server:messages_get()), + ct:pal("Trace #1 =~n~p", [Received1]), + Received1 = Expected1 ++ Expected2 ++ Expected3, + + try_read_file(Log, {ok,<<"first\nsecond\nthird\nfourth\n">>}, 1000), + + {ok,_} = dbg:tracer(process, {TraceFun, Tester}), + {ok,_} = dbg:p(whereis(?MODULE), [c]), + {ok,_} = dbg:tpl(logger_std_h, handle_cast, 2, []), + + %% switch repeated filesync on and verify that the looping works + SyncInt = 1000, + WaitT = 4500, + logger:set_handler_config(?MODULE, logger_std_h, + #{filesync_repeat_interval => SyncInt}), + SyncInt = maps:get(filesync_repeat_interval, logger_std_h:info(?MODULE)), + timer:sleep(WaitT), + logger:set_handler_config(?MODULE, logger_std_h, + #{filesync_repeat_interval => no_repeat}), + dbg:stop_clear(), + + Received2 = lists:map(fun({trace,_M,handle_cast,[{Op,_},_]}) -> {trace,Op}; + (Other) -> Other + end, test_server:messages_get()), + ct:pal("Trace #2 =~n~p", [Received2]), + OneSync = [{trace,repeated_filesync}], + %% receive 1 initial repeated_filesync, then 1 per sec + Received2 = + lists:flatten([OneSync || _ <- lists:seq(1, 1 + trunc(WaitT/SyncInt))]), + ok. +filesync(cleanup, _Config) -> + logger:remove_handler(?MODULE). + +write_failure(Config) -> + Dir = ?config(priv_dir, Config), + File = lists:concat([?MODULE,"_",?FUNCTION_NAME,".log"]), + Log = filename:join(Dir, File), + Node = start_std_h_on_new_node(Config, ?FUNCTION_NAME, Log), + false = (undefined == rpc:call(Node, ets, whereis, [?TEST_HOOKS_TAB])), + rpc:call(Node, ets, insert, [?TEST_HOOKS_TAB,{tester,self()}]), + rpc:call(Node, ?MODULE, set_internal_log, [?MODULE,internal_log]), + rpc:call(Node, ?MODULE, set_result, [file_write,ok]), + + ok = log_on_remote_node(Node, "Logged1"), + rpc:call(Node, logger_std_h, filesync, [?STANDARD_HANDLER]), + ?check_no_log, + try_read_file(Log, {ok,<<"Logged1\n">>}, ?FILESYNC_REP_INT), + + rpc:call(Node, ?MODULE, set_result, [file_write,{error,terminated}]), + ok = log_on_remote_node(Node, "Cause simple error printout"), + + ?check({error,{?STANDARD_HANDLER,write,Log,{error,terminated}}}), + + ok = log_on_remote_node(Node, "No second error printout"), + ?check_no_log, + + rpc:call(Node, ?MODULE, set_result, [file_write,{error,eacces}]), + ok = log_on_remote_node(Node, "Cause simple error printout"), + ?check({error,{?STANDARD_HANDLER,write,Log,{error,eacces}}}), + + rpc:call(Node, ?MODULE, set_result, [file_write,ok]), + ok = log_on_remote_node(Node, "Logged2"), + rpc:call(Node, logger_std_h, filesync, [?STANDARD_HANDLER]), + ?check_no_log, + try_read_file(Log, {ok,<<"Logged1\nLogged2\n">>}, ?FILESYNC_REP_INT), + ok. +write_failure(cleanup, _Config) -> + Nodes = nodes(), + [test_server:stop_node(Node) || Node <- Nodes]. + +sync_failure(Config) -> + Dir = ?config(priv_dir, Config), + File = lists:concat([?MODULE,"_",?FUNCTION_NAME,".log"]), + Log = filename:join(Dir, File), + Node = start_std_h_on_new_node(Config, ?FUNCTION_NAME, Log), + false = (undefined == rpc:call(Node, ets, whereis, [?TEST_HOOKS_TAB])), + rpc:call(Node, ets, insert, [?TEST_HOOKS_TAB,{tester,self()}]), + rpc:call(Node, ?MODULE, set_internal_log, [?MODULE,internal_log]), + rpc:call(Node, ?MODULE, set_result, [file_datasync,ok]), + + SyncInt = 500, + ok = rpc:call(Node, logger, set_handler_config, + [?STANDARD_HANDLER, logger_std_h, + #{filesync_repeat_interval => SyncInt}]), + Info = rpc:call(Node, logger_std_h, info, [?STANDARD_HANDLER]), + SyncInt = maps:get(filesync_repeat_interval, Info), + + ok = log_on_remote_node(Node, "Logged1"), + ?check_no_log, + + rpc:call(Node, ?MODULE, set_result, [file_datasync,{error,terminated}]), + ok = log_on_remote_node(Node, "Cause simple error printout"), + + ?check({error,{?STANDARD_HANDLER,filesync,Log,{error,terminated}}}), + + ok = log_on_remote_node(Node, "No second error printout"), + ?check_no_log, + + rpc:call(Node, ?MODULE, set_result, [file_datasync,{error,eacces}]), + ok = log_on_remote_node(Node, "Cause simple error printout"), + ?check({error,{?STANDARD_HANDLER,filesync,Log,{error,eacces}}}), + + rpc:call(Node, ?MODULE, set_result, [file_datasync,ok]), + ok = log_on_remote_node(Node, "Logged2"), + ?check_no_log, + ok. +sync_failure(cleanup, _Config) -> + Nodes = nodes(), + [test_server:stop_node(Node) || Node <- Nodes]. + +start_std_h_on_new_node(_Config, Func, Log) -> + Pa = filename:dirname(code:which(?MODULE)), + Dest = + case os:type() of + {win32,_} -> + lists:concat([" {file,\\\"",Log,"\\\"}"]); + _ -> + lists:concat([" \'{file,\"",Log,"\"}\'"]) + end, + Args = lists:concat([" -kernel ",logger_dest,Dest," -pa ",Pa]), + Name = lists:concat([?MODULE,"_",Func]), + ct:pal("Starting ~s with ~tp", [Name,Args]), + {ok,Node} = test_server:start_node(Name, peer, [{args, Args}]), + Pid = rpc:call(Node,erlang,whereis,[?STANDARD_HANDLER]), + true = is_pid(Pid), + ok = rpc:call(Node,logger,set_handler_config,[?STANDARD_HANDLER,formatter, + {?MODULE,nl}]), + Node. + +%% functions for test hook macros to be called by rpc +set_internal_log(Mod, Func) -> + ?set_internal_log({Mod,Func}). +set_result(Op, Result) -> + ?set_result(Op, Result). +set_defaults() -> + ?set_defaults(). + +%% internal log function that sends the term to the test case process +internal_log(Type, Term) -> + [{tester,Tester}] = ets:lookup(?TEST_HOOKS_TAB, tester), + Tester ! {log,{Type,Term}}, + logger:internal_log(Type, Term), + ok. + + +%%%----------------------------------------------------------------- +%%% Overload protection tests + +op_switch_to_sync_file(Config) -> + {Log,HConfig,StdHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + NewHConfig = + HConfig#{logger_std_h => StdHConfig#{toggle_sync_qlen => 3, + drop_new_reqs_qlen => 501, + flush_reqs_qlen => 2000, + enable_burst_limit => false}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + %% TRecvPid = start_op_trace(), + NumOfReqs = 500, + send_burst({n,NumOfReqs}, seq, {chars,79}, info), + NumOfReqs = count_lines(Log), + %% true = analyse_trace(TRecvPid, + %% fun(Events) -> find_mode(async,Events) end), + %% true = analyse_trace(TRecvPid, + %% fun(Events) -> find_mode(sync,Events) end), + %% true = analyse_trace(TRecvPid, + %% fun(Events) -> find_switch(async,sync,Events) end), + %% false = analyse_trace(TRecvPid, + %% fun(Events) -> find_mode(drop,Events) end), + %% false = analyse_trace(TRecvPid, + %% fun(Events) -> find_mode(flush,Events) end), + ok = file:delete(Log), + %% stop_op_trace(TRecvPid), + ok. +op_switch_to_sync_file(cleanup, _Config) -> + ok = stop_handler(?MODULE). + +op_switch_to_sync_tty(Config) -> + {HConfig,StdHConfig} = start_handler(?MODULE, standard_io, Config), + NewHConfig = + HConfig#{logger_std_h => StdHConfig#{toggle_sync_qlen => 3, + drop_new_reqs_qlen => 501, + flush_reqs_qlen => 2000, + enable_burst_limit => false}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + NumOfReqs = 500, + send_burst({n,NumOfReqs}, seq, {chars,79}, info), + ok. +op_switch_to_sync_tty(cleanup, _Config) -> + ok = stop_handler(?MODULE). + +op_switch_to_drop_file(Config) -> + {Log,HConfig,StdHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + + NewHConfig = + HConfig#{logger_std_h => StdHConfig#{toggle_sync_qlen => 2, + drop_new_reqs_qlen => 3, + flush_reqs_qlen => 600, + enable_burst_limit => false}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + %% TRecvPid = start_op_trace(), + NumOfReqs = 500, + send_burst({n,NumOfReqs}, seq, {chars,79}, info), + Logged = count_lines(Log), + ct:pal("Number of messages dropped = ~w (~w)", + [NumOfReqs-Logged,NumOfReqs]), + true = (Logged < NumOfReqs), + %% true = analyse_trace(TRecvPid, + %% fun(Events) -> find_mode(async,Events) end), + %% true = analyse_trace(TRecvPid, + %% fun(Events) -> find_mode(drop,Events) end), + %% false = analyse_trace(TRecvPid, + %% fun(Events) -> find_mode(flush,Events) end), + %% true = analyse_trace(TRecvPid, + %% fun(Events) -> find_switch(async,drop,Events) + %% orelse find_switch(sync,drop,Events) + %% end), + ok = file:delete(Log), + %% stop_op_trace(TRecvPid), + ok. +op_switch_to_drop_file(cleanup, _Config) -> + ok = stop_handler(?MODULE). + +op_switch_to_drop_tty(Config) -> + {HConfig,StdHConfig} = start_handler(?MODULE, standard_io, Config), + NewHConfig = + HConfig#{logger_std_h => StdHConfig#{toggle_sync_qlen => 2, + drop_new_reqs_qlen => 3, + flush_reqs_qlen => 600, + enable_burst_limit => false}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + NumOfReqs = 500, + send_burst({n,NumOfReqs}, seq, {chars,79}, info), + ok. +op_switch_to_drop_tty(cleanup, _Config) -> + ok = stop_handler(?MODULE). + +op_switch_to_flush_file() -> + [{timetrap,{seconds,60}}]. +op_switch_to_flush_file(Config) -> + {Log,HConfig,StdHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + + %% it's important that both async and sync requests have been queued + %% when the flush happens (verify with coverage of flush_log_requests/2) + + NewHConfig = + HConfig#{logger_std_h => StdHConfig#{toggle_sync_qlen => 2, + drop_new_reqs_qlen => 99, + flush_reqs_qlen => 100, + enable_burst_limit => false}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + NumOfReqs = 10000, + Procs = 100, + send_burst({n,NumOfReqs}, {spawn,Procs,0}, {chars,79}, info), + Logged = count_lines(Log), + ct:pal("Number of messages flushed/dropped = ~w (~w)", + [(NumOfReqs*Procs)-Logged,NumOfReqs*Procs]), + true = (Logged < (NumOfReqs*Procs)), + + %%! --- Thu Apr 12 13:46:00 2018 --- peppe was here! + %%! TODO: Verify that handler has switched to flush mode + + ok = file:delete(Log), + ok. +op_switch_to_flush_file(cleanup, _Config) -> + ok = stop_handler(?MODULE). + +op_switch_to_flush_tty(Config) -> + {HConfig,StdHConfig} = start_handler(?MODULE, standard_io, Config), + + %% it's important that both async and sync requests have been queued + %% when the flush happens (verify with coverage of flush_log_requests/2) + + NewHConfig = + HConfig#{logger_std_h => StdHConfig#{toggle_sync_qlen => 2, + drop_new_reqs_qlen => 99, + flush_reqs_qlen => 100, + enable_burst_limit => false}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + NumOfReqs = 10000, + Procs = 10, + send_burst({n,NumOfReqs}, {spawn,Procs,0}, {chars,79}, info), + ok. +op_switch_to_flush_tty(cleanup, _Config) -> + ok = stop_handler(?MODULE). + +limit_burst_disabled(Config) -> + {Log,HConfig,StdHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + NewHConfig = + HConfig#{logger_std_h => StdHConfig#{enable_burst_limit => false, + burst_limit_size => 10, + burst_window_time => 2000, + drop_new_reqs_qlen => 200, + flush_reqs_qlen => 300}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + NumOfReqs = 100, + send_burst({n,NumOfReqs}, seq, {chars,79}, info), + Logged = count_lines(Log), + ct:pal("Number of messages logged = ~w", [Logged]), + ok = file:delete(Log), + NumOfReqs = Logged. +limit_burst_disabled(cleanup, _Config) -> + ok = stop_handler(?MODULE). + +limit_burst_enabled_one(Config) -> + {Log,HConfig,StdHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + ReqLimit = 10, + NewHConfig = + HConfig#{logger_std_h => StdHConfig#{enable_burst_limit => true, + burst_limit_size => ReqLimit, + burst_window_time => 2000, + drop_new_reqs_qlen => 200, + flush_reqs_qlen => 300}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + NumOfReqs = 100, + send_burst({n,NumOfReqs}, seq, {chars,79}, info), + Logged = count_lines(Log), + ct:pal("Number of messages logged = ~w", [Logged]), + ok = file:delete(Log), + ReqLimit = Logged. +limit_burst_enabled_one(cleanup, _Config) -> + ok = stop_handler(?MODULE). + +limit_burst_enabled_period(Config) -> + {Log,HConfig,StdHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + ReqLimit = 10, + BurstTWin = 1000, + NewHConfig = + HConfig#{logger_std_h => StdHConfig#{enable_burst_limit => true, + burst_limit_size => ReqLimit, + burst_window_time => BurstTWin, + drop_new_reqs_qlen => 20000, + flush_reqs_qlen => 20001}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + + Windows = 3, + Sent = send_burst({t,BurstTWin*Windows}, seq, {chars,79}, info), + Logged = count_lines(Log), + ct:pal("Number of messages sent = ~w~nNumber of messages logged = ~w", + [Sent,Logged]), + ok = file:delete(Log), + true = (Logged > (ReqLimit*Windows)) andalso + (Logged < (ReqLimit*(Windows+2))). +limit_burst_enabled_period(cleanup, _Config) -> + ok = stop_handler(?MODULE). + +kill_disabled(Config) -> + {Log,HConfig,StdHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + NewHConfig = + HConfig#{logger_std_h=>StdHConfig#{enable_kill_overloaded=>false, + handler_overloaded_qlen=>10, + handler_overloaded_mem=>100}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + NumOfReqs = 100, + send_burst({n,NumOfReqs}, seq, {chars,79}, info), + Logged = count_lines(Log), + ct:pal("Number of messages logged = ~w", [Logged]), + ok = file:delete(Log), + true = is_pid(whereis(?MODULE)), + ok. +kill_disabled(cleanup, _Config) -> + ok = stop_handler(?MODULE). + +qlen_kill_new(Config) -> + {Log,HConfig,StdHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + Pid0 = whereis(?MODULE), + {_,Mem0} = process_info(Pid0, memory), + RestartAfter = 2000, + NewHConfig = + HConfig#{logger_std_h=>StdHConfig#{enable_kill_overloaded=>true, + handler_overloaded_qlen=>10, + handler_overloaded_mem=>Mem0+50000, + handler_restart_after=>RestartAfter}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + MRef = erlang:monitor(process, Pid0), + NumOfReqs = 100, + Procs = 2, + send_burst({n,NumOfReqs}, {spawn,Procs,0}, {chars,79}, info), + %% send_burst({n,NumOfReqs}, seq, {chars,79}, info), + receive + {'DOWN', MRef, _, _, Info} -> + case Info of + {shutdown,{overloaded,?MODULE,QLen,Mem}} -> + ct:pal("Terminated with qlen = ~w, mem = ~w", [QLen,Mem]); + killed -> + ct:pal("Slow shutdown, handler process was killed!", []) + end, + timer:sleep(RestartAfter + 1000), + true = is_pid(whereis(?MODULE)), + ok + after + 5000 -> + Info = logger_std_h:info(?MODULE), + ct:pal("Handler state = ~p", [Info]), + ct:fail("Handler not dead! It should not have survived this!") + end. +qlen_kill_new(cleanup, _Config) -> + ok = stop_handler(?MODULE). + +%% choke the standard handler on remote node to verify the termination +%% works as expected +qlen_kill_std(Config) -> + %%! HERE + %% Dir = ?config(priv_dir, Config), + %% File = lists:concat([?MODULE,"_",?FUNCTION_NAME,".log"]), + %% Log = filename:join(Dir, File), + %% Node = start_std_h_on_new_node(Config, ?FUNCTION_NAME, Log), + %% ok = rpc:call(Node, logger, set_handler_config, + %% [?STANDARD_HANDLER, logger_std_h, + %% #{enable_kill_overloaded=>true, + %% handler_overloaded_qlen=>10, + %% handler_overloaded_mem=>100000}]), + {skip,"Not done yet"}. + +mem_kill_new(Config) -> + {Log,HConfig,StdHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + Pid0 = whereis(?MODULE), + {_,Mem0} = process_info(Pid0, memory), + RestartAfter = 2000, + NewHConfig = + HConfig#{logger_std_h=>StdHConfig#{enable_kill_overloaded=>true, + handler_overloaded_qlen=>50000, + handler_overloaded_mem=>Mem0+500, + handler_restart_after=>RestartAfter}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + MRef = erlang:monitor(process, Pid0), + NumOfReqs = 100, + Procs = 2, + send_burst({n,NumOfReqs}, {spawn,Procs,0}, {chars,79}, info), + %% send_burst({n,NumOfReqs}, seq, {chars,79}, info), + receive + {'DOWN', MRef, _, _, Info} -> + case Info of + {shutdown,{overloaded,?MODULE,QLen,Mem}} -> + ct:pal("Terminated with qlen = ~w, mem = ~w", [QLen,Mem]); + killed -> + ct:pal("Slow shutdown, handler process was killed!", []) + end, + timer:sleep(RestartAfter * 2), + true = is_pid(whereis(?MODULE)), + ok + after + 5000 -> + Info = logger_std_h:info(?MODULE), + ct:pal("Handler state = ~p", [Info]), + ct:fail("Handler not dead! It should not have survived this!") + end. +mem_kill_new(cleanup, _Config) -> + ok = stop_handler(?MODULE). + +%% choke the standard handler on remote node to verify the termination +%% works as expected +mem_kill_std(Config) -> + {skip,"Not done yet"}. + +restart_after(Config) -> + {Log,HConfig,StdHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + NewHConfig1 = + HConfig#{logger_std_h=>StdHConfig#{enable_kill_overloaded=>true, + handler_overloaded_qlen=>10, + handler_restart_after=>never}}, + ok = logger:set_handler_config(?MODULE, NewHConfig1), + MRef1 = erlang:monitor(process, whereis(?MODULE)), + %% kill handler + send_burst({n,100}, {spawn,2,0}, {chars,79}, info), + receive + {'DOWN', MRef1, _, _, _Info1} -> + timer:sleep(?HANDLER_RESTART_AFTER + 1000), + undefined = whereis(?MODULE), + ok + after + 5000 -> + ct:fail("Handler not dead! It should not have survived this!") + end, + + {Log,_,_} = start_handler(?MODULE, ?FUNCTION_NAME, Config), + RestartAfter = 2000, + NewHConfig2 = + HConfig#{logger_std_h=>StdHConfig#{enable_kill_overloaded=>true, + handler_overloaded_qlen=>10, + handler_restart_after=>RestartAfter}}, + ok = logger:set_handler_config(?MODULE, NewHConfig2), + Pid0 = whereis(?MODULE), + MRef2 = erlang:monitor(process, Pid0), + %% kill handler + send_burst({n,100}, {spawn,2,0}, {chars,79}, info), + receive + {'DOWN', MRef2, _, _, _Info2} -> + timer:sleep(RestartAfter + 1000), + Pid1 = whereis(?MODULE), + true = is_pid(Pid1), + false = (Pid1 == Pid0), + ok + after + 5000 -> + ct:fail("Handler not dead! It should not have survived this!") + end, + ok. +restart_after(cleanup, _Config) -> + ok = stop_handler(?MODULE). + +%% send handler requests (filesync, info, reset, change_config) +%% during high load to verify that sync, dropping and flushing is +%% handled correctly. +handler_requests_under_load() -> + [{timetrap,{seconds,60}}]. +handler_requests_under_load(Config) -> + {Log,HConfig,StdHConfig} = + start_handler(?MODULE, ?FUNCTION_NAME, Config), + NewHConfig = + HConfig#{logger_std_h => StdHConfig#{toggle_sync_qlen => 2, + drop_new_reqs_qlen => 1000, + flush_reqs_qlen => 2000, + enable_burst_limit => false}}, + ok = logger:set_handler_config(?MODULE, NewHConfig), + Pid = spawn_link(fun() -> send_requests(?MODULE, 1, [{filesync,[]}, + {info,[]}, + {reset,[]}, + {change_config,[]}]) + end), + Sent = send_burst({t,10000}, seq, {chars,79}, info), + Pid ! {self(),finish}, + ReqResult = receive {Pid,Result} -> Result end, + Logged = count_lines(Log), + ct:pal("Number of messages sent = ~w~nNumber of messages logged = ~w", + [Sent,Logged]), + FindError = fun(Res) -> + [E || E <- Res, + is_tuple(E) andalso (element(1,E) == error)] + end, + Errors = [{Req,FindError(Res)} || {Req,Res} <- ReqResult], + NoOfReqs = lists:foldl(fun({_,Res}, N) -> N + length(Res) end, 0, ReqResult), + ct:pal("~w requests made. Errors: ~n~p", [NoOfReqs,Errors]), + ok = file:delete(Log). +handler_requests_under_load(cleanup, Config) -> + ok = stop_handler(?MODULE). + +send_requests(HName, TO, Reqs = [{Req,Res}|Rs]) -> + receive + {From,finish} -> + From ! {self(),Reqs} + after + TO -> + Result = + case Req of + change_config -> + logger:set_handler_config(HName, logger_std_h, + #{enable_kill_overloaded => + false}); + Func -> + logger_std_h:Func(HName) + end, + send_requests(HName, TO, Rs ++ [{Req,[Result|Res]}]) + end. + + +%%%----------------------------------------------------------------- +%%% +start_handler(Name, TTY, Config) when TTY == standard_io; + TTY == standard_error-> + ok = logger:add_handler(Name, + logger_std_h, + #{logger_std_h => #{type => TTY}, + filter_default=>log, + filters=>?DEFAULT_HANDLER_FILTERS([Name]), + formatter=>{?MODULE,op}}), + {ok,{_,HConfig = #{logger_std_h := StdHConfig}}} = + logger:get_handler_config(Name), + {HConfig,StdHConfig}; + +start_handler(Name, FuncName, Config) -> + Dir = ?config(priv_dir,Config), + Log = filename:join(Dir, lists:concat([FuncName,".log"])), + ct:pal("Logging to ~tp", [Log]), + Type = {file,Log}, + ok = logger:add_handler(Name, + logger_std_h, + #{logger_std_h => #{type => Type}, + filter_default=>log, + filters=>?DEFAULT_HANDLER_FILTERS([Name]), + formatter=>{?MODULE,op}}), + {ok,{_,HConfig = #{logger_std_h := StdHConfig}}} = + logger:get_handler_config(Name), + {Log,HConfig,StdHConfig}. + +stop_handler(Name) -> + ok = logger:remove_handler(Name), + ct:pal("Handler ~p stopped!", [Name]). + +count_lines(File) -> + wait_until_written(File, -1), + count_lines1(File). + +wait_until_written(File, Sz) -> + timer:sleep(2000), + case file:read_file_info(File) of + {ok,#file_info{size = Sz}} -> + timer:sleep(1000), + case file:read_file_info(File) of + {ok,#file_info{size = Sz1}} -> + ok; + {ok,#file_info{size = Sz2}} -> + wait_until_written(File, Sz2) + end; + {ok,#file_info{size = Sz1}} -> + wait_until_written(File, Sz1) + end. + +count_lines1(File) -> + Counter = fun Cnt(Dev,LC) -> + case file:read_line(Dev) of + eof -> LC; + _ -> Cnt(Dev,LC+1) + end + end, + {_,Dev} = file:open(File, [read]), + Lines = Counter(Dev, 0), + file:close(Dev), + Lines. + +send_burst(NorT, Type, {chars,Sz}, Class) -> + Text = [34 + rand:uniform(126-34) || _ <- lists:seq(1,Sz)], + case NorT of + {n,N} -> + %% process_flag(priority, high), + send_n_burst(N, Type, Text, Class), + %% process_flag(priority, normal), + N; + {t,T} -> + ct:pal("Sending messages sequentially for ~w ms", [T]), + T0 = erlang:monotonic_time(millisecond), + send_t_burst(T0, T, Text, Class, 0) + end. + +send_n_burst(0, _, _Text, _Class) -> + ok; +send_n_burst(N, seq, Text, Class) -> + ok = logger:Class(Text, ?domain), + send_n_burst(N-1, seq, Text, Class); +send_n_burst(N, {spawn,Ps,TO}, Text, Class) -> + ct:pal("~w processes each sending ~w messages", [Ps,N]), + PerProc = fun() -> + send_n_burst(N, seq, Text, Class) + end, + MRefs = [begin if TO == 0 -> ok; true -> timer:sleep(TO) end, + monitor(process,spawn_link(PerProc)) end || + _ <- lists:seq(1,Ps)], + lists:foreach(fun(MRef) -> + receive + {'DOWN', MRef, _, _, _} -> + ok + end + end, MRefs), + ct:pal("Message burst sent", []), + ok. + +send_t_burst(T0, T, Text, Class, N) -> + T1 = erlang:monotonic_time(millisecond), + if (T1-T0) > T -> + N; + true -> + ok = logger:Class(Text, ?domain), + send_t_burst(T0, T, Text, Class, N+1) + end. + +%%%----------------------------------------------------------------- +%%% Formatter callback +%%% Using this to send the formatted string back to the test case +%%% process - so it can check for logged events. +format(_,bad_return) -> + bad_return; +format(_,crash) -> + erlang:error(formatter_crashed); +format(#{msg:={string,String0}},no_nl) -> + String = unicode:characters_to_list(String0), + String; +format(#{msg:={string,String0}},nl) -> + String = unicode:characters_to_list(String0), + String++"\n"; +format(#{msg:={string,String0}},op) -> + String = unicode:characters_to_list(String0), + String++"\n"; +format(#{msg:={report,#{label:={supervisor,progress}}}},op) -> + ""; +format(#{msg:={report,#{label:={gen_server,terminate}}}},op) -> + ""; +format(#{msg:={report,#{label:={proc_lib,crash}}}},op) -> + ""; +format(#{msg:={F,A}},Pid) when is_list(F), is_list(A) -> + String = lists:flatten(io_lib:format(F,A)), + Pid ! {log,String}, + String++"\n"; +format(#{msg:={string,String0}},Pid) -> + String = unicode:characters_to_list(String0), + Pid ! {log,String}, + String++"\n". + +add_remove_instance_nofile(Type) -> + ok = logger:add_handler(?MODULE,logger_std_h, + #{logger_std_h => #{type => Type}, + filter_default=>stop, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), + formatter=>{?MODULE,self()}}), + Pid = whereis(?MODULE), + true = is_pid(Pid), + group_leader(group_leader(),Pid), % to get printouts in test log + logger:info(M1=?msg,?domain), + ?check(M1), + %% check that filesync doesn't do damage even if not relevant + ok = logger_std_h:filesync(?MODULE), + ok = logger:remove_handler(?MODULE), + timer:sleep(500), + undefined = whereis(?MODULE), + logger:info(?msg,?domain), + ?check_no_log, + ok. + +logger_std_h_remove() -> + logger:remove_handler(?MODULE). +logger_std_h_remove(Id) -> + logger:remove_handler(Id). + +try_read_file(FileName, Expected, Time) when Time > 0 -> + case file:read_file(FileName) of + Expected -> + ok; + Error = {error,_Reason} -> + ct:pal("Can't read ~tp: ~tp", [FileName,Error]), + erlang:error(Error); + Got -> + ct:pal("try_read_file got ~tp", [Got]), + timer:sleep(500), + try_read_file(FileName, Expected, Time-500) + end; +try_read_file(FileName, Expected, _) -> + ct:pal("Missing pattern ~tp in ~tp", [Expected,FileName]), + erlang:error({error,missing_expected_pattern}). + +try_match_file(FileName, Pattern, Time) -> + try_match_file(FileName, Pattern, Time, <<>>). + +try_match_file(FileName, Pattern, Time, _) when Time > 0 -> + case file:read_file(FileName) of + {ok, Bin} -> + case re:run(Bin,Pattern,[{capture,none}]) of + match -> + unicode:characters_to_list(Bin); + _ -> + timer:sleep(100), + try_match_file(FileName, Pattern, Time-100, Bin) + end; + Error -> + erlang:error(Error) + end; +try_match_file(_,Pattern,_,Incorrect) -> + ct:pal("try_match_file did not match pattern: ~p~nGot: ~p~n", + [Pattern,Incorrect]), + erlang:error({error,not_matching_pattern,Pattern,Incorrect}). + +%%%----------------------------------------------------------------- +%%% +start_op_trace() -> + TraceFun = fun({trace,_,call,{_Mod,Func,Details}}, Pid) -> + Pid ! {trace_call,Func,Details}, + Pid; + ({trace,_,return_from,{_Mod,Func,_},RetVal}, Pid) -> + Pid ! {trace_return,Func,RetVal}, + Pid + end, + TRecvPid = spawn_link(fun() -> trace_receiver(5000) end), + {ok,_} = dbg:tracer(process, {TraceFun, TRecvPid}), + + {ok,_} = dbg:p(whereis(?MODULE), [c]), + {ok,_} = dbg:p(self(), [c]), + + MS1 = dbg:fun2ms(fun([_]) -> return_trace() end), + {ok,_} = dbg:tp(logger_h_common, check_load, 1, MS1), + + {ok,_} = dbg:tpl(logger_h_common, flush_log_requests, 2, []), + + MS2 = dbg:fun2ms(fun([_,mode]) -> return_trace() end), + {ok,_} = dbg:tpl(ets, lookup, 2, MS2), + + ct:pal("Tracing started!", []), + TRecvPid. + +stop_op_trace(TRecvPid) -> + dbg:stop_clear(), + unlink(TRecvPid), + exit(TRecvPid, kill), + ok. + +find_mode(flush, Events) -> + lists:any(fun({trace_call,flush_log_requests,[_,_]}) -> true; + (_) -> false + end, Events); +find_mode(Mode, Events) -> + lists:keymember([{mode,Mode}], 3, Events). + +find_switch(From, To, Events) -> + try lists:foldl(fun({trace_return,check_load,{To,_,_,_}}, + {trace_call,check_load,[#{mode := From}]}) -> + throw(match); + (Event, _) -> + Event + end, undefined, Events) of + _ -> false + catch + throw:match -> true + end. + +analyse_trace(TRecvPid, TestFun) -> + TRecvPid ! {test,self(),TestFun}, + receive + {result,TRecvPid,Result} -> + Result + after + 60000 -> + fails + end. + +trace_receiver(IdleT) -> + Msgs = receive_until_idle(IdleT, 5, []), + ct:pal("~w trace events generated", [length(Msgs)]), + analyse(Msgs). + +receive_until_idle(IdleT, WaitN, Msgs) -> + receive + Msg = {trace_call,_,_} -> + receive_until_idle(IdleT, 5, [Msg | Msgs]); + Msg = {trace_return,_,_} -> + receive_until_idle(IdleT, 5, [Msg | Msgs]) + after + IdleT -> + if WaitN == 0 -> + Msgs; + true -> + receive_until_idle(IdleT, WaitN-1, Msgs) + end + end. + +analyse(Msgs) -> + receive + {test,From,TestFun} -> + From ! {result,self(),TestFun(Msgs)}, + analyse(Msgs) + end. diff --git a/lib/kernel/test/prim_file_SUITE.erl b/lib/kernel/test/prim_file_SUITE.erl index ab62f4dc34..5bb230d1c4 100644 --- a/lib/kernel/test/prim_file_SUITE.erl +++ b/lib/kernel/test/prim_file_SUITE.erl @@ -1306,7 +1306,7 @@ e_delete(Config) when is_list(Config) -> Base, #file_info {mode=0}), {error, eacces} = ?PRIM_FILE:delete(Afile), ?PRIM_FILE:write_file_info( - Base, #file_info {mode=8#600}) + Base, #file_info {mode=8#700}) end, ok. @@ -1442,7 +1442,7 @@ e_make_dir(Config) when is_list(Config) -> ?PRIM_FILE:write_file_info(Base, #file_info {mode=0}), {error, eacces} = ?PRIM_FILE:make_dir(filename:join(Base, "xxxx")), - ?PRIM_FILE:write_file_info(Base, #file_info {mode=8#600}) + ?PRIM_FILE:write_file_info(Base, #file_info {mode=8#700}) end, ok. @@ -1492,7 +1492,7 @@ e_del_dir(Config) when is_list(Config) -> ?PRIM_FILE:write_file_info(Base, #file_info {mode=0}), {error, eacces} = ?PRIM_FILE:del_dir(ADirectory), ?PRIM_FILE:write_file_info( - Base, #file_info {mode=8#600}) + Base, #file_info {mode=8#700}) end, ok. diff --git a/lib/kernel/test/seq_trace_SUITE.erl b/lib/kernel/test/seq_trace_SUITE.erl index be23a1933f..aae8a83304 100644 --- a/lib/kernel/test/seq_trace_SUITE.erl +++ b/lib/kernel/test/seq_trace_SUITE.erl @@ -25,7 +25,7 @@ -export([token_set_get/1, tracer_set_get/1, print/1, send/1, distributed_send/1, recv/1, distributed_recv/1, trace_exit/1, distributed_exit/1, call/1, port/1, - match_set_seq_token/1, gc_seq_token/1]). + match_set_seq_token/1, gc_seq_token/1, label_capability_mismatch/1]). %% internal exports -export([simple_tracer/2, one_time_receiver/0, one_time_receiver/1, @@ -47,7 +47,7 @@ all() -> [token_set_get, tracer_set_get, print, send, distributed_send, recv, distributed_recv, trace_exit, distributed_exit, call, port, match_set_seq_token, - gc_seq_token]. + gc_seq_token, label_capability_mismatch]. groups() -> []. @@ -90,8 +90,8 @@ do_token_set_get(TsType) -> %% Test that initial seq_trace is disabled [] = seq_trace:get_token(), %% Test setting and reading the different fields - 0 = seq_trace:set_token(label,17), - {label,17} = seq_trace:get_token(label), + 0 = seq_trace:set_token(label,{my_label,1}), + {label,{my_label,1}} = seq_trace:get_token(label), false = seq_trace:set_token(print,true), {print,true} = seq_trace:get_token(print), false = seq_trace:set_token(send,true), @@ -101,12 +101,12 @@ do_token_set_get(TsType) -> false = seq_trace:set_token(TsType,true), {TsType,true} = seq_trace:get_token(TsType), %% Check the whole token - {Flags,17,0,Self,0} = seq_trace:get_token(), % all flags are set + {Flags,{my_label,1},0,Self,0} = seq_trace:get_token(), % all flags are set %% Test setting and reading the 'serial' field {0,0} = seq_trace:set_token(serial,{3,5}), {serial,{3,5}} = seq_trace:get_token(serial), %% Check the whole token, test that a whole token can be set and get - {Flags,17,5,Self,3} = seq_trace:get_token(), + {Flags,{my_label,1},5,Self,3} = seq_trace:get_token(), seq_trace:set_token({Flags,19,7,Self,5}), {Flags,19,7,Self,5} = seq_trace:get_token(), %% Check that receive timeout does not reset token @@ -166,11 +166,13 @@ do_send(TsType) -> seq_trace:reset_trace(), start_tracer(), Receiver = spawn(?MODULE,one_time_receiver,[]), + Label = make_ref(), + seq_trace:set_token(label,Label), set_token_flags([send, TsType]), Receiver ! send, Self = self(), seq_trace:reset_trace(), - [{0,{send,_,Self,Receiver,send}, Ts}] = stop_tracer(1), + [{Label,{send,_,Self,Receiver,send}, Ts}] = stop_tracer(1), check_ts(TsType, Ts). distributed_send(Config) when is_list(Config) -> @@ -184,14 +186,19 @@ do_distributed_send(TsType) -> seq_trace:reset_trace(), start_tracer(), Receiver = spawn(Node,?MODULE,one_time_receiver,[]), + + %% Make sure complex labels survive the trip. + Label = make_ref(), + seq_trace:set_token(label,Label), set_token_flags([send,TsType]), + Receiver ! send, Self = self(), seq_trace:reset_trace(), stop_node(Node), - [{0,{send,_,Self,Receiver,send}, Ts}] = stop_tracer(1), + [{Label,{send,_,Self,Receiver,send}, Ts}] = stop_tracer(1), check_ts(TsType, Ts). - + recv(Config) when is_list(Config) -> lists:foreach(fun do_recv/1, ?TIMESTAMP_MODES). @@ -220,7 +227,12 @@ do_distributed_recv(TsType) -> seq_trace:reset_trace(), rpc:call(Node,?MODULE,start_tracer,[]), Receiver = spawn(Node,?MODULE,one_time_receiver,[]), + + %% Make sure complex labels survive the trip. + Label = make_ref(), + seq_trace:set_token(label,Label), set_token_flags(['receive',TsType]), + Receiver ! 'receive', %% let the other process receive the message: receive after 1 -> ok end, @@ -229,7 +241,7 @@ do_distributed_recv(TsType) -> Result = rpc:call(Node,?MODULE,stop_tracer,[1]), stop_node(Node), ok = io:format("~p~n",[Result]), - [{0,{'receive',_,Self,Receiver,'receive'}, Ts}] = Result, + [{Label,{'receive',_,Self,Receiver,'receive'}, Ts}] = Result, check_ts(TsType, Ts). trace_exit(Config) when is_list(Config) -> @@ -240,7 +252,12 @@ do_trace_exit(TsType) -> start_tracer(), Receiver = spawn_link(?MODULE, one_time_receiver, [exit]), process_flag(trap_exit, true), + + %% Make sure complex labels survive the trip. + Label = make_ref(), + seq_trace:set_token(label,Label), set_token_flags([send, TsType]), + Receiver ! {before, exit}, %% let the other process receive the message: receive @@ -254,8 +271,8 @@ do_trace_exit(TsType) -> Result = stop_tracer(2), seq_trace:reset_trace(), ok = io:format("~p~n", [Result]), - [{0, {send, {0,1}, Self, Receiver, {before, exit}}, Ts0}, - {0, {send, {1,2}, Receiver, Self, + [{Label, {send, {0,1}, Self, Receiver, {before, exit}}, Ts0}, + {Label, {send, {1,2}, Receiver, Self, {'EXIT', Receiver, {exit, {before, exit}}}}, Ts1}] = Result, check_ts(TsType, Ts0), check_ts(TsType, Ts1). @@ -291,6 +308,74 @@ do_distributed_exit(TsType) -> {'EXIT', Receiver, {exit, {before, exit}}}}, Ts}] = Result, check_ts(TsType, Ts). +label_capability_mismatch(Config) when is_list(Config) -> + Releases = ["20_latest"], + Available = [Rel || Rel <- Releases, test_server:is_release_available(Rel)], + case Available of + [] -> {skipped, "No incompatible releases available"}; + _ -> + lists:foreach(fun do_incompatible_labels/1, Available), + lists:foreach(fun do_compatible_labels/1, Available), + ok + end. + +do_incompatible_labels(Rel) -> + Cookie = atom_to_list(erlang:get_cookie()), + {ok, Node} = test_server:start_node( + list_to_atom(atom_to_list(?MODULE)++"_"++Rel), peer, + [{args, " -setcookie "++Cookie}, {erl, [{release, Rel}]}]), + + {_,Dir} = code:is_loaded(?MODULE), + Mdir = filename:dirname(Dir), + true = rpc:call(Node,code,add_patha,[Mdir]), + seq_trace:reset_trace(), + rpc:call(Node,?MODULE,start_tracer,[]), + Receiver = spawn(Node,?MODULE,one_time_receiver,[]), + + %% This node does not support arbitrary labels, so it must fail with a + %% timeout as the token is dropped silently. + seq_trace:set_token(label,make_ref()), + seq_trace:set_token('receive',true), + + Receiver ! 'receive', + %% let the other process receive the message: + receive after 10 -> ok end, + seq_trace:reset_trace(), + + {error,timeout} = rpc:call(Node,?MODULE,stop_tracer,[1]), + stop_node(Node), + ok. + +do_compatible_labels(Rel) -> + Cookie = atom_to_list(erlang:get_cookie()), + {ok, Node} = test_server:start_node( + list_to_atom(atom_to_list(?MODULE)++"_"++Rel), peer, + [{args, " -setcookie "++Cookie}, {erl, [{release, Rel}]}]), + + {_,Dir} = code:is_loaded(?MODULE), + Mdir = filename:dirname(Dir), + true = rpc:call(Node,code,add_patha,[Mdir]), + seq_trace:reset_trace(), + rpc:call(Node,?MODULE,start_tracer,[]), + Receiver = spawn(Node,?MODULE,one_time_receiver,[]), + + %% This node does not support arbitrary labels, but small integers should + %% still work. + Label = 1234, + seq_trace:set_token(label,Label), + seq_trace:set_token('receive',true), + + Receiver ! 'receive', + %% let the other process receive the message: + receive after 10 -> ok end, + Self = self(), + seq_trace:reset_trace(), + Result = rpc:call(Node,?MODULE,stop_tracer,[1]), + stop_node(Node), + ok = io:format("~p~n",[Result]), + [{Label,{'receive',_,Self,Receiver,'receive'}, _}] = Result, + ok. + call(doc) -> "Tests special forms {is_seq_trace} and {get_seq_token} " "in trace match specs."; diff --git a/lib/kernel/test/wrap_log_reader_SUITE.erl b/lib/kernel/test/wrap_log_reader_SUITE.erl index 40a016aed0..b1ee29a11f 100644 --- a/lib/kernel/test/wrap_log_reader_SUITE.erl +++ b/lib/kernel/test/wrap_log_reader_SUITE.erl @@ -428,13 +428,14 @@ stop() -> ok = wrap_log_test:stop(), dl_wait(). -%% Give disk logs opened by 'logger' and 'wlt' time to close after +%% Give disk logs opened by 'wlr_logger' and 'wlt' time to close after %% receiving EXIT signals. dl_wait() -> case disk_log:accessible_logs() of {[], []} -> ok; - _ -> + _X -> + erlang:display(_X), timer:sleep(100), dl_wait() end. @@ -507,27 +508,27 @@ add_ext(Name, Ext) -> %% disk_log. open(Log, File, Where) -> - logger ! {open, self(), Log, File}, + wlr_logger ! {open, self(), Log, File}, rec1(ok, Where). open_ext(Log, File, Where) -> - logger ! {open_ext, self(), Log, File}, + wlr_logger ! {open_ext, self(), Log, File}, rec1(ok, Where). close(Log) -> - logger ! {close, self(), Log}, + wlr_logger ! {close, self(), Log}, rec(ok, ?LINE). sync(Log) -> - logger ! {sync, self(), Log}, + wlr_logger ! {sync, self(), Log}, rec(ok, ?LINE). log_terms(File, Terms) -> - logger ! {log_terms, self(), File, Terms}, + wlr_logger ! {log_terms, self(), File, Terms}, rec(ok, ?LINE). blog_terms(File, Terms) -> - logger ! {blog_terms, self(), File, Terms}, + wlr_logger ! {blog_terms, self(), File, Terms}, rec(ok, ?LINE). rec1(M, Where) -> diff --git a/lib/kernel/test/wrap_log_reader_SUITE_data/wrap_log_test.erl b/lib/kernel/test/wrap_log_reader_SUITE_data/wrap_log_test.erl index 38449b6bb3..2b24ccc66f 100644 --- a/lib/kernel/test/wrap_log_reader_SUITE_data/wrap_log_test.erl +++ b/lib/kernel/test/wrap_log_reader_SUITE_data/wrap_log_test.erl @@ -36,9 +36,9 @@ -endif. init() -> - spawn(fun() -> start(logger) end), + spawn(fun() -> start(wlr_logger) end), spawn(fun() -> start2(wlt) end), - wait_registered(logger), + wait_registered(wlr_logger), wait_registered(wlt), ok. @@ -52,9 +52,9 @@ wait_registered(Name) -> end. stop() -> - catch logger ! exit, + catch wlr_logger ! exit, catch wlt ! exit, - wait_unregistered(logger), + wait_unregistered(wlr_logger), wait_unregistered(wlt), ok. @@ -82,47 +82,47 @@ loop() -> {open, Pid, Name, File} -> R = disk_log:open([{name, Name}, {type, wrap}, {file, File}, {size, {?fsize, ?fno}}]), - ?format("logger: open ~p -> ~p~n", [Name, R]), + ?format("wlr_logger: open ~p -> ~p~n", [Name, R]), Pid ! R, loop(); {open_ext, Pid, Name, File} -> R = disk_log:open([{name, Name}, {type, wrap}, {file, File}, {format, external}, {size, {?fsize, ?fno}}]), - ?format("logger: open ~p -> ~p~n", [Name, R]), + ?format("wlr_logger: open ~p -> ~p~n", [Name, R]), Pid ! R, loop(); {close, Pid, Name} -> R = disk_log:close(Name), - ?format("logger: close ~p -> ~p~n", [Name, R]), + ?format("wlr_logger: close ~p -> ~p~n", [Name, R]), Pid ! R, loop(); {sync, Pid, Name} -> R = disk_log:sync(Name), - ?format("logger: sync ~p -> ~p~n", [Name, R]), + ?format("wlr_logger: sync ~p -> ~p~n", [Name, R]), Pid ! R, loop(); {log_terms, Pid, Name, Terms} -> R = disk_log:log_terms(Name, Terms), - ?format("logger: log_terms ~p -> ~p~n", [Name, R]), + ?format("wlr_logger: log_terms ~p -> ~p~n", [Name, R]), Pid ! R, loop(); {blog_terms, Pid, Name, Terms} -> R = disk_log:blog_terms(Name, Terms), - ?format("logger: blog_terms ~p -> ~p~n", [Name, R]), + ?format("wlr_logger: blog_terms ~p -> ~p~n", [Name, R]), Pid ! R, loop(); exit -> - ?format("Stopping logger~n", []), + ?format("Stopping wlr_logger~n", []), exit(normal); _Else -> - ?format("logger: ignored: ~p~n", [_Else]), + ?format("wlr_logger: ignored: ~p~n", [_Else]), loop() end. diff --git a/lib/kernel/test/zzz_SUITE.erl b/lib/kernel/test/zzz_SUITE.erl new file mode 100644 index 0000000000..59c7fd7404 --- /dev/null +++ b/lib/kernel/test/zzz_SUITE.erl @@ -0,0 +1,37 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2006-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(zzz_SUITE). + +%% The sole purpose of this test suite is for things we want to run last +%% before the VM terminates. + +-export([all/0]). + +-export([lc_graph/1]). + + +all() -> + [lc_graph]. + +lc_graph(_Config) -> + %% Create "lc_graph" file in current working dir + %% if lock checker is enabled. + erts_debug:lc_graph(), + ok. diff --git a/lib/mnesia/doc/src/Makefile b/lib/mnesia/doc/src/Makefile index 82fcf66256..aed46d50db 100644 --- a/lib/mnesia/doc/src/Makefile +++ b/lib/mnesia/doc/src/Makefile @@ -49,16 +49,18 @@ XML_PART_FILES = \ XML_CHAPTER_FILES = \ Mnesia_chap1.xml \ Mnesia_overview.xml \ + Mnesia_chap8.xml \ + notes.xml + +XML_CHAPTER_GEN_FILES = \ Mnesia_chap2.xml \ Mnesia_chap3.xml \ Mnesia_chap4.xml \ Mnesia_chap5.xml \ Mnesia_chap7.xml \ - Mnesia_chap8.xml \ Mnesia_App_A.xml \ Mnesia_App_B.xml \ - Mnesia_App_C.xml \ - notes.xml + Mnesia_App_C.xml BOOK_FILES = book.xml @@ -66,6 +68,8 @@ XML_FILES = \ $(BOOK_FILES) $(XML_CHAPTER_FILES) \ $(XML_PART_FILES) $(XML_REF3_FILES) $(XML_APPLICATION_FILES) +XML_GEN_FILES = $(XML_CHAPTER_GEN_FILES:%=$(XMLDIR)/%) + GIF_FILES = \ company.gif diff --git a/lib/mnesia/test/mnesia_recovery_test.erl b/lib/mnesia/test/mnesia_recovery_test.erl index 82d6e6ac6a..1783d2ae94 100644 --- a/lib/mnesia/test/mnesia_recovery_test.erl +++ b/lib/mnesia/test/mnesia_recovery_test.erl @@ -730,6 +730,7 @@ do_trans_loop2(Tab, Father) -> do_trans_loop2(Tab, Father); Else -> ?error("Transaction failed: ~p ~n", [Else]), + io:format("INFO: ~p~n",[erlang:process_info(self())]), Father ! test_done, exit(shutdown) end. diff --git a/lib/observer/src/observer_pro_wx.erl b/lib/observer/src/observer_pro_wx.erl index 1c40afba46..e2372cde33 100644 --- a/lib/observer/src/observer_pro_wx.erl +++ b/lib/observer/src/observer_pro_wx.erl @@ -50,6 +50,7 @@ -define(ID_TRACE_NEW, 208). -define(ID_TRACE_ALL, 209). -define(ID_ACCUMULATE, 210). +-define(ID_GARBAGE_COLLECT, 211). -define(TRACE_PIDS_STR, "Trace selected process identifiers"). -define(TRACE_NAMES_STR, "Trace selected processes, " @@ -147,11 +148,11 @@ create_list_box(Panel, Holder) -> ListCtrl = wxListCtrl:new(Panel, [{style, Style}, {onGetItemText, fun(_, Row, Col) -> - call(Holder, {get_row, self(), Row, Col}) + safe_call(Holder, {get_row, self(), Row, Col}) end}, {onGetItemAttr, fun(_, Item) -> - call(Holder, {get_attr, self(), Item}) + safe_call(Holder, {get_attr, self(), Item}) end} ]), Li = wxListItem:new(), @@ -208,17 +209,26 @@ start_procinfo(Pid, Frame, Opened) -> Opened end. + +safe_call(Holder, What) -> + case call(Holder, What, 2000) of + Res when is_atom(Res) -> ""; + Res -> Res + end. + call(Holder, What) -> + call(Holder, What, infinity). + +call(Holder, What, TMO) -> Ref = erlang:monitor(process, Holder), Holder ! What, receive - {'DOWN', Ref, _, _, _} -> ""; + {'DOWN', Ref, _, _, _} -> holder_dead; {Holder, Res} -> erlang:demonitor(Ref), Res - after 2000 -> - io:format("Hanging call ~tp~n",[What]), - "" + after TMO -> + timeout end. %%%%%%%%%%%%%%%%%%%%%%% Callbacks %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -269,7 +279,10 @@ code_change(_, _, State) -> handle_call(get_config, _, #state{holder=Holder, timer=Timer}=State) -> Conf = observer_lib:timer_config(Timer), - Accum = call(Holder, {get_accum, self()}), + Accum = case safe_call(Holder, {get_accum, self()}) of + Bool when is_boolean(Bool) -> Bool; + _ -> false + end, {reply, Conf#{acc=>Accum}, State}; handle_call(Msg, _From, State) -> @@ -316,6 +329,9 @@ handle_event(#wx{id=?ID_KILL}, #state{right_clicked_pid=Pid, sel=Sel0}=State) -> Sel = rm_selected(Pid,Sel0), {noreply, State#state{sel=Sel}}; +handle_event(#wx{id=?ID_GARBAGE_COLLECT}, #state{sel={_, Pids}}=State) -> + _ = [rpc:call(node(Pid), erlang, garbage_collect, [Pid]) || Pid <- Pids], + {noreply, State}; handle_event(#wx{id=?ID_PROC}, #state{panel=Panel, right_clicked_pid=Pid, procinfo_menu_pids=Opened}=State) -> @@ -370,6 +386,7 @@ handle_event(#wx{event=#wxList{type=command_list_item_right_click, wxMenu:append(Menu, ?ID_TRACE_NAMES, "Trace selected processes by name (all nodes)", [{help, ?TRACE_NAMES_STR}]), + wxMenu:append(Menu, ?ID_GARBAGE_COLLECT, "Garbage collect processes"), wxMenu:append(Menu, ?ID_KILL, "Kill process " ++ pid_to_list(P)), wxWindow:popupMenu(Panel, Menu), wxMenu:destroy(Menu), @@ -465,7 +482,7 @@ init_table_holder(Parent, Accum0, Attrs) -> Backend = spawn_link(node(), observer_backend, procs_info, [self()]), Accum = case Accum0 of true -> true; - false -> [] + _ -> [] end, table_holder(#holder{parent=Parent, info=array:new(), diff --git a/lib/observer/test/crashdump_viewer_SUITE.erl b/lib/observer/test/crashdump_viewer_SUITE.erl index 41ca3f3ce9..8bd7ad387b 100644 --- a/lib/observer/test/crashdump_viewer_SUITE.erl +++ b/lib/observer/test/crashdump_viewer_SUITE.erl @@ -674,7 +674,7 @@ do_create_dumps(DataDir,Rel) -> end, case Rel of current -> - CD3 = dump_with_args(DataDir,Rel,"instr","+Mim true"), + CD3 = dump_with_args(DataDir,Rel,"instr","+Muatags true"), CD4 = dump_with_strange_module_name(DataDir,Rel,"strangemodname"), CD5 = dump_with_size_limit_reached(DataDir,Rel,"trunc_bytes"), CD6 = dump_with_unicode_atoms(DataDir,Rel,"unicode"), diff --git a/lib/os_mon/src/memsup.erl b/lib/os_mon/src/memsup.erl index 95cb798ba5..a30d962ad4 100644 --- a/lib/os_mon/src/memsup.erl +++ b/lib/os_mon/src/memsup.erl @@ -705,7 +705,7 @@ get_os_wordsize_with_uname() -> _ -> 32 end. -clean_string(String) -> lists:flatten(string:lexemes(String,"\r\n\t ")). +clean_string(String) -> lists:flatten(string:lexemes(String,[[$\r,$\n]|"\n\t "])). %%--Replying to pending clients----------------------------------------- diff --git a/lib/public_key/doc/src/public_key.xml b/lib/public_key/doc/src/public_key.xml index dea35bc390..7284da0499 100644 --- a/lib/public_key/doc/src/public_key.xml +++ b/lib/public_key/doc/src/public_key.xml @@ -95,10 +95,12 @@ <p><c>| {#'PBEParameter{}, digest_type()} | #'PBES2-params'{}}</c></p> </item> - <tag><c>public_key() =</c></tag> + <tag><marker id="type-public_key"/> + <c>public_key() =</c></tag> <item><p><c>rsa_public_key() | dsa_public_key() | ec_public_key()</c></p></item> - <tag><c>private_key() =</c></tag> + <tag><marker id="type-private_key"/> + <c>private_key() =</c></tag> <item><p><c>rsa_private_key() | dsa_private_key() | ec_private_key()</c></p></item> <tag><c>rsa_public_key() =</c></tag> diff --git a/lib/reltool/doc/src/reltool_examples.xml b/lib/reltool/doc/src/reltool_examples.xml index 30cb3c13b6..2a103119e6 100644 --- a/lib/reltool/doc/src/reltool_examples.xml +++ b/lib/reltool/doc/src/reltool_examples.xml @@ -5,7 +5,7 @@ <header> <copyright> <year>2009</year> - <year>2017</year> + <year>2018</year> <holder>Ericsson AB, All Rights Reserved</holder> </copyright> <legalnotice> @@ -150,13 +150,13 @@ Eshell V9.0 (abort with ^G) {mod_cond,all}, {incl_cond,derived}, {erts,[{app,erts, - [{vsn,"9.0"}, - {lib_dir,"/usr/local/lib/erlang/lib/erts-9.0"}, + [{vsn,"10.0"}, + {lib_dir,"/usr/local/lib/erlang/lib/erts-10.0"}, {mod,erl_prim_loader,[]}, {mod,erl_tracer,[]}, {mod,erlang,[]}, {mod,erts_code_purger,[]}, - {mod,erts_dirty_process_code_checker,[]}, + {mod,erts_dirty_process_signal_handler,[]}, {mod,erts_internal,[]}, {mod,erts_literal_area_collector,[]}, {mod,init,[]}, @@ -309,9 +309,9 @@ Eshell V9.0 (abort with ^G) <section> <title>Generate release and script files</title> <pre> -Erlang/OTP 20 [erts-9.0] [source-c13b302] [64-bit] [smp:4:4] [ds:4:4:10] [async-threads:10] +Erlang/OTP 20 [erts-10.0] [source-c13b302] [64-bit] [smp:4:4] [ds:4:4:10] [async-threads:10] [hipe] [kernel-poll:false] -Eshell V9.0 (abort with ^G) +Eshell V10.0 (abort with ^G) 1> 1> {ok, Server} = reltool:start_server([{config, {sys, [{boot_rel, "NAME"}, {rel, "NAME", "VSN", @@ -324,13 +324,13 @@ Eshell V9.0 (abort with ^G) 3> 3> reltool:get_rel(Server, "NAME"). {ok,{release,{"NAME","VSN"}, - {erts,"9.0"}, + {erts,"10.0"}, [{kernel,"5.2"},{stdlib,"3.3"},{sasl,"3.0.3"}]}} 4> 4> reltool:get_script(Server, "NAME"). {ok,{script,{"NAME","VSN"}, [{preLoaded,[erl_prim_loader,erl_tracer,erlang, - erts_code_purger,erts_dirty_process_code_checker, + erts_code_purger,erts_dirty_process_signal_handler, erts_internal,erts_literal_area_collector,init,otp_ring0, prim_eval,prim_file,prim_inet,prim_zip,zlib]}, {progress,preloaded}, @@ -374,9 +374,9 @@ ok <section> <title>Create a target system</title> <pre> -Erlang/OTP 20 [erts-9.0] [source-c13b302] [64-bit] [smp:4:4] [ds:4:4:10] [async-threads:10] +Erlang/OTP 20 [erts-10.0] [source-c13b302] [64-bit] [smp:4:4] [ds:4:4:10] [async-threads:10] [hipe] [kernel-poll:false] -Eshell V9.0 (abort with ^G) +Eshell V10.0 (abort with ^G) 1> 1> Config = {sys, [{escript, "examples/display_args", [{incl_cond, include}]}, {app, inets, [{incl_cond, include}]}, @@ -393,7 +393,7 @@ Eshell V9.0 (abort with ^G) 2> 2> {ok, Spec} = reltool:get_target_spec([Config]). {ok,[{create_dir,"releases", - [{write_file,"start_erl.data","9.0 1.0\n"}, + [{write_file,"start_erl.data","10.0 1.0\n"}, {create_dir,"1.0", [{write_file,"start_clean.rel", [37,37,32,114,101,108,32,103,101,110,101,114,97,116|...]}, @@ -410,17 +410,17 @@ Eshell V9.0 (abort with ^G) {create_dir,"bin", [{copy_file,"display_args.escript", "/usr/local/lib/erlang/lib/reltool-0.7.3/examples/display_args"}, - {copy_file,"display_args","erts-9.0/bin/escript"}, - {copy_file,"start","erts-9.0/bin/start"}, - {copy_file,"ct_run","erts-9.0/bin/ct_run"}, - {copy_file,"dialyzer","erts-9.0/bin/dialyzer"}, - {copy_file,"run_erl","erts-9.0/bin/run_erl"}, - {copy_file,"erl","erts-9.0/bin/dyn_erl"}, - {copy_file,"to_erl","erts-9.0/bin/to_erl"}, - {copy_file,"epmd","erts-9.0/bin/epmd"}, - {copy_file,"erlc","erts-9.0/bin/erlc"}, - {copy_file,"typer","erts-9.0/bin/typer"}, - {copy_file,"escript","erts-9.0/bin/escript"}, + {copy_file,"display_args","erts-10.0/bin/escript"}, + {copy_file,"start","erts-10.0/bin/start"}, + {copy_file,"ct_run","erts-10.0/bin/ct_run"}, + {copy_file,"dialyzer","erts-10.0/bin/dialyzer"}, + {copy_file,"run_erl","erts-10.0/bin/run_erl"}, + {copy_file,"erl","erts-10.0/bin/dyn_erl"}, + {copy_file,"to_erl","erts-10.0/bin/to_erl"}, + {copy_file,"epmd","erts-10.0/bin/epmd"}, + {copy_file,"erlc","erts-10.0/bin/erlc"}, + {copy_file,"typer","erts-10.0/bin/typer"}, + {copy_file,"escript","erts-10.0/bin/escript"}, {write_file,"start_clean.boot",<<131,104,3,119,6,115,...>>}, {write_file,"start_sasl.boot",<<131,104,3,119,6,...>>}, {write_file,"start.boot",<<131,104,3,119,...>>}]}, @@ -451,7 +451,7 @@ Eshell V9.0 (abort with ^G) {copy_file,[...]}, {copy_file,...}, {...}]}]}, - {create_dir,"erts-9.0", + {create_dir,"erts-10.0", [{create_dir,"bin", [{copy_file,"start"}, {copy_file,"ct_run"}, @@ -459,7 +459,7 @@ Eshell V9.0 (abort with ^G) {copy_file,"dialyzer"}, {copy_file,"beam.smp"}, {copy_file,"run_erl"}, - {copy_file,"erl","erts-9.0/bin/dyn_erl"}, + {copy_file,"erl","erts-10.0/bin/dyn_erl"}, {copy_file,"to_erl"}, {copy_file,"epmd"}, {copy_file,"erl_child_setup"}, @@ -511,8 +511,8 @@ Eshell V9.0 (abort with ^G) [{create_dir,"priv", [{create_dir,"lib",[{copy_file,[...]},{copy_file,...}]}, {create_dir,"obj",[{copy_file,...},{...}|...]}]}]}, - {archive,"erts-9.0.ez",[], - [{create_dir,"erts-9.0", + {archive,"erts-10.0.ez",[], + [{create_dir,"erts-10.0", [{create_dir,"src",[{...}|...]}, {create_dir,"ebin",[...]}]}]}, {archive,"hipe-3.15.4.ez",[], @@ -549,14 +549,14 @@ ok ok 7> 7> file:list_dir(TargetDir). -{ok,["bin","Install","lib","misc","usr","erts-9.0", +{ok,["bin","Install","lib","misc","usr","erts-10.0", "releases"]} 8> 8> file:list_dir(filename:join([TargetDir,"lib"])). {ok,["tools-2.9.1.ez","kernel-5.2.ez","inets-6.3.9.ez", "kernel-5.2","sasl-3.0.3.ez","hipe-3.15.4.ez","inets-6.3.9", "crypto-3.7.4","crypto-3.7.4.ez","stdlib-3.3.ez", - "erts-9.0.ez","stdlib-3.3","compiler-7.0.4.ez"]} + "erts-10.0.ez","stdlib-3.3","compiler-7.0.4.ez"]} 9> 9> file:make_dir("/tmp/yet_another_target_dir"). ok @@ -565,7 +565,7 @@ ok ok 11> 11> file:list_dir("/tmp/yet_another_target_dir"). -{ok,["bin","Install","lib","misc","usr","erts-9.0", +{ok,["bin","Install","lib","misc","usr","erts-10.0", "releases"]} </pre> diff --git a/lib/reltool/src/reltool_target.erl b/lib/reltool/src/reltool_target.erl index 503e1971b9..0eeeca4a61 100644 --- a/lib/reltool/src/reltool_target.erl +++ b/lib/reltool/src/reltool_target.erl @@ -55,7 +55,7 @@ mandatory_modules() -> kernel_processes(KernelApp) -> [ {kernelProcess, heart, {heart, start, []}}, - {kernelProcess, error_logger , {error_logger, start_link, []}}, + {kernelProcess, logger , {logger_server, start_link, []}}, {kernelProcess, application_controller, {application_controller, start, [KernelApp]}} diff --git a/lib/runtime_tools/doc/src/Makefile b/lib/runtime_tools/doc/src/Makefile index a9b0056a93..11583406b7 100644 --- a/lib/runtime_tools/doc/src/Makefile +++ b/lib/runtime_tools/doc/src/Makefile @@ -60,8 +60,9 @@ BOOK_FILES = book.xml XML_FILES = \ $(BOOK_FILES) $(XML_CHAPTER_FILES) \ $(XML_PART_FILES) $(XML_REF3_FILES) \ - $(XML_REF6_FILES) $(XML_APPLICATION_FILES) \ - $(GENERATED_XML_FILES) + $(XML_REF6_FILES) $(XML_APPLICATION_FILES) + +XML_GEN_FILES = $(GENERATED_XML_FILES:%=$(XMLDIR)/%) GIF_FILES = @@ -97,7 +98,7 @@ SPECS_FLAGS = -I../../include -I../../../kernel/src # Targets # ---------------------------------------------------- -%.xml: $(ERL_TOP)/HOWTO/%.md $(ERL_TOP)/make/emd2exml +$(XMLDIR)/%.xml: $(ERL_TOP)/HOWTO/%.md $(ERL_TOP)/make/emd2exml $(ERL_TOP)/make/emd2exml $< $@ $(HTMLDIR)/%.gif: %.gif diff --git a/lib/runtime_tools/doc/src/dbg.xml b/lib/runtime_tools/doc/src/dbg.xml index 95f74d4607..276a41c415 100644 --- a/lib/runtime_tools/doc/src/dbg.xml +++ b/lib/runtime_tools/doc/src/dbg.xml @@ -815,7 +815,7 @@ Error: fun containing local erlang function calls ('is_atomm' called in guard)\ <v>HandlerSpec = {HandlerFun, InitialData}</v> <v>HandlerFun = fun() (two arguments)</v> <v>ModuleSpec = fun() (no arguments) | {TracerModule, TracerState}</v> - <v>ModuleModule = atom()</v> + <v>TracerModule = atom()</v> <v>InitialData = TracerState = term()</v> </type> <desc> diff --git a/lib/runtime_tools/src/appmon_info.erl b/lib/runtime_tools/src/appmon_info.erl index b5500085a3..9c587475ca 100644 --- a/lib/runtime_tools/src/appmon_info.erl +++ b/lib/runtime_tools/src/appmon_info.erl @@ -690,7 +690,7 @@ find_avoid() -> [P|Accu]; _ -> Accu end end, [undefined], - [application_controller, init, error_logger, gs, + [application_controller, init, gs, node_serv, appmon, appmon_a, appmon_info]). diff --git a/lib/runtime_tools/test/Makefile b/lib/runtime_tools/test/Makefile index de37b2570d..29cf7545c9 100644 --- a/lib/runtime_tools/test/Makefile +++ b/lib/runtime_tools/test/Makefile @@ -10,7 +10,8 @@ MODULES = \ dbg_SUITE \ erts_alloc_config_SUITE \ scheduler_SUITE \ - msacc_SUITE + msacc_SUITE \ + zzz_SUITE ERL_FILES= $(MODULES:%=%.erl) diff --git a/lib/runtime_tools/test/dbg_SUITE.erl b/lib/runtime_tools/test/dbg_SUITE.erl index cfe8412e33..c5dcccb887 100644 --- a/lib/runtime_tools/test/dbg_SUITE.erl +++ b/lib/runtime_tools/test/dbg_SUITE.erl @@ -478,8 +478,7 @@ port(Config) when is_list(Config) -> TraceFileDrv = list_to_atom(lists:flatten(["trace_file_drv n ",TestFile])), [{trace,Port,open,S,TraceFileDrv}, {trace,Port,getting_linked,S}, - {trace,Port,closed,normal}, - {trace,Port,unlink,S}] = flush() + {trace,Port,closed,normal}] = flush() after dbg:stop() end, diff --git a/lib/runtime_tools/test/zzz_SUITE.erl b/lib/runtime_tools/test/zzz_SUITE.erl new file mode 100644 index 0000000000..59c7fd7404 --- /dev/null +++ b/lib/runtime_tools/test/zzz_SUITE.erl @@ -0,0 +1,37 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2006-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(zzz_SUITE). + +%% The sole purpose of this test suite is for things we want to run last +%% before the VM terminates. + +-export([all/0]). + +-export([lc_graph/1]). + + +all() -> + [lc_graph]. + +lc_graph(_Config) -> + %% Create "lc_graph" file in current working dir + %% if lock checker is enabled. + erts_debug:lc_graph(), + ok. diff --git a/lib/sasl/doc/src/appup.xml b/lib/sasl/doc/src/appup.xml index a43a966dcb..d9d339884f 100644 --- a/lib/sasl/doc/src/appup.xml +++ b/lib/sasl/doc/src/appup.xml @@ -249,9 +249,17 @@ <pre> {restart_application, Application} Application = atom()</pre> - <p>Restarting an application means that the application is - stopped and then started again, similar to using the instructions - <c>remove_application</c> and <c>add_application</c> in sequence.</p> + <p>Restarting an application means that the application is stopped + and then started again, similar to using the instructions + <c>remove_application</c> and <c>add_application</c> in sequence. + Note that, even if the application has been started before the + release upgrade is performed, <c>restart_application</c> may only + <c>load</c> it rather than <c>start</c> it, depending on the + application's <c>start type</c>: + If <c>Type = load</c>, the application is only loaded. + If <c>Type = none</c>, the application is not loaded and not + started, although the code for its modules is loaded. + </p> </section> <section> diff --git a/lib/sasl/doc/src/notes.xml b/lib/sasl/doc/src/notes.xml index e532c3cd6f..791e9c063a 100644 --- a/lib/sasl/doc/src/notes.xml +++ b/lib/sasl/doc/src/notes.xml @@ -31,6 +31,26 @@ </header> <p>This document describes the changes made to the SASL application.</p> +<section><title>SASL 3.1.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + When upgrading with instruction 'restart_new_emulator', + the generated temporary boot file used 'kernelProcess' + statements from the old release instead of the new + release. This is now corrected.</p> + <p> + This correction is needed for upgrade to OTP-21.</p> + <p> + Own Id: OTP-15017</p> + </item> + </list> + </section> + +</section> + <section><title>SASL 3.1.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/sasl/doc/src/sasl_app.xml b/lib/sasl/doc/src/sasl_app.xml index e0693fcb60..48b0b8eafb 100644 --- a/lib/sasl/doc/src/sasl_app.xml +++ b/lib/sasl/doc/src/sasl_app.xml @@ -34,12 +34,9 @@ <p>The SASL application provides the following services:</p> <list type="bulleted"> <item><c>alarm_handler</c></item> - <item><c>rb</c></item> <item><c>release_handler</c></item> <item><c>systools</c></item> </list> - <p>The SASL application also includes <c>error_logger</c> event - handlers for formatting SASL error and crash reports.</p> <note> <p>The SASL application in OTP has nothing to do with "Simple Authentication and Security Layer" (RFC 4422).</p> @@ -47,51 +44,109 @@ </description> <section> - <title>Error Logger Event Handlers</title> - <p>The following error logger event handlers are used by - the SASL application.</p> + <title>Configuration</title> + <p>The following configuration parameters are defined for the SASL + application. For more information about configuration parameters, see + <seealso marker="kernel:app"><c>app(4)</c></seealso> in Kernel.</p> + <p>All configuration parameters are optional.</p> + <taglist> + <tag><c><![CDATA[start_prg = string() ]]></c></tag> + <item> + <p>Specifies the program to be used when restarting the system + during release installation. Default is + <c>$OTP_ROOT/bin/start</c>.</p> + </item> + <tag><c><![CDATA[masters = [atom()] ]]></c></tag> + <item> + <p>Specifies the nodes used by this node to read/write release + information. This parameter is ignored if parameter + <c>client_directory</c> is not set.</p> + </item> + <tag><c><![CDATA[client_directory = string() ]]></c></tag> + <item> + <p>This parameter specifies the client directory at the master + nodes. For details, see + <seealso marker="doc/design_principles:release_handling">Release Handling</seealso> + in <em>OTP Design Principles</em>. This parameter is + ignored if parameter <c>masters</c> is not set.</p> + </item> + <tag><c><![CDATA[static_emulator = true | false ]]></c></tag> + <item> + <p>Indicates if the Erlang emulator is statically installed. A + node with a static emulator cannot switch dynamically to a + new emulator, as the executable files are written into memory + statically. This parameter is ignored if parameters <c>masters</c> + and <c>client_directory</c> are not set.</p> + </item> + <tag><c><![CDATA[releases_dir = string() ]]></c></tag> + <item> + <p>Indicates where the <c>releases</c> directory is located. + The release handler writes all its files to this directory. + If this parameter is not set, the OS environment parameter + <c>RELDIR</c> is used. By default, this is + <c>$OTP_ROOT/releases</c>.</p> + </item> + <tag><c><![CDATA[utc_log = true | false ]]></c></tag> + <item> + <p>If set to <c>true</c>, all dates in textual log outputs are + displayed in Universal Coordinated Time with the string + <c>UTC</c> appended.</p> + </item> + </taglist> + </section> + + <section> + <title>Deprecated Error Logger Event Handlers and Configuration</title> + <p>In OTP-21, a new API for logging was added to Erlang/OTP. The + old <c>error_logger</c> event manager, and event handlers + running on this manager, will still work, but they are not used + by default.</p> + <p>The error logger event handlers <c>sasl_report_tty_h</c> + and <c>sasl_report_file_h</c>, were earliger used for printing + the so called SASL reports, i.e. <em>supervisor + reports</em>, <em>crash reports</em>, and <em>progress + reports</em>. These reports are now also printed by the standard + logger handler started by the Kernel application. Progress + reports are by default stopped by a filter, but can easily be + added by setting the Kernel configuration + parameter <seealso marker="kernel:kernel_app#logger_log_progress"><c>logger_log_progress=true</c></seealso>.</p> + <p>If the old error logger event handlers are still desired, they + must be added by + calling <c>error_logger:add_report_handler/1,2</c>.</p> <taglist> <tag><c>sasl_report_tty_h</c></tag> <item> <p>Formats and writes <em>supervisor reports</em>, <em>crash reports</em>, and <em>progress reports</em> to <c>stdio</c>. This error logger event handler uses - <seealso marker="kernel:kernel_app#error_logger_format_depth">error_logger_format_depth</seealso> - in the Kernel application to limit how much detail is - printed in crash and supervisor reports.</p> + <seealso marker="kernel:kernel_app#logger_format_depth"><c>logger_format_depth</c></seealso> + in the Kernel application to limit how much detail is printed + in crash and supervisor reports. If <c>logger_format_depth</c> + is not set, it uses the old <c>error_logger_format_depth</c> + instead.</p> </item> <tag><c>sasl_report_file_h</c></tag> <item> <p>Formats and writes <em>supervisor reports</em>, <em>crash report</em>, and <em>progress report</em> to a single file. This error logger event handler uses - <seealso marker="kernel:kernel_app#error_logger_format_depth">error_logger_format_depth</seealso> - in the Kernel application to limit the details - printed in crash and supervisor reports.</p> - </item> - <tag><c>log_mf_h</c></tag> - <item> - <p>This error logger writes <em>all</em> events sent to the - error logger to disk. Multiple files and log rotation are - used. For efficiency reasons, each event is written as a - binary. For more information about this handler, - see <seealso marker="stdlib:log_mf_h">the STDLIB Reference - Manual</seealso>.</p> - <p>To activate this event handler, three SASL - configuration parameters must be set, - <c>error_logger_mf_dir</c>, <c>error_logger_mf_maxbytes</c>, - and <c>error_logger_mf_maxfiles</c>. The next section provides - more information about the configuration parameters.</p> + <seealso marker="kernel:kernel_app#logger_format_depth"><c>logger_format_depth</c></seealso> + in the Kernel application to limit the details printed in + crash and supervisor reports. If <c>logger_format_depth</c> is + not set, it uses the old <c>error_logger_format_depth</c> + instead.</p> </item> </taglist> - </section> - - <section> - <title>Configuration</title> - <p>The following configuration parameters are defined for the SASL - application. For more information about configuration parameters, see - <seealso marker="kernel:app"><c>app(4)</c></seealso> in Kernel.</p> - <p>All configuration parameters are optional.</p> + <p>A similar behaviour, but still using the new logger API, can be + obtained by setting the Kernel application environment + variable <seealso marker="kernel:kernel_app#logger_sasl_compatible"><c>logger_sasl_compatible=true</c></seealso>. This will add a + second instance of the standard logger handler + named <c>sasl_h</c>, which will only print the SASL reports. No + SASL reports will then be printed by the Kernel logger + handler.</p> + <p>The <c>sasl_h</c> handler will be configured according to the + values of the following SASL application environment + variables.</p> <taglist> <tag><c><![CDATA[sasl_error_logger = Value ]]></c></tag> <item> @@ -124,6 +179,19 @@ <c>sasl_error_logger</c> to error reports or progress reports, or both. Default is <c>all</c>.</p> </item> + </taglist> + + <p>The error logger event handler <c>log_mf_h</c> can also still + be used. This event handler writes <em>all</em> events sent to + the error logger to disk. Multiple files and log rotation are + used. For efficiency reasons, each event is written as a + binary. For more information about this handler, + see <seealso marker="stdlib:log_mf_h">the STDLIB Reference + Manual</seealso>.</p> + <p>To activate this event handler, three SASL configuration + parameters must be + set:</p> + <taglist> <tag><c><![CDATA[error_logger_mf_dir = string() | false ]]></c></tag> <item> <p>Specifies in which directory <c>log_mf_h</c> is to store @@ -142,49 +210,12 @@ this parameter is undefined, the <c>log_mf_h</c> handler is not installed.</p> </item> - <tag><c><![CDATA[start_prg = string() ]]></c></tag> - <item> - <p>Specifies the program to be used when restarting the system - during release installation. Default is - <c>$OTP_ROOT/bin/start</c>.</p> - </item> - <tag><c><![CDATA[masters = [atom()] ]]></c></tag> - <item> - <p>Specifies the nodes used by this node to read/write release - information. This parameter is ignored if parameter - <c>client_directory</c> is not set.</p> - </item> - <tag><c><![CDATA[client_directory = string() ]]></c></tag> - <item> - <p>This parameter specifies the client directory at the master - nodes. For details, see - <seealso marker="doc/design_principles:release_handling">Release Handling</seealso> - in <em>OTP Design Principles</em>. This parameter is - ignored if parameter <c>masters</c> is not set.</p> - </item> - <tag><c><![CDATA[static_emulator = true | false ]]></c></tag> - <item> - <p>Indicates if the Erlang emulator is statically installed. A - node with a static emulator cannot switch dynamically to a - new emulator, as the executable files are written into memory - statically. This parameter is ignored if parameters <c>masters</c> - and <c>client_directory</c> are not set.</p> - </item> - <tag><c><![CDATA[releases_dir = string() ]]></c></tag> - <item> - <p>Indicates where the <c>releases</c> directory is located. - The release handler writes all its files to this directory. - If this parameter is not set, the OS environment parameter - <c>RELDIR</c> is used. By default, this is - <c>$OTP_ROOT/releases</c>.</p> - </item> - <tag><c><![CDATA[utc_log = true | false ]]></c></tag> - <item> - <p>If set to <c>true</c>, all dates in textual log outputs are - displayed in Universal Coordinated Time with the string - <c>UTC</c> appended.</p> - </item> </taglist> + <p>The new <seealso marker="kernel:logger_disk_log_h"> + <c>logger_disk_log_h</c></seealso> might be an alternative + to <c>log_mf_h</c> if log rotation is desired. This does, + however, write the log events in clear text and not as binaries.</p> + </section> <section> diff --git a/lib/sasl/src/release_handler.erl b/lib/sasl/src/release_handler.erl index bfa49fc05d..4fd3fc0d36 100644 --- a/lib/sasl/src/release_handler.erl +++ b/lib/sasl/src/release_handler.erl @@ -1052,8 +1052,8 @@ new_emulator_make_tmp_release(CurrentRelease,ToRelease,RelDir,Opts,Masters) -> ToVsn = ToRelease#release.vsn, TmpVsn = ?tmp_vsn(CurrentVsn), case get_base_libs(ToRelease#release.libs) of - {ok,{Kernel,Stdlib,Sasl}=BaseLibs,_} -> - case get_base_libs(ToRelease#release.libs) of + {ok,{Kernel,Stdlib,Sasl},_} -> + case get_base_libs(CurrentRelease#release.libs) of {ok,_,RestLibs} -> TmpErtsVsn = ToRelease#release.erts_vsn, TmpLibs = [Kernel,Stdlib,Sasl|RestLibs], @@ -1062,7 +1062,7 @@ new_emulator_make_tmp_release(CurrentRelease,ToRelease,RelDir,Opts,Masters) -> libs = TmpLibs, status = unpacked}, new_emulator_make_hybrid_boot(CurrentVsn,ToVsn,TmpVsn, - BaseLibs,RelDir,Opts,Masters), + RelDir,Opts,Masters), new_emulator_make_hybrid_config(CurrentVsn,ToVsn,TmpVsn, RelDir,Masters), {TmpVsn,TmpRelease}; @@ -1095,7 +1095,7 @@ get_base_libs([],_Kernel,_Stdlib,undefined,_Rest) -> get_base_libs([],Kernel,Stdlib,Sasl,Rest) -> {ok,{Kernel,Stdlib,Sasl},lists:reverse(Rest)}. -new_emulator_make_hybrid_boot(CurrentVsn,ToVsn,TmpVsn,BaseLibs,RelDir,Opts,Masters) -> +new_emulator_make_hybrid_boot(CurrentVsn,ToVsn,TmpVsn,RelDir,Opts,Masters) -> FromBootFile = filename:join([RelDir,CurrentVsn,"start.boot"]), ToBootFile = filename:join([RelDir,ToVsn,"start.boot"]), TmpBootFile = filename:join([RelDir,TmpVsn,"start.boot"]), @@ -1103,11 +1103,7 @@ new_emulator_make_hybrid_boot(CurrentVsn,ToVsn,TmpVsn,BaseLibs,RelDir,Opts,Maste Args = [ToVsn,Opts], {ok,FromBoot} = read_file(FromBootFile,Masters), {ok,ToBoot} = read_file(ToBootFile,Masters), - {{_,_,KernelPath},{_,_,StdlibPath},{_,_,SaslPath}} = BaseLibs, - Paths = {filename:join(KernelPath,"ebin"), - filename:join(StdlibPath,"ebin"), - filename:join(SaslPath,"ebin")}, - case systools_make:make_hybrid_boot(TmpVsn,FromBoot,ToBoot,Paths,Args) of + case systools_make:make_hybrid_boot(TmpVsn,FromBoot,ToBoot,Args) of {ok,TmpBoot} -> write_file(TmpBootFile,TmpBoot,Masters); {error,Reason} -> diff --git a/lib/sasl/src/sasl.app.src b/lib/sasl/src/sasl.app.src index 1e8e58a978..60d08ffa54 100644 --- a/lib/sasl/src/sasl.app.src +++ b/lib/sasl/src/sasl.app.src @@ -40,8 +40,7 @@ ]}, {registered, [sasl_sup, alarm_handler, release_handler]}, {applications, [kernel, stdlib]}, - {env, [{sasl_error_logger, tty}, - {errlog_type, all}]}, + {env, []}, {mod, {sasl, []}}, {runtime_dependencies, ["tools-2.6.14","stdlib-3.4","kernel-5.3", "erts-9.0"]}]}. diff --git a/lib/sasl/src/sasl.appup.src b/lib/sasl/src/sasl.appup.src index 94af164b20..221427874c 100644 --- a/lib/sasl/src/sasl.appup.src +++ b/lib/sasl/src/sasl.appup.src @@ -18,7 +18,11 @@ %% %CopyrightEnd% {"%VSN%", %% Up from - max one major revision back - [{<<"3\\.0\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-20.* + [{<<"3\\.0\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-20.0 + {<<"3\\.1(\\.[0-2]+)*">>,[restart_new_emulator]}, % OTP-20.1+ + {<<"3\\.1(\\.[3-9]+)*">>,[restart_new_emulator]}], % OTP-21 %% Down to - max one major revision back - [{<<"3\\.0\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-20.* + [{<<"3\\.0\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-20.* + {<<"3\\.1(\\.[0-2]+)*">>,[restart_new_emulator]}, % OTP-20.1+ + {<<"3\\.1(\\.[3-9]+)*">>,[restart_new_emulator]}] % OTP-21 }. diff --git a/lib/sasl/src/sasl.erl b/lib/sasl/src/sasl.erl index 24afaee183..657eb6688a 100644 --- a/lib/sasl/src/sasl.erl +++ b/lib/sasl/src/sasl.erl @@ -31,45 +31,52 @@ %%%----------------------------------------------------------------- -behaviour(application). --record(state, {sasl_error_logger, error_logger_mf}). +-record(state, {sasl_logger, error_logger_mf}). start(_, []) -> - Handler = get_sasl_error_logger(), - Type = get_sasl_error_logger_type(), + {Dest,Level} = get_logger_info(), Mf = get_error_logger_mf(), - add_sasl_error_logger(Handler, Type), + add_sasl_logger(Dest, Level), add_error_logger_mf(Mf), - State = #state{sasl_error_logger = Handler, error_logger_mf = Mf}, + State = #state{sasl_logger = Dest, error_logger_mf = Mf}, case supervisor:start_link({local, sasl_sup}, sasl, []) of {ok, Pid} -> {ok, Pid, State}; Error -> Error end. stop(State) -> - delete_sasl_error_logger(State#state.sasl_error_logger), + delete_sasl_logger(State#state.sasl_logger), delete_error_logger_mf(State#state.error_logger_mf). %%----------------------------------------------------------------- %% Internal functions %%----------------------------------------------------------------- -get_sasl_error_logger() -> +get_logger_info() -> + case application:get_env(kernel, logger_sasl_compatible) of + {ok,true} -> + {get_logger_dest(),get_logger_level()}; + _ -> + {std,undefined} + end. + +get_logger_dest() -> case application:get_env(sasl, sasl_error_logger) of - {ok, false} -> undefined; - {ok, tty} -> tty; - {ok, {file, File}} when is_list(File) -> {file, File, [write]}; - {ok, {file, File, Modes}} when is_list(File), is_list(Modes) -> - {file, File, Modes}; - {ok, Bad} -> exit({bad_config, {sasl, {sasl_error_logger, Bad}}}); - _ -> undefined + {ok, false} -> undefined; + {ok, tty} -> standard_io; + {ok, {file, File}} when is_list(File) -> {file, File}; + {ok, {file, File, Modes}} when is_list(File), is_list(Modes) -> + {file, File, Modes}; + {ok, Bad} -> exit({bad_config, {sasl, {sasl_logger_dest, Bad}}}); + undefined -> standard_io end. -get_sasl_error_logger_type() -> +get_logger_level() -> case application:get_env(sasl, errlog_type) of - {ok, error} -> error; - {ok, progress} -> progress; - {ok, all} -> all; - {ok, Bad} -> exit({bad_config, {sasl, {errlog_type, Bad}}}); - _ -> all + {ok, error} -> error; + {ok, progress} -> info; + {ok, all} -> info; + {ok, Bad} -> exit({bad_config, {sasl, {errlog_type, Bad}}}); + _ -> info end. get_error_logger_mf() -> @@ -119,23 +126,32 @@ get_mf_maxf() -> {ok, Bad} -> exit({bad_config, {sasl, {error_logger_mf_maxfiles, Bad}}}) end. -add_sasl_error_logger(undefined, _Type) -> ok; -add_sasl_error_logger(Handler, Type) -> - error_logger:add_report_handler(mod(Handler), args(Handler, Type)). - -delete_sasl_error_logger(undefined) -> ok; -delete_sasl_error_logger(Type) -> - error_logger:delete_report_handler(mod(Type)). - -mod(tty) -> sasl_report_tty_h; -mod({file, _File, _Modes}) -> sasl_report_file_h. - -args({file, File, Modes}, Type) -> {File, Modes, type(Type)}; -args(_, Type) -> type(Type). - -type(error) -> error; -type(progress) -> progress; -type(_) -> all. +add_sasl_logger(undefined, _Level) -> ok; +add_sasl_logger(std, undefined) -> ok; +add_sasl_logger(Dest, Level) -> + FC0 = #{legacy_header=>true, + template=>[{logger_formatter,header},"\n",msg,"\n"]}, + FC = case application:get_env(sasl,utc_log) of + {ok,Bool} when is_boolean(Bool) -> + FC0#{utc=>Bool}; + _ -> + FC0 + end, + ok = logger:add_handler(sasl_h,logger_std_h, + #{level=>Level, + filter_default=>stop, + filters=> + [{sasl_domain, + {fun logger_filters:domain/2, + {log,equals,[beam,erlang,otp,sasl]}}}], + logger_std_h=>#{type=>Dest}, + formatter=>{logger_formatter,FC}}). + +delete_sasl_logger(undefined) -> ok; +delete_sasl_logger(std) -> ok; +delete_sasl_logger(_Type) -> + _ = logger:remove_handler(sasl_h), + ok. add_error_logger_mf(undefined) -> ok; add_error_logger_mf({Dir, MaxB, MaxF}) -> diff --git a/lib/sasl/src/systools_make.erl b/lib/sasl/src/systools_make.erl index fa3182cc08..f4b1b54fd1 100644 --- a/lib/sasl/src/systools_make.erl +++ b/lib/sasl/src/systools_make.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2017. All Rights Reserved. +%% Copyright Ericsson AB 1996-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -32,7 +32,7 @@ -export([read_application/4]). --export([make_hybrid_boot/5]). +-export([make_hybrid_boot/4]). -import(lists, [filter/2, keysort/2, keysearch/3, map/2, reverse/1, append/1, foldl/3, member/2, foreach/2]). @@ -178,94 +178,153 @@ return({error,Mod,Error},_,Flags) -> %% and sasl. %% %% TmpVsn = string(), -%% Paths = {KernelPath,StdlibPath,SaslPath} %% Returns {ok,Boot} | {error,Reason} %% Boot1 = Boot2 = Boot = binary() %% Reason = {app_not_found,App} | {app_not_replaced,App} -%% App = kernel | stdlib | sasl -make_hybrid_boot(TmpVsn, Boot1, Boot2, Paths, Args) -> - catch do_make_hybrid_boot(TmpVsn, Boot1, Boot2, Paths, Args). -do_make_hybrid_boot(TmpVsn, Boot1, Boot2, Paths, Args) -> - {script,{_RelName1,_RelVsn1},Script1} = binary_to_term(Boot1), - {script,{RelName2,_RelVsn2},Script2} = binary_to_term(Boot2), - MatchPaths = get_regexp_path(Paths), - NewScript1 = replace_paths(Script1,MatchPaths), - {Kernel,Stdlib,Sasl} = get_apps(Script2,undefined,undefined,undefined), - NewScript2 = replace_apps(NewScript1,Kernel,Stdlib,Sasl), - NewScript3 = add_apply_upgrade(NewScript2,Args), - Boot = term_to_binary({script,{RelName2,TmpVsn},NewScript3}), +%% App = stdlib | sasl +make_hybrid_boot(TmpVsn, Boot1, Boot2, Args) -> + catch do_make_hybrid_boot(TmpVsn, Boot1, Boot2, Args). +do_make_hybrid_boot(TmpVsn, OldBoot, NewBoot, Args) -> + {script,{_RelName1,_RelVsn1},OldScript} = binary_to_term(OldBoot), + {script,{NewRelName,_RelVsn2},NewScript} = binary_to_term(NewBoot), + + %% Everyting upto kernel_load_completed must come from the new script + Fun1 = fun({progress,kernel_load_completed}) -> false; + (_) -> true + end, + {_OldKernelLoad,OldRest1} = lists:splitwith(Fun1,OldScript), + {NewKernelLoad,NewRest1} = lists:splitwith(Fun1,NewScript), + + Fun2 = fun({progress,modules_loaded}) -> false; + (_) -> true + end, + {OldModLoad,OldRest2} = lists:splitwith(Fun2,OldRest1), + {NewModLoad,NewRest2} = lists:splitwith(Fun2,NewRest1), + + Fun3 = fun({kernelProcess,_,_}) -> false; + (_) -> true + end, + {OldPaths,OldRest3} = lists:splitwith(Fun3,OldRest2), + {NewPaths,NewRest3} = lists:splitwith(Fun3,NewRest2), + + Fun4 = fun({progress,init_kernel_started}) -> false; + (_) -> true + end, + {_OldKernelProcs,OldApps} = lists:splitwith(Fun4,OldRest3), + {NewKernelProcs,NewApps} = lists:splitwith(Fun4,NewRest3), + + %% Then comes all module load, which for each app consist of: + %% {path,[AppPath]}, + %% {primLoad,ModuleList} + %% Replace kernel, stdlib and sasl here + MatchPaths = get_regexp_path(), + ModLoad = replace_module_load(OldModLoad,NewModLoad,MatchPaths), + Paths = replace_paths(OldPaths,NewPaths,MatchPaths), + + {Stdlib,Sasl} = get_apps(NewApps,undefined,undefined), + Apps0 = replace_apps(OldApps,Stdlib,Sasl), + Apps = add_apply_upgrade(Apps0,Args), + + Script = NewKernelLoad++ModLoad++Paths++NewKernelProcs++Apps, + Boot = term_to_binary({script,{NewRelName,TmpVsn},Script}), {ok,Boot}. %% For each app, compile a regexp that can be used for finding its path -get_regexp_path({KernelPath,StdlibPath,SaslPath}) -> +get_regexp_path() -> {ok,KernelMP} = re:compile("kernel-[0-9\.]+",[unicode]), {ok,StdlibMP} = re:compile("stdlib-[0-9\.]+",[unicode]), {ok,SaslMP} = re:compile("sasl-[0-9\.]+",[unicode]), - [{KernelMP,KernelPath},{StdlibMP,StdlibPath},{SaslMP,SaslPath}]. - -%% For each path in the script, check if it matches any of the MPs -%% found above, and if so replace it with the correct new path. -replace_paths([{path,Path}|Script],MatchPaths) -> - [{path,replace_path(Path,MatchPaths)}|replace_paths(Script,MatchPaths)]; -replace_paths([Stuff|Script],MatchPaths) -> - [Stuff|replace_paths(Script,MatchPaths)]; -replace_paths([],_) -> + [KernelMP,StdlibMP,SaslMP]. + +replace_module_load(Old,New,[MP|MatchPaths]) -> + replace_module_load(do_replace_module_load(Old,New,MP),New,MatchPaths); +replace_module_load(Script,_,[]) -> + Script. + +do_replace_module_load([{path,[OldAppPath]},{primLoad,OldMods}|OldRest],New,MP) -> + case re:run(OldAppPath,MP,[{capture,none}]) of + nomatch -> + [{path,[OldAppPath]},{primLoad,OldMods}| + do_replace_module_load(OldRest,New,MP)]; + match -> + get_module_load(New,MP) ++ OldRest + end; +do_replace_module_load([Other|Rest],New,MP) -> + [Other|do_replace_module_load(Rest,New,MP)]; +do_replace_module_load([],_,_) -> + []. + +get_module_load([{path,[AppPath]},{primLoad,Mods}|Rest],MP) -> + case re:run(AppPath,MP,[{capture,none}]) of + nomatch -> + get_module_load(Rest,MP); + match -> + [{path,[AppPath]},{primLoad,Mods}] + end; +get_module_load([_|Rest],MP) -> + get_module_load(Rest,MP); +get_module_load([],_) -> []. -replace_path([Path|Paths],MatchPaths) -> - [do_replace_path(Path,MatchPaths)|replace_path(Paths,MatchPaths)]; -replace_path([],_) -> +replace_paths([{path,OldPaths}|Old],New,MatchPaths) -> + {path,NewPath} = lists:keyfind(path,1,New), + [{path,do_replace_paths(OldPaths,NewPath,MatchPaths)}|Old]; +replace_paths([Other|Old],New,MatchPaths) -> + [Other|replace_paths(Old,New,MatchPaths)]. + +do_replace_paths(Old,New,[MP|MatchPaths]) -> + do_replace_paths(do_replace_paths1(Old,New,MP),New,MatchPaths); +do_replace_paths(Paths,_,[]) -> + Paths. + +do_replace_paths1([P|Ps],New,MP) -> + case re:run(P,MP,[{capture,none}]) of + nomatch -> + [P|do_replace_paths1(Ps,New,MP)]; + match -> + get_path(New,MP) ++ Ps + end; +do_replace_paths1([],_,_) -> []. -do_replace_path(Path,[{MP,ReplacePath}|MatchPaths]) -> - case re:run(Path,MP,[{capture,none}]) of - nomatch -> do_replace_path(Path,MatchPaths); - match -> ReplacePath +get_path([P|Ps],MP) -> + case re:run(P,MP,[{capture,none}]) of + nomatch -> + get_path(Ps,MP); + match -> + [P] end; -do_replace_path(Path,[]) -> - Path. - -%% Return the entries for loading the three base applications -get_apps([{kernelProcess,application_controller, - {application_controller,start,[{application,kernel,_}]}}=Kernel| - Script],_,Stdlib,Sasl) -> - get_apps(Script,Kernel,Stdlib,Sasl); +get_path([],_) -> + []. + + +%% Return the entries for loading stdlib and sasl get_apps([{apply,{application,load,[{application,stdlib,_}]}}=Stdlib|Script], - Kernel,_,Sasl) -> - get_apps(Script,Kernel,Stdlib,Sasl); + _,Sasl) -> + get_apps(Script,Stdlib,Sasl); get_apps([{apply,{application,load,[{application,sasl,_}]}}=Sasl|_Script], - Kernel,Stdlib,_) -> - {Kernel,Stdlib,Sasl}; -get_apps([_|Script],Kernel,Stdlib,Sasl) -> - get_apps(Script,Kernel,Stdlib,Sasl); -get_apps([],undefined,_,_) -> - throw({error,{app_not_found,kernel}}); -get_apps([],_,undefined,_) -> + Stdlib,_) -> + {Stdlib,Sasl}; +get_apps([_|Script],Stdlib,Sasl) -> + get_apps(Script,Stdlib,Sasl); +get_apps([],undefined,_) -> throw({error,{app_not_found,stdlib}}); -get_apps([],_,_,undefined) -> +get_apps([],_,undefined) -> throw({error,{app_not_found,sasl}}). - -%% Replace the entries for loading the base applications -replace_apps([{kernelProcess,application_controller, - {application_controller,start,[{application,kernel,_}]}}| - Script],Kernel,Stdlib,Sasl) -> - [Kernel|replace_apps(Script,undefined,Stdlib,Sasl)]; +%% Replace the entries for loading the stdlib and sasl replace_apps([{apply,{application,load,[{application,stdlib,_}]}}|Script], - Kernel,Stdlib,Sasl) -> - [Stdlib|replace_apps(Script,Kernel,undefined,Sasl)]; + Stdlib,Sasl) -> + [Stdlib|replace_apps(Script,undefined,Sasl)]; replace_apps([{apply,{application,load,[{application,sasl,_}]}}|Script], - _Kernel,_Stdlib,Sasl) -> + _Stdlib,Sasl) -> [Sasl|Script]; -replace_apps([Stuff|Script],Kernel,Stdlib,Sasl) -> - [Stuff|replace_apps(Script,Kernel,Stdlib,Sasl)]; -replace_apps([],undefined,undefined,_) -> +replace_apps([Stuff|Script],Stdlib,Sasl) -> + [Stuff|replace_apps(Script,Stdlib,Sasl)]; +replace_apps([],undefined,_) -> throw({error,{app_not_replaced,sasl}}); -replace_apps([],undefined,_,_) -> - throw({error,{app_not_replaced,stdlib}}); -replace_apps([],_,_,_) -> - throw({error,{app_not_replaced,kernel}}). - +replace_apps([],_,_) -> + throw({error,{app_not_replaced,stdlib}}). %% Finally add an apply of release_handler:new_emulator_upgrade - which will %% complete the execution of the upgrade script (relup). @@ -275,8 +334,6 @@ add_apply_upgrade(Script,Args) -> {apply,{release_handler,new_emulator_upgrade,Args}} | RevScript]). - - %%----------------------------------------------------------------- %% Create a release package from a release file. %% Options is a list of {path, Path} | silent | @@ -1489,6 +1546,12 @@ mandatory_modules() -> gen_server, heart, kernel, + logger, + logger_filters, + logger_server, + logger_backend, + logger_config, + logger_simple, lists, proc_lib, supervisor @@ -1500,7 +1563,7 @@ mandatory_modules() -> preloaded() -> %% Sorted [erl_prim_loader,erl_tracer,erlang, - erts_code_purger,erts_dirty_process_code_checker, + erts_code_purger,erts_dirty_process_signal_handler, erts_internal,erts_literal_area_collector, init,otp_ring0,prim_buffer,prim_eval,prim_file, prim_inet,prim_zip,zlib]. @@ -1513,7 +1576,7 @@ preloaded() -> kernel_processes() -> [{heart, heart, start, []}, - {error_logger, error_logger, start_link, []}, + {logger, logger_server, start_link, []}, {application_controller, application_controller, start, fun(Appls) -> [{_,App}] = filter(fun({{kernel,_},_App}) -> true; diff --git a/lib/sasl/test/sasl_SUITE.erl b/lib/sasl/test/sasl_SUITE.erl index f12bde9b3d..7b63684c53 100644 --- a/lib/sasl/test/sasl_SUITE.erl +++ b/lib/sasl/test/sasl_SUITE.erl @@ -21,6 +21,7 @@ -include_lib("common_test/include/ct.hrl"). %% Test server specific exports +-export([init_per_suite/1,end_per_suite/1]). -export([all/0,groups/0,init_per_group/2,end_per_group/2]). -export([init_per_testcase/2, end_per_testcase/2]). @@ -37,6 +38,19 @@ all() -> groups() -> []. +init_per_suite(Config) -> + S = application:get_env(kernel,logger_sasl_compatible), + application:set_env(kernel,logger_sasl_compatible,true), + [{sasl_compatible,S}|Config]. + +end_per_suite(Config) -> + case ?config(sasl_compatible,Config) of + {ok,X} -> + application:set_env(kernel,logger_sasl_compatible,X); + undefined -> + application:unset_env(kernel,logger_sasl_compatible) + end. + init_per_group(_GroupName, Config) -> Config. diff --git a/lib/sasl/test/sasl_report_SUITE.erl b/lib/sasl/test/sasl_report_SUITE.erl index 92df5e6e40..96975aaf69 100644 --- a/lib/sasl/test/sasl_report_SUITE.erl +++ b/lib/sasl/test/sasl_report_SUITE.erl @@ -53,13 +53,17 @@ gen_server_crash_unicode(Config) -> gen_server_crash(Config, unicode). gen_server_crash(Config, Encoding) -> + StopFilter = {fun(_,_) -> stop end, ok}, + logger:add_handler_filter(logger_std_h,stop_all,StopFilter), + logger:add_handler_filter(cth_log_redirect,stop_all,StopFilter), try do_gen_server_crash(Config, Encoding) after - error_logger:tty(true), + ok = application:unset_env(kernel, logger_sasl_compatible), ok = application:unset_env(sasl, sasl_error_logger), ok = application:unset_env(kernel, error_logger_format_depth), - error_logger:add_report_handler(cth_log_redirect) + logger:remove_handler_filter(logger_std_h,stop_all), + logger:remove_handler_filter(cth_log_redirect,stop_all) end, ok. @@ -70,26 +74,26 @@ do_gen_server_crash(Config, Encoding) -> SaslLog = filename:join(LogDir, "sasl.log"), ok = filelib:ensure_dir(SaslLog), - error_logger:delete_report_handler(cth_log_redirect), - error_logger:tty(false), application:stop(sasl), Modes = [write, {encoding, Encoding}], + ok = application:set_env(kernel, logger_sasl_compatible, true), ok = application:set_env(sasl, sasl_error_logger, {file,SaslLog,Modes}, [{persistent,true}]), application:set_env(kernel, error_logger_format_depth, 30), error_logger:logfile({open,KernelLog}), application:start(sasl), - io:format("~p\n", [gen_event:which_handlers(error_logger)]), + logger:i(print), crash_me(), error_logger:logfile(close), + application:stop(sasl), check_file(KernelLog, utf8, 70000, 150000), check_file(SaslLog, Encoding, 70000, 150000), - %% ok = file:delete(KernelLog), - %% ok = file:delete(SaslLog), + ok = file:delete(KernelLog), + ok = file:delete(SaslLog), ok. check_file(File, Encoding, Min, Max) -> diff --git a/lib/sasl/test/systools_SUITE.erl b/lib/sasl/test/systools_SUITE.erl index c8b2f31120..ad61186921 100644 --- a/lib/sasl/test/systools_SUITE.erl +++ b/lib/sasl/test/systools_SUITE.erl @@ -1795,27 +1795,28 @@ normal_hybrid(Config) -> ok = file:set_cwd(OldDir), - BasePaths = {"testkernelpath","teststdlibpath","testsaslpath"}, {ok,Hybrid} = systools_make:make_hybrid_boot("tmp_vsn",Boot1,Boot2, - BasePaths, [dummy,args]), + [dummy,args]), {script,{"Test release","tmp_vsn"},Script} = binary_to_term(Hybrid), ct:log("~p.~n",[Script]), %% Check that all paths to base apps are replaced by paths from BaseLib Boot1Str = io_lib:format("~p~n",[binary_to_term(Boot1)]), + Boot2Str = io_lib:format("~p~n",[binary_to_term(Boot2)]), HybridStr = io_lib:format("~p~n",[binary_to_term(Hybrid)]), ReOpts = [global,{capture,first,list},unicode], {match,OldKernelMatch} = re:run(Boot1Str,"kernel-[0-9\.]+",ReOpts), {match,OldStdlibMatch} = re:run(Boot1Str,"stdlib-[0-9\.]+",ReOpts), {match,OldSaslMatch} = re:run(Boot1Str,"sasl-[0-9\.]+",ReOpts), - nomatch = re:run(HybridStr,"kernel-[0-9\.]+",ReOpts), - nomatch = re:run(HybridStr,"stdlib-[0-9\.]+",ReOpts), - nomatch = re:run(HybridStr,"sasl-[0-9\.]+",ReOpts), - {match,NewKernelMatch} = re:run(HybridStr,"testkernelpath",ReOpts), - {match,NewStdlibMatch} = re:run(HybridStr,"teststdlibpath",ReOpts), - {match,NewSaslMatch} = re:run(HybridStr,"testsaslpath",ReOpts), + {match,NewKernelMatch} = re:run(Boot2Str,"kernel-[0-9\.]+",ReOpts), + {match,NewStdlibMatch} = re:run(Boot2Str,"stdlib-[0-9\.]+",ReOpts), + {match,NewSaslMatch} = re:run(Boot2Str,"sasl-[0-9\.]+",ReOpts), + + {match,NewKernelMatch} = re:run(HybridStr,"kernel-[0-9\.]+",ReOpts), + {match,NewStdlibMatch} = re:run(HybridStr,"stdlib-[0-9\.]+",ReOpts), + {match,NewSaslMatch} = re:run(HybridStr,"sasl-[0-9\.]+",ReOpts), NewKernelN = length(NewKernelMatch), NewKernelN = length(OldKernelMatch), @@ -1824,6 +1825,11 @@ normal_hybrid(Config) -> NewSaslN = length(NewSaslMatch), NewSaslN = length(OldSaslMatch), + %% Check that kernelProcesses are taken from new boot script + {script,_,Script2} = binary_to_term(Boot2), + NewKernelProcs = [KP || KP={kernelProcess,_,_} <- Script2], + NewKernelProcs = [KP || KP={kernelProcess,_,_} <- Script], + %% Check that application load instruction has correct versions Apps = application:loaded_applications(), {_,_,KernelVsn} = lists:keyfind(kernel,1,Apps), @@ -1894,10 +1900,8 @@ hybrid_no_old_sasl(Config) -> {ok,Boot1} = file:read_file(Name1 ++ ".boot"), {ok,Boot2} = file:read_file(Name2 ++ ".boot"), - BasePaths = {"testkernelpath","teststdlibpath","testsaslpath"}, {error,{app_not_replaced,sasl}} = - systools_make:make_hybrid_boot("tmp_vsn",Boot1,Boot2, - BasePaths,[dummy,args]), + systools_make:make_hybrid_boot("tmp_vsn",Boot1,Boot2,[dummy,args]), ok = file:set_cwd(OldDir), ok. @@ -1927,10 +1931,8 @@ hybrid_no_new_sasl(Config) -> {ok,Boot1} = file:read_file(Name1 ++ ".boot"), {ok,Boot2} = file:read_file(Name2 ++ ".boot"), - BasePaths = {"testkernelpath","teststdlibpath","testsaslpath"}, {error,{app_not_found,sasl}} = - systools_make:make_hybrid_boot("tmp_vsn",Boot1,Boot2, - BasePaths,[dummy,args]), + systools_make:make_hybrid_boot("tmp_vsn",Boot1,Boot2,[dummy,args]), ok = file:set_cwd(OldDir), ok. diff --git a/lib/sasl/vsn.mk b/lib/sasl/vsn.mk index 2488197ec5..52b168598a 100644 --- a/lib/sasl/vsn.mk +++ b/lib/sasl/vsn.mk @@ -1 +1 @@ -SASL_VSN = 3.1.1 +SASL_VSN = 3.1.2 diff --git a/lib/ssh/doc/specs/.gitignore b/lib/ssh/doc/specs/.gitignore new file mode 100644 index 0000000000..322eebcb06 --- /dev/null +++ b/lib/ssh/doc/specs/.gitignore @@ -0,0 +1 @@ +specs_*.xml diff --git a/lib/ssh/doc/src/Makefile b/lib/ssh/doc/src/Makefile index f54f5e0708..7c4dbd7af8 100644 --- a/lib/ssh/doc/src/Makefile +++ b/lib/ssh/doc/src/Makefile @@ -38,19 +38,22 @@ RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN) # Target Specs # ---------------------------------------------------- XML_APPLICATION_FILES = ref_man.xml -XML_REF3_FILES = ssh.xml \ - ssh_channel.xml \ - ssh_connection.xml \ +XML_REF3_FILES = \ + ssh.xml \ + ssh_client_channel.xml \ ssh_client_key_api.xml \ + ssh_connection.xml \ + ssh_server_channel.xml \ ssh_server_key_api.xml \ ssh_sftp.xml \ ssh_sftpd.xml \ XML_REF6_FILES = ssh_app.xml -XML_PART_FILES = \ - usersguide.xml -XML_CHAPTER_FILES = notes.xml \ +XML_PART_FILES = usersguide.xml + +XML_CHAPTER_FILES = \ + notes.xml \ introduction.xml \ using_ssh.xml \ configure_algos.xml @@ -84,12 +87,19 @@ HTML_REF_MAN_FILE = $(HTMLDIR)/index.html TOP_PDF_FILE = $(PDFDIR)/$(APPLICATION)-$(VSN).pdf +SPECS_FILES = $(XML_REF3_FILES:%.xml=$(SPECDIR)/specs_%.xml) + +TOP_SPECS_FILE = specs.xml + # ---------------------------------------------------- # FLAGS # ---------------------------------------------------- XML_FLAGS += DVIPS_FLAGS += +#SPECS_FLAGS = -I../../include -I../../../kernel/include +SPECS_FLAGS = -I../../../public_key/include -I../../../public_key/src -I../../.. + # ---------------------------------------------------- # Targets # ---------------------------------------------------- @@ -111,6 +121,7 @@ clean clean_docs: rm -rf $(HTMLDIR)/* rm -f $(MAN3DIR)/* rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) + rm -f $(SPECS_FILES) rm -f errs core *~ man: $(MAN3_FILES) $(MAN6_FILES) diff --git a/lib/ssh/doc/src/configure_algos.xml b/lib/ssh/doc/src/configure_algos.xml index dd60324851..15aece8968 100644 --- a/lib/ssh/doc/src/configure_algos.xml +++ b/lib/ssh/doc/src/configure_algos.xml @@ -117,6 +117,7 @@ <p>Due to this, it impossible to list in documentation what algorithms that are available in a certain installation.</p> <p>There is an important command to list the actual algorithms and their ordering: <seealso marker="ssh#default_algorithms-0">ssh:default_algorithms/0</seealso>.</p> + <marker id="example_default_algorithms"/> <code type="erl"> 0> ssh:default_algorithms(). [{kex,['ecdh-sha2-nistp384','ecdh-sha2-nistp521', @@ -156,7 +157,7 @@ <section> <title>Replacing the default set: preferred_algorithms</title> - <p>See the <seealso marker="ssh#option_preferred_algorithms">Reference Manual</seealso> for details</p> + <p>See the <seealso marker="ssh#type-preferred_algorithms_common_option">Reference Manual</seealso> for details</p> <p>Here follows a series of examples ranging from simple to more complex.</p> @@ -301,7 +302,7 @@ First one has to list them with <c>ssh:default_algorithms()</c> and then do changes in the lists.</p> <p>To facilitate addition or removal of algorithms the option <c>modify_algorithms</c> is available. - See the <seealso marker="ssh#option_modify_algorithms">Reference Manual</seealso> for details.</p> + See the <seealso marker="ssh#type-modify_algorithms_common_option">Reference Manual</seealso> for details.</p> <p>The option takes a list with instructions to append, prepend or remove algorithms:</p> <code type="erl"> diff --git a/lib/ssh/doc/src/introduction.xml b/lib/ssh/doc/src/introduction.xml index b7a73e2597..6fd8425adf 100644 --- a/lib/ssh/doc/src/introduction.xml +++ b/lib/ssh/doc/src/introduction.xml @@ -145,7 +145,7 @@ data that can be sent to the channel peer without adjusting the window. Typically, an SSH client opens a channel, sends data (commands), receives data (control information), and then closes the channel. - The <seealso marker="ssh_channel">ssh_channel</seealso> behaviour + The <seealso marker="ssh_client_channel">ssh_client_channel</seealso> behaviour handles generic parts of SSH channel management. This makes it easy to write your own SSH client/server processes that use flow-control and thus opens for more focus on the application logic. diff --git a/lib/ssh/doc/src/notes.xml b/lib/ssh/doc/src/notes.xml index ba563335a2..d78309167a 100644 --- a/lib/ssh/doc/src/notes.xml +++ b/lib/ssh/doc/src/notes.xml @@ -30,6 +30,43 @@ <file>notes.xml</file> </header> +<section><title>Ssh 4.6.8</title> + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + An ssh_sftp server (running version 6) could fail if it + is told to remove a file which in fact is a directory.</p> + <p> + Own Id: OTP-15004</p> + </item> + <item> + <p> + Fix rare spurios shutdowns of ssh servers when receiveing + <c>{'EXIT',_,normal}</c> messages.</p> + <p> + Own Id: OTP-15018</p> + </item> + </list> + </section> + +</section> + +<section><title>Ssh 4.6.7</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix bad spec in ssh.hrl: <c>double_algs()</c>.</p> + <p> + Own Id: OTP-14990</p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.6.6</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -44,8 +81,6 @@ </item> </list> </section> - - <section><title>Improvements and New Features</title> <list> <item> @@ -453,6 +488,20 @@ </section> +<section><title>Ssh 4.4.2.3</title> + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + An ssh_sftp server (running version 6) could fail if it + is told to remove a file which in fact is a directory.</p> + <p> + Own Id: OTP-15004</p> + </item> + </list> + </section> +</section> + <section><title>Ssh 4.4.2.2</title> <section><title>Improvements and New Features</title> <list> @@ -856,6 +905,21 @@ </section> +<section><title>Ssh 4.2.2.6</title> + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix rare spurios shutdowns of ssh servers when receiveing + <c>{'EXIT',_,normal}</c> messages.</p> + <p> + Own Id: OTP-15018</p> + </item> + </list> + </section> +</section> + + <section><title>Ssh 4.2.2.5</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/ssh/doc/src/ref_man.xml b/lib/ssh/doc/src/ref_man.xml index 140ebd8c76..76e6520f94 100644 --- a/lib/ssh/doc/src/ref_man.xml +++ b/lib/ssh/doc/src/ref_man.xml @@ -35,7 +35,8 @@ </description> <xi:include href="ssh_app.xml"/> <xi:include href="ssh.xml"/> - <xi:include href="ssh_channel.xml"/> + <xi:include href="ssh_client_channel.xml"/> + <xi:include href="ssh_server_channel.xml"/> <xi:include href="ssh_connection.xml"/> <xi:include href="ssh_client_key_api.xml"/> <xi:include href="ssh_server_key_api.xml"/> diff --git a/lib/ssh/doc/src/specs.xml b/lib/ssh/doc/src/specs.xml new file mode 100644 index 0000000000..acdbe2ddfd --- /dev/null +++ b/lib/ssh/doc/src/specs.xml @@ -0,0 +1,13 @@ +<?xml version="1.0" encoding="utf-8" ?> +<specs xmlns:xi="http://www.w3.org/2001/XInclude"> + <xi:include href="../specs/specs_ssh.xml"/> + <xi:include href="../specs/specs_ssh_client_channel.xml"/> + <xi:include href="../specs/specs_ssh_client_key_api.xml"/> + <xi:include href="../specs/specs_ssh_connection.xml"/> + <xi:include href="../specs/specs_ssh_server_channel.xml"/> + <xi:include href="../specs/specs_ssh_server_key_api.xml"/> + <xi:include href="../specs/specs_ssh_sftp.xml"/> + <xi:include href="../specs/specs_ssh_sftpd.xml"/> +</specs> + + diff --git a/lib/ssh/doc/src/ssh.xml b/lib/ssh/doc/src/ssh.xml index acf94ff6af..0223831cb1 100644 --- a/lib/ssh/doc/src/ssh.xml +++ b/lib/ssh/doc/src/ssh.xml @@ -11,8 +11,8 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 + + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -31,185 +31,211 @@ <module>ssh</module> <modulesummary>Main API of the ssh application</modulesummary> <description> - <p>Interface module for the <c>ssh</c> application.</p> - <p>See <seealso marker="ssh:SSH_app#supported">ssh(6)</seealso> for details of supported version, - algorithms and unicode support.</p> + <p>This is the interface module for the <c>SSH</c> application. + The Secure Shell (SSH) Protocol is a protocol for secure remote login + and other secure network services over an insecure network. + See <seealso marker="ssh:SSH_app#supported">ssh(6)</seealso> for details of supported RFCs, versions, + algorithms and unicode handling. + </p> + <p>With the SSH application it is possible to start <i>clients</i> and to start <i>daemons</i> (servers). + </p> + <p>Clients are started with + <seealso marker="#connect/2">connect/2</seealso>, + <seealso marker="#connect/3">connect/3</seealso> or + <seealso marker="#connect/4">connect/4</seealso>. They open an encrypted connection on top of TCP/IP. + In that encrypted connection one or more channels could be opened with + <seealso marker="ssh_connection#session_channel/2">ssh_connection:session_channel/2,4</seealso>. + </p> + <p>Each channel is an isolated "pipe" between a client-side process and a server-side process. Thoose process + pairs could handle for example file transfers (sftp) or remote command execution (shell, exec and/or cli). + If a custom shell is implemented, the user of the client could execute the special commands remotely. Note that + the user is not necessarily a human but probably a system interfacing the SSH app. + </p> + <p>A server-side subssystem (channel) server is requested by the client with + <seealso marker="ssh_connection#subsystem/4">ssh_connection:subsystem/4</seealso>. + </p> + <p>A server (daemon) is started with + <seealso marker="#daemon/2">daemon/1</seealso>, + <seealso marker="#daemon/2">daemon/2</seealso> or + <seealso marker="#daemon/2">daemon/3</seealso>. + Possible channel handlers (subsystems) are declared with the + <seealso marker="#type-subsystem_daemon_option">subsystem</seealso> option when the daemon is started. + </p> + <p>To just run a shell on a remote machine, there are functions that bundles the needed + three steps needed into one: + <seealso marker="#shell/1">shell/1,2,3</seealso>. + Similarily, to just open an sftp (file transfer) connection to a remote machine, the simplest way is to use + <seealso marker="ssh_sftp#start_channel/1">ssh_sftp:start_channel/1,2,3</seealso>. + </p> + <p>To write your own client channel handler, use the behaviour + <seealso marker="ssh_client_channel">ssh_client_channel</seealso>. For server channel handlers use + <seealso marker="ssh_server_channel">ssh_server_channel</seealso> behaviour (replaces ssh_daemon_channel). + </p> + <p>Both clients and daemons accepts options that controls the exact behaviour. Some options are common to both. + The three sets are called + <seealso marker="#type-client_options">Client Options</seealso>, + <seealso marker="#type-daemon_options">Daemon Options</seealso> and + <seealso marker="#type-common_options">Common Options</seealso>. + </p> + <p>The descriptions of the options uses the + <seealso marker="doc/reference_manual:typespec">Erlang Type Language</seealso> with explaining text. + </p> + <note> + <p>The <seealso marker="users_guide">User's Guide</seealso> has examples and a + <seealso marker="using_ssh">Getting Started</seealso> + section. + </p> + </note> </description> <section> - <title>OPTIONS</title> - <p>The exact behaviour of some functions can be adjusted with the use of options which are documented together - with the functions. Generally could each option be used at most one time in each function call. If given two or more - times, the effect is not predictable unless explicitly documented.</p> - <p>The options are of different kinds:</p> - <taglist> - <tag>Limits</tag> - <item><p>which alters limits in the system, for example number of simultaneous login attempts.</p></item> - - <tag>Timeouts</tag> - <item><p>which give some defined behaviour if too long time elapses before a given event or action, - for example time to wait for an answer.</p></item> - - <tag>Callbacks</tag> - <item><p>which gives the caller of the function the possibility to execute own code on some events, - for example calling an own logging function or to perform an own login function</p></item> - - <tag>Behaviour</tag> - <item><p>which changes the systems behaviour.</p></item> - </taglist> + <title>Keys and files</title> + <p>A number of objects must be present for the SSH application to work. + Thoose objects are per default stored in files. + The default names, paths and file formats are the same as for + <url href="http://www.openssh.com">OpenSSH</url>. Keys could be generated with the <c>ssh-keygen</c> + program from OpenSSH. See the + <seealso marker="using_ssh#running-an-erlang-ssh-daemon">User's Guide</seealso>. + </p> + + <p>The paths could easily be changed by options: + <seealso marker="#type-user_dir_common_option"><c>user_dir</c></seealso> and + <seealso marker="#type-system_dir_daemon_option"><c>system_dir</c></seealso>. + </p> + <p>A completly different storage could be interfaced by writing call-back modules + using the behaviours + <seealso marker="ssh_client_key_api">ssh_client_key_api</seealso> and/or + <seealso marker="ssh_server_key_api">ssh_server_key_api</seealso>. + A callback module is installed with the option + <seealso marker="#type-key_cb_common_option"><c>key_cb</c></seealso> + to the client and/or the daemon. + </p> + + <section> + <title>Daemons</title> + <p>The keys are by default stored in files:</p> + <list> + <item>Mandatory: one or more <i>Host key(s)</i>, both private and public. Default is to + store them in the directory <c>/etc/ssh</c> in the files + <list> + <item><c>ssh_host_dsa_key</c> and <c>ssh_host_dsa_key.pub</c></item> + <item><c>ssh_host_rsa_key</c> and <c>ssh_host_rsa_key.pub</c></item> + <item><c>ssh_host_ecdsa_key</c> and <c>ssh_host_ecdsa_key.pub</c></item> + </list> + <p>The host keys directory could be changed with the option + <seealso marker="#type-system_dir_daemon_option"><c>system_dir</c></seealso>.</p> + </item> + <item>Optional: one or more <i>User's public key</i> in case of <c>publickey</c> authorization. + Default is to store them concatenated in the file <c>.ssh/authorized_keys</c> in the user's home directory. + <p>The user keys directory could be changed with the option + <seealso marker="#type-user_dir_common_option"><c>user_dir</c></seealso>.</p> + </item> + </list> + </section> + + <section> + <title>Clients</title> + <p>The keys and some other data are by default stored in files in the directory <c>.ssh</c> + in the user's home directory.</p> + <p>The directory could be changed with the option + <seealso marker="#type-user_dir_common_option"><c>user_dir</c></seealso>. + </p> + <list> + <item>Optional: a list of <i>Host public key(s)</i> for previously connected hosts. This list + is handled by the SSH application without any need of user assistance. The default + is to store them in the file <c>known_hosts</c>. + <p>The + <seealso marker="#type-host_accepting_client_options">host_accepting_client_options()</seealso> + are associated with this list of keys. + </p> + </item> + <item>Optional: one or more <i>User's private key(s)</i> in case of <c>publickey</c> authorization. + The default files are + <list> + <item><c>id_dsa</c> and <c>id_dsa.pub</c></item> + <item><c>id_rsa</c> and <c>id_rsa.pub</c></item> + <item><c>id_ecdsa</c> and <c>id_ecdsa.pub</c></item> + </list> + </item> + </list> + </section> + </section> - <section> - <title>DATA TYPES</title> - <p>Type definitions that are used more than once in - this module, or abstractions to indicate the intended use of the data - type, or both:</p> - <taglist> - <tag><c>boolean() =</c></tag> - <item><p><c>true | false</c></p></item> - <tag><c>string() =</c></tag> - <item><p><c>[byte()]</c></p></item> - <tag><c>ssh_daemon_ref() =</c></tag> - <item><p>opaque() - - as returned by <c>ssh:daemon/[1,2,3]</c></p></item> - <tag><c>ssh_connection_ref() =</c></tag> - <item><p>opaque() - as returned by <c>ssh:connect/3</c></p></item> - <tag><c>ip_address() =</c></tag> - <item><p><c>inet::ip_address</c></p></item> - <tag><c>subsystem_spec() =</c></tag> - <item><p><c>{subsystem_name(), - {channel_callback(), channel_init_args()}}</c></p></item> - <tag><c>subsystem_name() =</c></tag> - <item><p><c>string()</c></p></item> - <tag><c>channel_callback() =</c></tag> - <item><p><c>atom()</c> - Name of the Erlang module - implementing the subsystem using the <c>ssh_channel</c> behavior, see - <seealso marker="ssh_channel">ssh_channel(3)</seealso></p></item> - <tag><c>key_cb() =</c></tag> - <item> - <p><c>atom() | {atom(), list()}</c></p> - <p><c>atom()</c> - Name of the erlang module implementing the behaviours - <seealso marker="ssh_client_key_api">ssh_client_key_api</seealso> or - <seealso marker="ssh_client_key_api">ssh_client_key_api</seealso> as the - case maybe.</p> - <p><c>list()</c> - List of options that can be passed to the callback module.</p> - </item> - <tag><c>channel_init_args() =</c></tag> - <item><p><c>list()</c></p></item> - - <tag><c>algs_list() =</c></tag> - <item><p><c>list( alg_entry() )</c></p></item> - - <tag><c>alg_entry() =</c></tag> - <item><p><c>{kex, simple_algs()} | {public_key, simple_algs()} | {cipher, double_algs()} | {mac, double_algs()} | {compression, double_algs()}</c></p></item> - - <tag><c>simple_algs() =</c></tag> - <item><p><c>list( atom() )</c></p></item> - - <tag><c>double_algs() =</c></tag> - <item><p><c>[{client2serverlist,simple_algs()},{server2client,simple_algs()}] | simple_algs()</c></p></item> + <!-- + ================================================================ + = Data types = + ================================================================ + --> - <tag><c>modify_algs_list() =</c></tag> - <item><p><c>list( {append,algs_list()} | {prepend,algs_list()} | {rm,algs_list()} )</c></p></item> - </taglist> -</section> + <datatypes> - <funcs> - - <func> - <name>close(ConnectionRef) -> ok </name> - <fsummary>Closes an SSH connection.</fsummary> - <type> - <v>ConnectionRef = ssh_connection_ref()</v> - </type> - <desc><p>Closes an SSH connection.</p> + <datatype_title>Client Options</datatype_title> + + <datatype> + <name name="client_options"/> + <name name="client_option"/> + <desc> + <p>Options for <seealso marker="#connect/3">clients</seealso>. + The individual options are further explained below or by following the hyperlinks. + </p> </desc> - </func> - - <func> - <name>connect(Host, Port, Options) -> </name> - <name>connect(Host, Port, Options, Timeout) -> </name> - <name>connect(TcpSocket, Options) -> </name> - <name>connect(TcpSocket, Options, Timeout) -> - {ok, ssh_connection_ref()} | {error, Reason}</name> - <fsummary>Connects to an SSH server.</fsummary> - <type> - <v>Host = string()</v> - <v>Port = integer()</v> - <d><c><![CDATA[22]]></c> is default, the assigned well-known port - number for SSH.</d> - <v>Options = [{Option, Value}]</v> - <v>Timeout = infinity | integer()</v> - <d>Negotiation time-out in milli-seconds. The default value is <c>infinity</c>. - For connection time-out, use option <c>{connect_timeout, timeout()}</c>.</d> - <v>TcpSocket = port()</v> - <d>The socket is supposed to be from <seealso marker="kernel:gen_tcp#connect-3">gen_tcp:connect</seealso> or <seealso marker="kernel:gen_tcp#accept-1">gen_tcp:accept</seealso> with option <c>{active,false}</c></d> - </type> + </datatype> + + <datatype> + <name name="pref_public_key_algs_client_option"/> + <desc> + <p>List of user (client) public key algorithms to try to use.</p> + <p>The default value is the <c>public_key</c> entry in the list returned by + <seealso marker="#default_algorithms/0">ssh:default_algorithms/0</seealso>. + </p> + <p>If there is no public key of a specified type available, the corresponding entry is ignored. + Note that the available set is dependent on the underlying cryptolib and current user's public keys. + </p> + <p>See also the option <seealso marker="#type-user_dir_common_option"><c>user_dir</c></seealso> + for specifying the path to the user's keys. + </p> + </desc> + </datatype> + + <datatype> + <name name="pubkey_passphrase_client_options"/> + <desc> + <p>If the user's DSA, RSA or ECDSA key is protected by a passphrase, it can be + supplied with thoose options. + </p> + </desc> + </datatype> + + <datatype> + <name name="host_accepting_client_options"/> + <name name="accept_hosts"/> + <name name="fp_digest_alg"/> + <name name="accept_callback"/> + <name name="fingerprint"/> <desc> - <p>Connects to an SSH server. No channel is started. This is done - by calling - <seealso marker="ssh_connection#session_channel/2"> - ssh_connection:session_channel/[2, 4]</seealso>.</p> - <p>Options:</p> <taglist> - <tag><c><![CDATA[{inet, inet | inet6}]]></c></tag> - <item> - <p>IP version to use.</p> - </item> - <tag><marker id="opt_user_dir"></marker><c><![CDATA[{user_dir, string()}]]></c></tag> - <item> - <p>Sets the user directory, that is, the directory containing - <c>ssh</c> configuration files for the user, such as - <c><![CDATA[known_hosts]]></c>, <c><![CDATA[id_rsa, - id_dsa]]></c>, and - <c><![CDATA[authorized_key]]></c>. Defaults to the - directory normally referred to as - <c><![CDATA[~/.ssh]]></c>.</p> - </item> - <tag><c><![CDATA[{dsa_pass_phrase, string()}]]></c></tag> - <item> - <p>If the user DSA key is protected by a passphrase, it can be - supplied with this option. - </p> - </item> - <tag><c><![CDATA[{rsa_pass_phrase, string()}]]></c></tag> - <item> - <p>If the user RSA key is protected by a passphrase, it can be - supplied with this option. - </p> - </item> - <tag><c><![CDATA[{ecdsa_pass_phrase, string()}]]></c></tag> - <item> - <p>If the user ECDSA key is protected by a passphrase, it can be - supplied with this option. - </p> - </item> - <tag> - <c><![CDATA[{silently_accept_hosts, boolean()}]]></c> <br/> - <c><![CDATA[{silently_accept_hosts, CallbackFun}]]></c> <br/> - <c><![CDATA[{silently_accept_hosts, {HashAlgoSpec, CallbackFun} }]]></c> <br/> - <br/> - <c><![CDATA[HashAlgoSpec = crypto:digest_type() | [ crypto:digest_type() ] ]]></c><br/> - <c><![CDATA[CallbackFun = fun(PeerName, FingerPrint) -> boolean()]]></c><br/> - <c><![CDATA[PeerName = string()]]></c><br/> - <c><![CDATA[FingerPrint = string() | [ string() ] ]]></c> - </tag> + <tag><c>silently_accept_hosts</c></tag> <item> - <p>This option guides the <c>connect</c> function how to act when the connected server presents a Host + <p>This option guides the <c>connect</c> function on how to act when the connected server presents a Host Key that the client has not seen before. The default is to ask the user with a question on stdio of whether to accept or reject the new Host Key. - See also the option <seealso marker="#opt_user_dir"><c>user_dir</c></seealso> - for the path to the file <c>known_hosts</c> where previously accepted Host Keys are recorded. - </p> + See the option <seealso marker="#type-user_dir_common_option"><c>user_dir</c></seealso> + for specifying the path to the file <c>known_hosts</c> where previously accepted Host Keys are recorded. + See also the option + <seealso marker="#type-key_cb_common_option">key_cb</seealso> + for the general way to handle keys. + </p> <p>The option can be given in three different forms as seen above:</p> <list> - <item>The value is a <c>boolean()</c>. The value <c>true</c> will make the client accept any unknown - Host Key without any user interaction. The value <c>false</c> keeps the default behaviour of asking the - the user on stdio. + <item>The value is a <c>boolean()</c>. + The value <c>true</c> will make the client accept any unknown Host Key without any user interaction. + The value <c>false</c> preserves the default behaviour of asking the user on stdio. </item> - <item>A <c>CallbackFun</c> will be called and the boolean return value <c>true</c> will make the client - accept the Host Key. A return value of <c>false</c> will make the client to reject the Host Key and therefore - also the connection will be closed. The arguments to the fun are: + <item>An <c>accept_callback()</c> will be called and the boolean return value <c>true</c> + will make the client + accept the Host Key. A return value of <c>false</c> will make the client to reject the Host Key and as a + result the connection will be closed. The arguments to the fun are: <list type="bulleted"> <item><c>PeerName</c> - a string with the name or address of the remote host.</item> <item><c>FingerPrint</c> - the fingerprint of the Host Key as @@ -218,471 +244,335 @@ </item> </list> </item> - <item>A tuple <c>{HashAlgoSpec, CallbackFun}</c>. The <c>HashAlgoSpec</c> specifies which hash algorithm - shall be used to calculate the fingerprint used in the call of the <c>CallbackFun</c>. The <c>HashALgoSpec</c> - is either an atom or a list of atoms as the first argument in - <seealso marker="public_key:public_key#ssh_hostkey_fingerprint-2">public_key:ssh_hostkey_fingerprint/2</seealso>. - If it is a list of hash algorithm names, the <c>FingerPrint</c> argument in the <c>CallbackFun</c> will be - a list of fingerprints in the same order as the corresponding name in the <c>HashAlgoSpec</c> list. + <item>A tuple <c>{HashAlgoSpec, accept_callback}</c>. The <c>HashAlgoSpec</c> + specifies which hash algorithm + shall be used to calculate the fingerprint used in the call of the <c>accept_callback()</c>. + The <c>HashALgoSpec</c> + is either an atom or a list of atoms as the first argument in + <seealso marker="public_key:public_key#ssh_hostkey_fingerprint-2">public_key:ssh_hostkey_fingerprint/2</seealso>. + If it is a list of hash algorithm names, the <c>FingerPrint</c> argument in the + <c>accept_callback()</c> will be + a list of fingerprints in the same order as the corresponding name in the <c>HashAlgoSpec</c> list. </item> </list> </item> - - <tag><c><![CDATA[{save_accepted_host, boolean()}]]></c></tag> - <item> - <p>If <c>true</c>, the client saves an accepted host key to avoid the - accept question the next time the same host is connected. If the option - <c>key_cb</c> is not present, the key is saved in the file "known_hosts". - </p> - <p>If <c>false</c>, the key is not saved and the key will still be unknown - at the next access of the same host. - </p> - </item> - - <tag><c><![CDATA[{user_interaction, boolean()}]]></c></tag> + + <tag><c>user_interaction</c></tag> <item> <p>If <c>false</c>, disables the client to connect to the server if any user interaction is needed, such as accepting the server to be added to the <c>known_hosts</c> file, or - supplying a password. Defaults to <c>true</c>. - Even if user interaction is allowed it can be + supplying a password.</p> + <p>Even if user interaction is allowed it can be suppressed by other options, such as <c>silently_accept_hosts</c> and <c>password</c>. However, those options are not always desirable to use from a security point of view.</p> + <p>Defaults to <c>true</c>.</p> </item> - <tag><c><![CDATA[{disconnectfun, fun(Reason:term()) -> _}]]></c></tag> + <tag><c>save_accepted_host</c></tag> <item> - <p>Provides a fun to implement your own logging when a server disconnects the client.</p> - </item> - - <tag><c><![CDATA[{unexpectedfun, fun(Message:term(), Peer) -> report | skip }]]></c></tag> - <item> - <p>Provides a fun to implement your own logging or other action when an unexpected message arrives. - If the fun returns <c>report</c> the usual info report is issued but if <c>skip</c> is returned no - report is generated.</p> - <p><c>Peer</c> is in the format of <c>{Host,Port}</c>.</p> - </item> - - <tag><c><![CDATA[{pref_public_key_algs, list()}]]></c></tag> - <item> - <p>List of user (client) public key algorithms to try to use.</p> - <p>The default value is the <c>public_key</c> entry in - <seealso marker="#default_algorithms/0">ssh:default_algorithms/0</seealso>. - </p> - <p>If there is no public key of a specified type available, the corresponding entry is ignored. - Note that the available set is dependent on the underlying cryptolib and current user's public keys. - </p> - </item> - - <tag><marker id="option_preferred_algorithms"></marker> - <c><![CDATA[{preferred_algorithms, algs_list()}]]></c></tag> - <item> - <p>List of algorithms to use in the algorithm negotiation. The default <c>algs_list()</c> can - be obtained from <seealso marker="#default_algorithms/0">default_algorithms/0</seealso>. + <p>If <c>true</c>, the client saves an accepted host key to avoid the + accept question the next time the same host is connected. If the option + <seealso marker="#type-key_cb_common_option"><c>key_cb</c></seealso> + is not present, the key is saved in the file "known_hosts". See option + <seealso marker="#type-user_dir_common_option"><c>user_dir</c></seealso> for + the location of that file. </p> - <p>If an alg_entry() is missing in the algs_list(), the default value is used for that entry.</p> - <p>Here is an example of this option:</p> - <code> -{preferred_algorithms, - [{public_key,['ssh-rsa','ssh-dss']}, - {cipher,[{client2server,['aes128-ctr']}, - {server2client,['aes128-cbc','3des-cbc']}]}, - {mac,['hmac-sha2-256','hmac-sha1']}, - {compression,[none,zlib]} - ] -} -</code> - <p>The example specifies different algorithms in the two directions (client2server and server2client), - for cipher but specifies the same algorithms for mac and compression in both directions. - The kex (key exchange) is implicit but public_key is set explicitly.</p> - - <p>For background and more examples see the <seealso marker="configure_algos#introduction">User's Guide</seealso>.</p> - - <warning> - <p>Changing the values can make a connection less secure. Do not change unless you - know exactly what you are doing. If you do not understand the values then you - are not supposed to change them.</p> - </warning> - </item> - - <tag><marker id="option_modify_algorithms"></marker> - <c><![CDATA[{modify_algorithms, modify_algs_list()}]]></c></tag> - <item> - <p>Modifies the list of algorithms to use in the algorithm negotiation. The modifications are - applied after the option <c>preferred_algorithms</c> (if existing) is applied.</p> - <p>The algoritm for modifications works like this:</p> - <list> - <item> - <p>Input is the <c>modify_algs_list()</c> and a set of algorithms <c>A</c> - obtained from the <c>preferred_algorithms</c> option if existing, or else from the - <seealso marker="ssh#default_algorithms-0">ssh:default_algorithms/0</seealso>. - </p> - </item> - <item> - <p>The head of the <c>modify_algs_list()</c> modifies <c>A</c> giving the result <c>A'</c>.</p> - <p>The possible modifications are:</p> - <list> - <item> - <p>Append or prepend supported but not enabled algorithm(s) to the list of - algorithms. If the wanted algorithms already are in <c>A</c> they will first - be removed and then appended or prepended, - </p> - </item> - <item> - <p>Remove (rm) one or more algorithms from <c>A</c>. - </p> - </item> - </list> - </item> - <item> - <p>Repeat the modification step with the tail of <c>modify_algs_list()</c> and the resulting - <c>A'</c>. - </p> - </item> - </list> - <p>If an unsupported algorithm is in the <c>modify_algs_list()</c>, it will be silently ignored</p> - <p>If there are more than one modify_algorithms options, the result is undefined.</p> - <p>Here is an example of this option:</p> - <code> -{modify_algorithms, - [{prepend, [{kex, ['diffie-hellman-group1-sha1']}], - {rm, [{compression, [none]}]} - ] -} -</code> - <p>The example specifies that:</p> - <list> - <item><p>the old key exchange algorithm 'diffie-hellman-group1-sha1' should be - the main alternative. It will be the main alternative since it is prepened to the list</p> - </item> - <item><p>The compression algorithm none (= no compression) is removed so compression is enforced</p> - </item> - </list> - <p>For background and more examples see the <seealso marker="configure_algos#introduction">User's Guide</seealso>.</p> - </item> - - <tag><c><![CDATA[{dh_gex_limits,{Min=integer(),I=integer(),Max=integer()}}]]></c></tag> - <item> - <p>Sets the three diffie-hellman-group-exchange parameters that guides the connected server in choosing a group. - See RFC 4419 for the function of thoose. The default value is <c>{1024, 6144, 8192}</c>. + <p>If <c>false</c>, the key is not saved and the key will still be unknown + at the next access of the same host. </p> + <p>Defaults to <c>true</c></p> </item> - <tag><c><![CDATA[{connect_timeout, timeout()}]]></c></tag> + <tag><c>quiet_mode</c></tag> <item> - <p>Sets a time-out on the transport layer - connection. For <c>gen_tcp</c> the time is in milli-seconds and the default value is - <c>infinity</c>.</p> + <p>If <c>true</c>, the client does not print anything on authorization.</p> + <p>Defaults to <c>false</c></p> </item> + </taglist> + </desc> + </datatype> - <tag><c><![CDATA[{auth_methods, string()}]]></c></tag> - <item> - <p>Comma-separated string that determines which - authentication methods that the client shall support and - in which order they are tried. Defaults to - <c><![CDATA["publickey,keyboard-interactive,password"]]></c></p> - </item> - - <tag><c><![CDATA[{user, string()}]]></c></tag> + <datatype> + <name name="authentication_client_options"/> + <desc> + <taglist> + <tag><c>user</c></tag> <item> - <p>Provides a username. If this option is not given, <c>ssh</c> + <p>Provides the username. If this option is not given, <c>ssh</c> reads from the environment (<c><![CDATA[LOGNAME]]></c> or <c><![CDATA[USER]]></c> on UNIX, <c><![CDATA[USERNAME]]></c> on Windows).</p> </item> - <tag><c><![CDATA[{password, string()}]]></c></tag> + <tag><c>password</c></tag> <item> <p>Provides a password for password authentication. If this option is not given, the user is asked for a password, if the password authentication method is attempted.</p> </item> - - <!--tag><c><![CDATA[{send_ext_info, boolean()}]]></c></tag> - <item> - <p>Send a list of extensions to the server if the server has asked for it. See - <url href="https://tools.ietf.org/html/draft-ietf-curdle-ssh-ext-info">Draft-ietf-curdle-ssh-ext-info (work in progress)</url> for details. - </p> - <p>Currently the client do not react on any extensions. - </p> - <p>Default value is <c>true</c>. - </p> - </item--> - - <tag><c><![CDATA[{recv_ext_info, boolean()}]]></c></tag> - <item> - <p>Tell the server that the client accepts extension negotiation. See - <url href="https://tools.ietf.org/html/draft-ietf-curdle-ssh-ext-info">Draft-ietf-curdle-ssh-ext-info (work in progress)</url> for details. - </p> - <p>Currently implemented extension is <c>server-sig-algs</c> which is the list of the server's preferred - user's public key algorithms. - </p> - <p>Default value is <c>true</c>. - </p> - </item> - - <tag><c><![CDATA[{key_cb, key_cb()}]]></c></tag> - <item> - <p>Module implementing the behaviour <seealso - marker="ssh_client_key_api">ssh_client_key_api</seealso>. Can be used to - customize the handling of public keys. If callback options are provided - along with the module name, they are made available to the callback - module via the options passed to it under the key 'key_cb_private'. - </p> - </item> - - <tag><c><![CDATA[{quiet_mode, atom() = boolean()}]]></c></tag> - <item> - <p>If <c>true</c>, the client does not print anything on authorization.</p> - </item> - - <tag><c><![CDATA[{id_string, random | string()}]]></c></tag> - <item> - <p>The string that the client presents to a connected server initially. The default value is "Erlang/VSN" where VSN is the ssh application version number. - </p> - <p>The value <c>random</c> will cause a random string to be created at each connection attempt. This is to make it a bit more difficult for a malicious peer to find the ssh software brand and version. - </p> - </item> - - <tag><c><![CDATA[{fd, file_descriptor()}]]></c></tag> - <item> - <p>Allows an existing file descriptor to be used - (by passing it on to the transport protocol).</p></item> - <tag><c><![CDATA[{rekey_limit, integer()}]]></c></tag> - <item> - <p>Provides, in bytes, when rekeying is to be initiated. - Defaults to once per each GB and once per hour.</p> - </item> - <tag><c><![CDATA[{idle_time, integer()}]]></c></tag> - <item> - <p>Sets a time-out on a connection when no channels are active. - Defaults to <c>infinity</c>.</p></item> - <tag><c><![CDATA[{ssh_msg_debug_fun, fun(ConnectionRef::ssh_connection_ref(), AlwaysDisplay::boolean(), Msg::binary(), LanguageTag::binary()) -> _}]]></c></tag> - <item> - <p>Provide a fun to implement your own logging of the SSH message SSH_MSG_DEBUG. The last three parameters are from the message, see RFC4253, section 11.3. The <c>ConnectionRef</c> is the reference to the connection on which the message arrived. The return value from the fun is not checked.</p> - <p>The default behaviour is ignore the message. - To get a printout for each message with <c>AlwaysDisplay = true</c>, use for example <c>{ssh_msg_debug_fun, fun(_,true,M,_)-> io:format("DEBUG: ~p~n", [M]) end}</c></p> - </item> - </taglist> - </desc> - </func> + </desc> + </datatype> - <func> - <name>connection_info(ConnectionRef, [Option]) ->[{Option, - Value}]</name> - <fsummary>Retrieves information about a connection.</fsummary> - <type> - <v>Option = client_version | server_version | user | peer | sockname </v> - <v>Value = [option_value()] </v> - <v>option_value() = {{Major::integer(), Minor::integer()}, VersionString::string()} | - User::string() | Peer::{inet:hostname(), {inet::ip_adress(), inet::port_number()}} | - Sockname::{inet::ip_adress(), inet::port_number()}</v> - </type> + <datatype> + <name name="diffie_hellman_group_exchange_client_option"/> <desc> - <p>Retrieves information about a connection.</p> + <p>Sets the three diffie-hellman-group-exchange parameters that guides the connected server in choosing a group. + See + <url href="https://tools.ietf.org/html/rfc4419">RFC 4419</url> + for the details. The default value is <c>{1024, 6144, 8192}</c>. + </p> </desc> - </func> - - <func> - <name>daemon(Port) -> </name> - <name>daemon(Port, Options) -> </name> - <name>daemon(HostAddress, Port, Options) -> </name> - <name>daemon(TcpSocket) -> </name> - <name>daemon(TcpSocket, Options) -> {ok, ssh_daemon_ref()} | {error, atom()}</name> - <fsummary>Starts a server listening for SSH connections - on the given port.</fsummary> - <type> - <v>Port = integer()</v> - <v>HostAddress = ip_address() | any | loopback</v> - <v>Options = [{Option, Value}]</v> - <v>Option = atom()</v> - <v>Value = term()</v> - <v>TcpSocket = port()</v> - <d>The socket is supposed to be from <seealso marker="kernel:gen_tcp#connect-3">gen_tcp:connect</seealso> or <seealso marker="kernel:gen_tcp#accept-1">gen_tcp:accept</seealso> with option <c>{active,false}</c></d> - </type> - <desc> - <p>Starts a server listening for SSH connections on the given - port. If the <c>Port</c> is 0, a random free port is selected. See - <seealso marker="#daemon_info/1">daemon_info/1</seealso> about how to find the selected port number.</p> + </datatype> - <p>Please note that by historical reasons both the <c>HostAddress</c> argument and the inet socket option - <c>ip</c> set the listening address. This is a source of possible inconsistent settings.</p> + <datatype> + <name name="connect_timeout_client_option"/> + <desc> + <p>Sets a timeout on the transport layer connect time. + For <seealso marker="kernel:gen_tcp"><c>gen_tcp</c></seealso> the time is in milli-seconds and the default + value is <c>infinity</c>. + </p> + <p>See the parameter <c>Timeout</c> in <seealso marker="#connect/4">connect/4</seealso> for + a timeout of the negotiation phase. + </p> + </desc> + </datatype> - <p>The rules for handling the two address passing options are:</p> - <list> - <item>if <c>HostAddress</c> is an IP-address, that IP-address is the listening address. - An 'ip'-option will be discarded if present.</item> + <datatype> + <name name="recv_ext_info_client_option"/> + <desc> + <p>Make the client tell the server that the client accepts extension negotiation, that is, + include <c>ext-info-c</c> in the kexinit message sent. See + <url href="https://tools.ietf.org/html/rfc8308">RFC 8308</url> + for details and <seealso marker="SSH_app#supported-ext-info">ssh(6)</seealso> + for a list of currently implemented extensions. + </p> + <p> + Default value is <c>true</c> which is compatible with other implementations not supporting ext-info. + </p> + </desc> + </datatype> - <item>if <c>HostAddress</c> is <c>loopback</c>, the listening address - is <c>loopback</c> and an loopback address will be choosen by the underlying layers. - An 'ip'-option will be discarded if present.</item> + <!--................................................................--> + <datatype_title>Daemon Options (Server Options)</datatype_title> - <item>if <c>HostAddress</c> is <c>any</c> and no 'ip'-option is present, the listening address is - <c>any</c> and the socket will listen to all addresses</item> + <datatype> + <name name="daemon_options"/> + <name name="daemon_option"/> + <desc> + <p>Options for <seealso marker="#daemon/1">daemons</seealso>. + The individual options are further explained below or by following the hyperlinks. + </p> + </desc> + </datatype> - <item>if <c>HostAddress</c> is <c>any</c> and an 'ip'-option is present, the listening address is - set to the value of the 'ip'-option</item> - </list> + + <datatype> + <name name="subsystem_daemon_option"/> + <name name="subsystem_spec"/> + <desc> + <p>Defines a subsystem in the daemon.</p> + <p>The <c>subsystem_name</c> is the name that a client requests to start with for example + <seealso marker="ssh_connection#subsystem/4">ssh_connection:subsystem/4</seealso>. + </p> + <p>The <c>channel_callback</c> is the module that implements the + <seealso marker="ssh_server_channel">ssh_server_channel</seealso> (replaces ssh_daemon_channel) + behaviour in the daemon. See the section + <seealso marker="using_ssh#usersguide_creating_a_subsystem">Creating a Subsystem</seealso> + in the User's Guide for more information and an example. + </p> + <p>If the subsystems option is not present, the value of <c>ssh_sftpd:subsystem_spec([])</c> is used. + This enables the sftp subsystem by default. + The option can be set to the empty list if you do not want the daemon to run any subsystems.</p> + </desc> + </datatype> - <p>Options:</p> - <taglist> - <tag><c><![CDATA[{inet, inet | inet6}]]></c></tag> - <item><p>IP version to use when the host address is specified as <c>any</c>.</p></item> - <tag><c><![CDATA[{subsystems, [subsystem_spec()]}]]></c></tag> + <datatype> + <name name="shell_daemon_option"/> + <name name="'shell_fun/1'"/> + <name name="'shell_fun/2'"/> + <desc> + <p>Defines the read-eval-print loop used in a daemon when a shell is requested by the client. + The default is to use the Erlang shell: <c><![CDATA[{shell, start, []}]]></c> + </p> + <p>See the option <seealso marker="#type-exec_daemon_option"><c>exec</c></seealso> + for a description of how the daemon execute exec-requests depending on + the shell- and exec-options.</p> + </desc> + </datatype> + + <datatype> + <name name="exec_daemon_option"/> + <name name="'exec_fun/1'"/> + <name name="'exec_fun/2'"/> + <name name="'exec_fun/3'"/> + <name name="exec_result"/> + <desc> + <p>This option changes how the daemon execute exec-requests from clients. The term in the return value + is formatted to a string if it is a non-string type. No trailing newline is added in the ok-case but in the + error case.</p> + <p>Error texts are returned on channel-type 1 which usually is piped to <c>stderr</c> on e.g Linux systems. + Texts from a successful execution will in similar manner be piped to <c>stdout</c>. The exit-status code + is set to 0 for success and -1 for errors. The exact results presented on the client side depends on the + client and the client's operating system. + </p> + <p>The option cooperates with the daemon-option <seealso marker="#type-shell_daemon_option"><c>shell</c></seealso> + in the following way:</p> + <taglist> + <tag>1. If the exec-option is present (the shell-option may or may not be present):</tag> <item> - <p>Provides specifications for handling of subsystems. The - "sftp" subsystem specification is retrieved by calling - <c>ssh_sftpd:subsystem_spec/1</c>. If the subsystems option is - not present, the value of - <c>[ssh_sftpd:subsystem_spec([])]</c> is used. - The option can be set to the empty list if - you do not want the daemon to run any subsystems.</p> + <p>The exec-option fun is called with the same number of parameters as the arity of the fun, + and the result is returned to the client. + </p> </item> - <tag><c><![CDATA[{shell, {Module, Function, Args} | - fun(string() = User) - > pid() | fun(string() = User, - ip_address() = PeerAddr) -> pid()}]]></c></tag> + + <tag>2. If the exec-option is absent, but a shell-option is present with the default Erlang shell:</tag> <item> - <p>Defines the read-eval-print loop used when a shell is - requested by the client. The default is to use the Erlang shell: - <c><![CDATA[{shell, start, []}]]></c></p> + <p>The default Erlang evaluator is used and the result is returned to the client.</p> </item> - <tag><c><![CDATA[{ssh_cli, {channel_callback(), - channel_init_args()} | no_cli}]]></c></tag> + + <tag>3. If the exec-option is absent, but a shell-option is present that is not the default Erlang shell:</tag> <item> - <p>Provides your own CLI implementation, that is, a channel callback - module that implements a shell and command execution. The shell - read-eval-print loop can be customized, using the - option <c>shell</c>. This means less work than implementing - an own CLI channel. If set to <c>no_cli</c>, the CLI channels - are disabled and only subsystem channels are allowed.</p> + <p>The exec-request is not evaluated and an error message is returned to the client.</p> </item> - <tag><c><![CDATA[{user_dir, string()}]]></c></tag> + + <tag>4. If neither the exec-option nor the shell-option is present:</tag> <item> - <p>Sets the user directory. That is, the directory containing - <c>ssh</c> configuration files for the user, such as - <c><![CDATA[known_hosts]]></c>, <c><![CDATA[id_rsa, - id_dsa]]></c>, and - <c><![CDATA[authorized_key]]></c>. Defaults to the - directory normally referred to as - <c><![CDATA[~/.ssh]]></c>.</p> + <p>The default Erlang evaluator is used and the result is returned to the client.</p> </item> - <tag><c><![CDATA[{system_dir, string()}]]></c></tag> + </taglist> + <p>If a custom CLI is installed (see the option <seealso marker="#type-ssh_cli_daemon_option"><c>ssh_cli</c></seealso>) + the rules above are replaced by thoose implied by the custom CLI. + </p> + <note> + <p>The exec-option has existed for a long time but has not previously been documented. The old + definition and behaviour are retained but obey the rules 1-4 above if conflicting. + The old and undocumented style should not be used in new programs.</p> + </note> + </desc> + </datatype> + + <datatype> + <name name="ssh_cli_daemon_option"/> + <desc> + <p>Provides your own CLI implementation in a daemon.</p> + <p>It is a channel callback module that implements a shell + and command execution. The shell's read-eval-print loop can be customized, using the + option <seealso marker="#type-shell_daemon_option"><c>shell</c></seealso>. This means less work than implementing + an own CLI channel. If <c>ssh_cli</c> is set to <c>no_cli</c>, the CLI channels + like <seealso marker="#type-shell_daemon_option"><c>shell</c></seealso> + and <seealso marker="#type-exec_daemon_option"><c>exec</c></seealso> + are disabled and only subsystem channels are allowed.</p> + </desc> + </datatype> + + <datatype> + <name name="authentication_daemon_options"/> + <name name="prompt_texts"/> + <name name="kb_int_tuple"/> + <name name="kb_int_fun_3"/> + <name name="pwdfun_2"/> + <name name="pwdfun_4"/> + <desc> + <taglist> + <tag><marker id="type-system_dir_daemon_option"/><c>system_dir</c></tag> <item> <p>Sets the system directory, containing the host key files that identify the host keys for <c>ssh</c>. Defaults to - <c><![CDATA[/etc/ssh]]></c>. For security reasons, - this directory is normally accessible only to the root user.</p> + <c>/etc/ssh</c>.</p> + <p>For security reasons, this directory is normally accessible only to the root user.</p> + <p>See also the option + <seealso marker="#type-key_cb_common_option">key_cb</seealso> + for the general way to handle keys. + </p> </item> - <tag><c><![CDATA[{auth_methods, string()}]]></c></tag> + <tag><c>auth_method_kb_interactive_data</c></tag> <item> - <p>Comma-separated string that determines which - authentication methods that the server is to support and - in what order they are tried. Defaults to - <c><![CDATA["publickey,keyboard-interactive,password"]]></c></p> - <p>Note that the client is free to use any order and to exclude methods.</p> - </item> - - <tag><c><![CDATA[{auth_method_kb_interactive_data, PromptTexts}]]></c> - <br/><c>where:</c> - <br/><c>PromptTexts = kb_int_tuple() | fun(Peer::{IP::tuple(),Port::integer()}, User::string(), Service::string()) -> kb_int_tuple()</c> - <br/><c>kb_int_tuple() = {Name::string(), Instruction::string(), Prompt::string(), Echo::boolean()}</c> - </tag> - <item> - <p>Sets the text strings that the daemon sends to the client for presentation to the user when using <c>keyboar-interactive</c> authentication. If the fun/3 is used, it is called when the actual authentication occurs and may therefore return dynamic data like time, remote ip etc.</p> + <p>Sets the text strings that the daemon sends to the client for presentation to the user when + using <c>keyboard-interactive</c> authentication.</p> + <p>If the fun/3 is used, it is called when the actual authentication occurs and may therefore + return dynamic data like time, remote ip etc.</p> <p>The parameter <c>Echo</c> guides the client about need to hide the password.</p> <p>The default value is: - <c>{auth_method_kb_interactive_data, {"SSH server", "Enter password for \""++User++"\"", "password: ", false}></c></p> + <c>{auth_method_kb_interactive_data, {"SSH server", "Enter password for \""++User++"\"", "password: ", false}></c> + </p> </item> - <tag><c><![CDATA[{user_passwords, [{string() = User, - string() = Password}]}]]></c></tag> + <tag><c>user_passwords</c></tag> <item> - <p>Provides passwords for password authentication. The passwords - are used when someone tries to connect to the server and - public key user-authentication fails. The option provides + <p>Provides passwords for password authentication. The passwords are used when someone tries + to connect to the server and public key user-authentication fails. The option provides a list of valid usernames and the corresponding passwords. </p> </item> - <tag><c><![CDATA[{password, string()}]]></c></tag> + + <tag><c>password</c></tag> <item> - <p>Provides a global password that authenticates any - user. From a security perspective this option makes - the server very vulnerable.</p> + <p>Provides a global password that authenticates any user.</p> + <warning> + <p>Intended to facilitate testing.</p> + <p>From a security perspective this option makes the server very vulnerable.</p> + </warning> </item> - <tag><c><![CDATA[{preferred_algorithms, algs_list()}]]></c></tag> + <tag><c>pwdfun</c> with <c>pwdfun_4()</c></tag> <item> - <p>List of algorithms to use in the algorithm negotiation. The default <c>algs_list()</c> can - be obtained from <seealso marker="#default_algorithms/0">default_algorithms/0</seealso>. + <p>Provides a function for password validation. This could used for calling an external system or handeling + passwords stored as hash values. </p> - <p>If an alg_entry() is missing in the algs_list(), the default value is used for that entry.</p> - <p>Here is an example of this option:</p> - <code> -{preferred_algorithms, - [{public_key,['ssh-rsa','ssh-dss']}, - {cipher,[{client2server,['aes128-ctr']}, - {server2client,['aes128-cbc','3des-cbc']}]}, - {mac,['hmac-sha2-256','hmac-sha1']}, - {compression,[none,zlib]} - ] -} -</code> - <p>The example specifies different algorithms in the two directions (client2server and server2client), - for cipher but specifies the same algorithms for mac and compression in both directions. - The kex (key exchange) is implicit but public_key is set explicitly.</p> - - <p>For background and more examples see the <seealso marker="configure_algos#introduction">User's Guide</seealso>.</p> + <p>This fun can also be used to make delays in authentication tries for example by calling + <seealso marker="stdlib:timer#sleep/1">timer:sleep/1</seealso>.</p> + <p>To facilitate for instance counting of failed tries, + the <c>State</c> variable could be used. This state is per connection only. The first time the pwdfun + is called for a connection, the <c>State</c> variable has the value <c>undefined</c>. + </p> + + <p>The fun should return: + </p> + <list type="bulleted"> + <item><c>true</c> if the user and password is valid</item> + <item><c>false</c> if the user or password is invalid</item> + <item><c>disconnect</c> if a SSH_MSG_DISCONNECT message should be sent immediately. It will + be followed by a close of the underlying tcp connection.</item> + <item><c>{true, NewState:any()}</c> if the user and password is valid</item> + <item><c>{false, NewState:any()}</c> if the user or password is invalid</item> + </list> - <warning> - <p>Changing the values can make a connection less secure. Do not change unless you - know exactly what you are doing. If you do not understand the values then you - are not supposed to change them.</p> - </warning> + <p>A third usage is to block login attempts from a missbehaving peer. The <c>State</c> described above + can be used for this. The return value <c>disconnect</c> is useful for this.</p> </item> - <tag><marker id="option_modify_algorithms"></marker> - <c><![CDATA[{modify_algorithms, modify_algs_list()}]]></c></tag> + <tag><c>pwdfun</c> with <c>pwdfun_2()</c></tag> <item> - <p>Modifies the list of algorithms to use in the algorithm negotiation. The modifications are - applied after the option <c>preferred_algorithms</c> is applied (if existing)</p> - <p>The possible modifications are to:</p> - <list> - <item><p>Append or prepend supported but not enabled algorithm(s) to the list of - algorithms.</p><p>If the wanted algorithms already are in the list of algorithms, they will first - be removed and then appended or prepended. - </p> - </item> - <item><p>Remove (rm) one or more algorithms from the list of algorithms.</p></item> + <p>Provides a function for password validation. This function is called with user and password + as strings, and returns:</p> + <list type="bulleted"> + <item><c>true</c> if the user and password is valid</item> + <item><c>false</c> if the user or password is invalid</item> </list> - <p>If an unsupported algorithm is in the list, it will be silently ignored</p> - - <p>Here is an example of this option:</p> - <code> -{modify_algorithms, - [{prepend, [{kex, ['diffie-hellman-group1-sha1']}], - {rm, [{compression, [none]}]} - ] -} -</code> - <p>The example specifies that:</p> - <list> - <item><p>the old key exchange algorithm 'diffie-hellman-group1-sha1' should be - the main alternative. It will be the main alternative since it is prepened to the list</p> - </item> - <item><p>The compression algorithm none (= no compression) is removed so compression is enforced</p> - </item> - </list> - <p>For background and more examples see the <seealso marker="configure_algos#introduction">User's Guide</seealso>.</p> + <p>This variant is kept for compatibility.</p> </item> + </taglist> + </desc> + </datatype> - <tag><c><![CDATA[{dh_gex_groups, [{Size=integer(),G=integer(),P=integer()}] | {file,filename()} {ssh_moduli_file,filename()} }]]></c></tag> + <datatype> + <name name="diffie_hellman_group_exchange_daemon_option"/> + <name name="explicit_group"/> + <name name="explicit_group_file"/> + <name name="ssh_moduli_file"/> + <desc> + <taglist> + <tag><c>dh_gex_groups</c></tag> <item> <p>Defines the groups the server may choose among when diffie-hellman-group-exchange is negotiated. - See RFC 4419 for details. The three variants of this option are: + See + <url href="https://tools.ietf.org/html/rfc4419">RFC 4419</url> + for details. The three variants of this option are: </p> <taglist> <tag><c>{Size=integer(),G=integer(),P=integer()}</c></tag> @@ -704,7 +594,7 @@ </p> </item> - <tag><c><![CDATA[{dh_gex_limits,{Min=integer(),Max=integer()}}]]></c></tag> + <tag><c>dh_gex_limits</c></tag> <item> <p>Limits what a client can ask for in diffie-hellman-group-exchange. The limits will be @@ -715,57 +605,29 @@ </p> <p>If <c>MaxUsed < MinUsed</c> in a key exchange, it will fail with a disconnect. </p> - <p>See RFC 4419 for the function of the Max and Min values.</p> - </item> - - <tag><c><![CDATA[{pwdfun, fun(User::string(), Password::string(), PeerAddress::{ip_adress(),port_number()}, State::any()) -> boolean() | disconnect | {boolean(),any()} }]]></c></tag> - <item> - <p>Provides a function for password validation. This could used for calling an external system or if - passwords should be stored as a hash. The fun returns: - </p> - <list type="bulleted"> - <item><c>true</c> if the user and password is valid and</item> - <item><c>false</c> otherwise.</item> - </list> - <p>This fun can also be used to make delays in authentication tries for example by calling - <seealso marker="stdlib:timer#sleep/1">timer:sleep/1</seealso>. To facilitate counting of failed tries - the <c>State</c> variable could be used. This state is per connection only. The first time the pwdfun - is called for a connection, the <c>State</c> variable has the value <c>undefined</c>. - The pwdfun can return - in addition to the values above - a new state - as: - </p> - <list type="bulleted"> - <item><c>{true, NewState:any()}</c> if the user and password is valid or</item> - <item><c>{false, NewState:any()}</c> if the user or password is invalid</item> - </list> - <p>A third usage is to block login attempts from a missbehaving peer. The <c>State</c> described above - can be used for this. In addition to the responses above, the following return value is introduced: - </p> - <list type="bulleted"> - <item><c>disconnect</c> if the connection should be closed immediately after sending a SSH_MSG_DISCONNECT - message.</item> - </list> - </item> - - <tag><c><![CDATA[{pwdfun, fun(User::string(), Password::string()) -> boolean()}]]></c></tag> - <item> - <p>Provides a function for password validation. This function is called - with user and password as strings, and returns - <c><![CDATA[true]]></c> if the password is valid and - <c><![CDATA[false]]></c> otherwise.</p> - <p>This option (<c>{pwdfun,fun/2}</c>) is the same as a subset of the previous - (<c>{pwdfun,fun/4}</c>). It is kept for compatibility.</p> + <p>See + <url href="https://tools.ietf.org/html/rfc4419">RFC 4419</url> + for the function of the Max and Min values.</p> </item> + </taglist> + </desc> + </datatype> - <tag><c><![CDATA[{negotiation_timeout, integer()}]]></c></tag> - <item> - <p>Maximum time in milliseconds for the authentication negotiation. - Defaults to 120000 (2 minutes). If the client fails to log in within this time, - the connection is closed. - </p> - </item> + <datatype> + <name name="negotiation_timeout_daemon_option"/> + <desc> + <p>Maximum time in milliseconds for the authentication negotiation. + Defaults to 120000 ms (2 minutes). If the client fails to log in within this time, + the connection is closed. + </p> + </desc> + </datatype> - <tag><c><![CDATA[{max_sessions, pos_integer()}]]></c></tag> + <datatype> + <name name="hardening_daemon_options"/> + <desc> + <taglist> + <tag><c>max_sessions</c></tag> <item> <p>The maximum number of simultaneous sessions that are accepted at any time for this daemon. This includes sessions that are being authorized. @@ -785,7 +647,7 @@ </p> </item> - <tag><c><![CDATA[{max_channels, pos_integer()}]]></c></tag> + <tag><c>max_channels</c></tag> <item> <p>The maximum number of channels with active remote subsystem that are accepted for each connection to this daemon</p> @@ -793,8 +655,7 @@ </p> </item> - - <tag><c><![CDATA[{parallel_login, boolean()}]]></c></tag> + <tag><c>parallel_login</c></tag> <item> <p>If set to false (the default value), only one login is handled at a time. If set to true, an unlimited number of login attempts are allowed simultaneously. @@ -811,171 +672,543 @@ </warning> </item> - <tag><c><![CDATA[{minimal_remote_max_packet_size, non_negative_integer()}]]></c></tag> + <tag><c>minimal_remote_max_packet_size</c></tag> <item> - <p>The least maximum packet size that the daemon will accept in channel open requests from the client. The default value is 0. + <p>The least maximum packet size that the daemon will accept in channel open requests from the client. + The default value is 0. </p> </item> + + </taglist> + </desc> + </datatype> + + <datatype> + <name name="callbacks_daemon_options"/> + <desc> + <taglist> + <tag><c>connectfun</c></tag> + <item> + <p>Provides a fun to implement your own logging when a user authenticates to the server.</p> + </item> - <tag><c><![CDATA[{id_string, random | string()}]]></c></tag> + <tag><c>failfun</c></tag> <item> - <p>The string the daemon will present to a connecting peer initially. The default value is "Erlang/VSN" where VSN is the ssh application version number. - </p> - <p>The value <c>random</c> will cause a random string to be created at each connection attempt. This is to make it a bit more difficult for a malicious peer to find the ssh software brand and version. - </p> + <p>Provides a fun to implement your own logging when a user fails to authenticate.</p> </item> + </taglist> + </desc> + </datatype> - <tag><c><![CDATA[{send_ext_info, boolean()}]]></c></tag> - <item> - <p>Send a list of extensions to the client if the client has asked for it. See - <url href="https://tools.ietf.org/html/draft-ietf-curdle-ssh-ext-info">Draft-ietf-curdle-ssh-ext-info (work in progress)</url> for details. - </p> - <p>Currently implemented extension is sending <c>server-sig-algs</c> which is the list of the server's preferred - user's public key algorithms. - </p> - <p>Default value is <c>true</c>. - </p> - </item> + <datatype> + <name name="send_ext_info_daemon_option"/> + <desc> + <p>Make the server (daemon) tell the client that the server accepts extension negotiation, that is, + include <c>ext-info-s</c> in the kexinit message sent. See + <url href="https://tools.ietf.org/html/rfc8308">RFC 8308</url> + for details and <seealso marker="SSH_app#supported-ext-info">ssh(6)</seealso> + for a list of currently implemented extensions. + </p> + <p>Default value is <c>true</c> which is compatible with other implementations not supporting ext-info. + </p> + </desc> + </datatype> - <!--tag><c><![CDATA[{recv_ext_info, boolean()}]]></c></tag> - <item> - <p>Tell the client that the server accepts extension negotiation. See - <url href="https://tools.ietf.org/html/draft-ietf-curdle-ssh-ext-info">Draft-ietf-curdle-ssh-ext-info (work in progress)</url> for details. - </p> - <p>Default value is <c>true</c>. - </p> - </item--> - <tag><c><![CDATA[{key_cb, key_cb()}]]></c></tag> + <!--................................................................--> + <datatype_title>Options common to clients and daemons</datatype_title> + <datatype> + <name name="common_options"/> + <name name="common_option"/> + <desc><p>The options above can be used both in clients and in daemons (servers). They are further explained below.</p> + </desc> + </datatype> + + <datatype> + <name name="user_dir_common_option"/> + <desc> + <p>Sets the user directory. That is, the directory containing <c>ssh</c> configuration + files for the user, such as + <c>known_hosts</c>, <c>id_rsa</c>, <c>id_dsa</c>>, <c>id_ecdsa</c> and <c>authorized_key</c>. + Defaults to the directory normally referred to as <c>~/.ssh</c>. + </p> + <p>See also the option + <seealso marker="#type-key_cb_common_option">key_cb</seealso> + for the general way to handle keys. + </p> + </desc> + </datatype> + + <datatype> + <name name="profile_common_option"/> + <desc> + <p>Used together with <c>ip-address</c> and <c>port</c> to + uniquely identify a ssh daemon. This can be useful in a + virtualized environment, where there can be more that one + server that has the same <c>ip-address</c> and + <c>port</c>. If this property is not explicitly set, it is + assumed that the the <c>ip-address</c> and <c>port</c> + uniquely identifies the SSH daemon. + </p> + </desc> + </datatype> + + <datatype> + <name name="max_idle_time_common_option"/> + <desc> + <p>Sets a time-out on a connection when no channels are active. Defaults to <c>infinity</c>.</p> + </desc> + </datatype> + + <datatype> + <name name="rekey_limit_common_option"/> + <desc> + <p>Sets a limit, in bytes, when rekeying is to be initiated. + Defaults to once per each GB and once per hour.</p> + </desc> + </datatype> + + <datatype> + <name name="key_cb_common_option"/> + <desc> + <p>Module implementing the behaviour + <seealso marker="ssh_client_key_api">ssh_client_key_api</seealso> and/or + <seealso marker="ssh_server_key_api">ssh_server_key_api</seealso>. + Can be used to + customize the handling of public keys. If callback options are provided + along with the module name, they are made available to the callback + module via the options passed to it under the key 'key_cb_private'. + </p> + <p>The <c>Opts</c> defaults to <c>[]</c> when only the <c>Module</c> is specified. + </p> + <p>The default value of this option is <c>{ssh_file, []}</c>. + </p> + <p>A call to the call-back function <c>F</c> will be</p> + <code> + Module:F(..., [{key_cb_private,Opts}|UserOptions]) + </code> + <p>where <c>...</c> are arguments to <c>F</c> as in + <seealso marker="ssh_client_key_api">ssh_client_key_api</seealso> and/or + <seealso marker="ssh_server_key_api">ssh_server_key_api</seealso>. + The <c>UserOptions</c> are the options given to <c>ssh:connect</c>, <c>ssh:shell</c> or <c>ssh:daemon</c>. + </p> + + </desc> + </datatype> + + <datatype> + <name name="disconnectfun_common_option"/> + <desc> + <p>Provides a fun to implement your own logging when the peer disconnects.</p> + </desc> + </datatype> + + <datatype> + <name name="unexpectedfun_common_option"/> + <desc> + <p>Provides a fun to implement your own logging or other action when an unexpected message arrives. + If the fun returns <c>report</c> the usual info report is issued but if <c>skip</c> is returned no + report is generated.</p> + </desc> + </datatype> + + <datatype> + <name name="ssh_msg_debug_fun_common_option"/> + <desc> + <p>Provide a fun to implement your own logging of the SSH message SSH_MSG_DEBUG. + The last three parameters are from the message, see + <url href="https://tools.ietf.org/html/rfc4253#section-11.3">RFC 4253, section 11.3</url>. + The <seealso marker="#type-connection_ref"><c>connection_ref()</c></seealso> is the reference + to the connection on which the message arrived. + The return value from the fun is not checked. + </p> + <p>The default behaviour is ignore the message. + To get a printout for each message with <c>AlwaysDisplay = true</c>, + use for example <c>{ssh_msg_debug_fun, fun(_,true,M,_)-> io:format("DEBUG: ~p~n", [M]) end}</c></p> + </desc> + </datatype> + + <datatype> + <name name="id_string_common_option"/> + <desc> + <p>The string the daemon will present to a connecting peer initially. + The default value is "Erlang/VSN" where VSN is the ssh application version number. + </p> + <p>The value <c>random</c> will cause a random string to be created at each connection attempt. + This is to make it a bit more difficult for a malicious peer to find the ssh software brand and version. + </p> + <p>The value <c>{random, Nmin, Nmax}</c> will make a random string with at least <c>Nmin</c> characters and + at most <c>Nmax</c> characters. + </p> + </desc> + </datatype> + + <datatype> + <name name="preferred_algorithms_common_option"/> + <name name="algs_list"/> + <name name="alg_entry"/> + <name name="kex_alg"/> + <name name="pubkey_alg"/> + <name name="cipher_alg"/> + <name name="mac_alg"/> + <name name="compression_alg"/> + <name name="double_algs"/> + + <desc> + <p>List of algorithms to use in the algorithm negotiation. The default <c>algs_list()</c> can + be obtained from <seealso marker="#default_algorithms/0">default_algorithms/0</seealso>. + </p> + <p>If an alg_entry() is missing in the algs_list(), the default value is used for that entry.</p> + <p>Here is an example of this option:</p> + <code> + {preferred_algorithms, + [{public_key,['ssh-rsa','ssh-dss']}, + {cipher,[{client2server,['aes128-ctr']}, + {server2client,['aes128-cbc','3des-cbc']}]}, + {mac,['hmac-sha2-256','hmac-sha1']}, + {compression,[none,zlib]} + ] + } + </code> + <p>The example specifies different algorithms in the two directions (client2server and server2client), + for cipher but specifies the same algorithms for mac and compression in both directions. + The kex (key exchange) is implicit but public_key is set explicitly.</p> + + <p>For background and more examples see the <seealso marker="configure_algos#introduction">User's Guide</seealso>.</p> + + <p>If an algorithm name occurs more than once in a list, the behaviour is undefined. The tags in the property lists + are also assumed to occur at most one time. + </p> + + <warning> + <p>Changing the values can make a connection less secure. Do not change unless you + know exactly what you are doing. If you do not understand the values then you + are not supposed to change them.</p> + </warning> + </desc> + </datatype> + + <datatype> + <name name="modify_algorithms_common_option"/> + <name name="modify_algs_list"/> + <desc> + <p>Modifies the list of algorithms to use in the algorithm negotiation. The modifications are + applied after the option <c>preferred_algorithms</c> (if existing) is applied.</p> + <p>The algoritm for modifications works like this:</p> + <list> <item> - <p>Module implementing the behaviour <seealso - marker="ssh_server_key_api">ssh_server_key_api</seealso>. Can be used to - customize the handling of public keys. If callback options are provided - along with the module name, they are made available to the callback - module via the options passed to it under the key 'key_cb_private'. + <p>Input is the <c>modify_algs_list()</c> and a set of algorithms <c>A</c> + obtained from the <c>preferred_algorithms</c> option if existing, or else from the + <seealso marker="ssh#default_algorithms-0">ssh:default_algorithms/0</seealso>. </p> </item> - - <tag><c>{profile, atom()}</c></tag> <item> - <p>Used together with <c>ip-address</c> and <c>port</c> to - uniquely identify a ssh daemon. This can be useful in a - virtualized environment, where there can be more that one - server that has the same <c>ip-address</c> and - <c>port</c>. If this property is not explicitly set, it is - assumed that the the <c>ip-address</c> and <c>port</c> - uniquely identifies the SSH daemon. - </p> + <p>The head of the <c>modify_algs_list()</c> modifies <c>A</c> giving the result <c>A'</c>.</p> + <p>The possible modifications are:</p> + <list> + <item> + <p>Append or prepend supported but not enabled algorithm(s) to the list of + algorithms. If the wanted algorithms already are in <c>A</c> they will first + be removed and then appended or prepended, + </p> + </item> + <item> + <p>Remove (rm) one or more algorithms from <c>A</c>. + </p> + </item> + </list> </item> - - <tag><c><![CDATA[{fd, file_descriptor()}]]></c></tag> <item> - <p>Allows an existing file-descriptor to be used - (passed on to the transport protocol).</p></item> - <tag><c><![CDATA[{failfun, fun(User::string(), - PeerAddress::ip_address(), Reason::term()) -> _}]]></c></tag> - <item> - <p>Provides a fun to implement your own logging when a user fails to authenticate.</p> + <p>Repeat the modification step with the tail of <c>modify_algs_list()</c> and the resulting + <c>A'</c>. + </p> </item> - <tag><c><![CDATA[{connectfun, fun(User::string(), PeerAddress::ip_address(), - Method::string()) ->_}]]></c></tag> - <item> - <p>Provides a fun to implement your own logging when a user authenticates to the server.</p> + </list> + <p>If an unsupported algorithm is in the <c>modify_algs_list()</c>, it will be silently ignored</p> + <p>If there are more than one modify_algorithms options, the result is undefined.</p> + <p>Here is an example of this option:</p> + <code> + {modify_algorithms, + [{prepend, [{kex, ['diffie-hellman-group1-sha1']}], + {rm, [{compression, [none]}]} + ] + } + </code> + <p>The example specifies that:</p> + <list> + <item><p>the old key exchange algorithm 'diffie-hellman-group1-sha1' should be + the main alternative. It will be the main alternative since it is prepened to the list</p> </item> - <tag><c><![CDATA[{disconnectfun, fun(Reason:term()) -> _}]]></c></tag> - <item> - <p>Provides a fun to implement your own logging when a user disconnects from the server.</p> + <item><p>The compression algorithm none (= no compression) is removed so compression is enforced</p> </item> + </list> + <p>For background and more examples see the <seealso marker="configure_algos#introduction">User's Guide</seealso>.</p> + </desc> + </datatype> - <tag><c><![CDATA[{unexpectedfun, fun(Message:term(), Peer) -> report | skip }]]></c></tag> - <item> - <p>Provides a fun to implement your own logging or other action when an unexpected message arrives. - If the fun returns <c>report</c> the usual info report is issued but if <c>skip</c> is returned no - report is generated.</p> - <p><c>Peer</c> is in the format of <c>{Host,Port}</c>.</p> - </item> - <tag><c><![CDATA[{idle_time, integer()}]]></c></tag> - <item> - <p>Sets a time-out on a connection when no channels are active. - Defaults to <c>infinity</c>.</p> - </item> + <datatype> + <name name="inet_common_option"/> + <desc> + <p>IP version to use when the host address is specified as <c>any</c>.</p> + </desc> + </datatype> - <tag><c><![CDATA[{ssh_msg_debug_fun, fun(ConnectionRef::ssh_connection_ref(), AlwaysDisplay::boolean(), Msg::binary(), LanguageTag::binary()) -> _}]]></c></tag> - <item> - <p>Provide a fun to implement your own logging of the SSH message SSH_MSG_DEBUG. The last three parameters are from the message, see RFC4253, section 11.3. The <c>ConnectionRef</c> is the reference to the connection on which the message arrived. The return value from the fun is not checked.</p> - <p>The default behaviour is ignore the message. - To get a printout for each message with <c>AlwaysDisplay = true</c>, use for example <c>{ssh_msg_debug_fun, fun(_,true,M,_)-> io:format("DEBUG: ~p~n", [M]) end}</c></p> - </item> + <datatype> + <name name="auth_methods_common_option"/> + <desc> + <p>Comma-separated string that determines which authentication methods that the client shall + support and in which order they are tried. Defaults to <c>"publickey,keyboard-interactive,password"</c> + </p> + <p>Note that the client is free to use any order and to exclude methods. + </p> + </desc> + </datatype> - </taglist> - </desc> - </func> + <datatype> + <name name="fd_common_option"/> + <desc> + <p>Allows an existing file-descriptor to be used (passed on to the transport protocol).</p> + </desc> + </datatype> - <func> - <name>daemon_info(Daemon) -> {ok, [DaemonInfo]} | {error,Error}</name> - <fsummary>Get info about a daemon</fsummary> - <type> - <v>DaemonInfo = {port,Port::pos_integer()} | {listen_address, any|ip_address()} | {profile,atom()}</v> - <v>Port = integer()</v> - <v>Error = bad_daemon_ref</v> - </type> + <!--................................................................--> + <datatype_title>Other data types</datatype_title> + + <datatype> + <name name="host"/> + <desc> + </desc> + </datatype> + + <datatype> + <name name="ip_port"/> + <desc> + </desc> + </datatype> + + <datatype> + <name name="mod_args"/> + <desc> + </desc> + </datatype> + + <datatype> + <name name="mod_fun_args"/> + <desc> + </desc> + </datatype> + + <datatype> + <name name="open_socket"/> + <desc> + <p>The socket is supposed to be result of a <seealso marker="kernel:gen_tcp#connect-3">gen_tcp:connect</seealso> + or a <seealso marker="kernel:gen_tcp#accept-1">gen_tcp:accept</seealso>. The socket must be in passive + mode (that is, opened with the option <c>{active,false})</c>. + </p> + </desc> + </datatype> + + <datatype> + <name name="daemon_ref"/> + <desc> + <p>Opaque data type representing a daemon.</p> + <p>Returned by the functions <seealso marker="ssh#daemon-1"><c>daemon/1,2,3</c></seealso>.</p> + </desc> + </datatype> + + <datatype> + <name>connection_ref()</name> + <desc> + <p>Opaque data type representing a connection between a client and a server (daemon).</p> + <p>Returned by the functions + <seealso marker="ssh#connect-3"><c>connect/2,3,4</c></seealso> and + <seealso marker="ssh_sftp#start_channel-2"><c>ssh_sftp:start_channel/2,3</c></seealso>. + </p> + </desc> + </datatype> + + <datatype> + <name name="channel_id"/> + <desc> + <p>Opaque data type representing a channel inside a connection.</p> + <p>Returned by the functions + <seealso marker="ssh_connection#session_channel/2">ssh_connection:session_channel/2,4</seealso>. + </p> + </desc> + </datatype> + + + <datatype> + <name>opaque_client_options</name> + <name>opaque_daemon_options</name> + <name>opaque_common_options</name> <desc> - <p>Returns a key-value list with information about the daemon. For now, only the listening port is returned. This is intended for the case the daemon is started with the port set to 0.</p> + <marker id="type-opaque_client_options"/> + <marker id="type-opaque_daemon_options"/> + <marker id="type-opaque_common_options"/> + <p>Opaque types that define experimental options that are not to be used in products.</p> </desc> + </datatype> + </datatypes> + +<!-- + ================================================================ + = Function definitions = + ================================================================ +--> + + <funcs> + +<!-- CLOSE/1 --> + <func> + <name name="close" arity="1"/> + <fsummary>Closes an SSH connection.</fsummary> + <desc><p>Closes an SSH connection.</p></desc> </func> + +<!-- CONNECT/2 etc --> + <func> + <name>connect(Host, Port, Options) -> Result </name> + <name>connect(Host, Port, Options, NegotiationTimeout) -> Result </name> + <name>connect(TcpSocket, Options) -> Result</name> + <name>connect(TcpSocket, Options, NegotiationTimeout) -> Result</name> + <fsummary>Connects to an SSH server.</fsummary> + <type> + <v>Host = <seealso marker="#type-host">host()</seealso></v> + <v>Port = <seealso marker="kernel:inet#type-port_number">inet:port_number()</seealso></v> + <v>Options = <seealso marker="#type-client_options">client_options()</seealso></v> + <v>TcpSocket = <seealso marker="#type-open_socket">open_socket()</seealso></v> + <v>NegotiationTimeout = timeout()</v> + <v>Result = {ok, <seealso marker="#type-connection_ref">connection_ref()</seealso>} | {error, term()}</v> + </type> + <desc> + <p>Connects to an SSH server at the <c>Host</c> on <c>Port</c>. + </p> + <p>As an alternative, an already open TCP socket could be passed to the function in <c>TcpSocket</c>. + The SSH initiation and negotiation will be initiated on that one with the SSH that should be at the + other end. + </p> + <p>No channel is started. This is done by calling <seealso marker="ssh_connection#session_channel/2"> + ssh_connection:session_channel/[2, 4]</seealso>. + </p> + <p>The <c>NegotiationTimeout</c> is in milli-seconds. The default value is <c>infinity</c>. + For connection timeout, use the option + <seealso marker="#type-connect_timeout_client_option"><c>connect_timeout</c></seealso>. + </p> + </desc> + </func> + +<!-- CONNECTION_INFO/1, CONNECTION_INFO/2 --> + <func> + <name name="connection_info" arity="2"/> + <fsummary>Retrieves information about a connection.</fsummary> + <desc> + <p>Retrieves information about a connection. The list <c>Keys</c> defines which information that + is returned.</p> + </desc> + </func> + +<!-- DEAMON/1,2,3 --> + <func> + <name>daemon(Port | TcpSocket) -> Result</name> + <name>daemon(Port | TcpSocket, Options) -> Result</name> + <name>daemon(HostAddress, Port, Options) -> Result</name> + <fsummary>Starts a server listening for SSH connections.</fsummary> + <type> + <v>Port = integer()</v> + <v>TcpSocket = <seealso marker="#type-open_socket">open_socket()</seealso></v> + <v>Options = <seealso marker="#type-daemon_options">daemon_options()</seealso></v> + <v>HostAddress = <seealso marker="#type-host">host()</seealso> | any</v> + <v>Result = {ok, <seealso marker="#type-daemon_ref">daemon_ref()</seealso>} | {error, atom()}</v> + </type> + <desc> + <p>Starts a server listening for SSH connections on the given port. If the <c>Port</c> is 0, + a random free port is selected. See <seealso marker="#daemon_info/1">daemon_info/1</seealso> + about how to find the selected port number. + </p> + <p>As an alternative, an already open TCP socket could be passed to the function in <c>TcpSocket</c>. + The SSH initiation and negotiation will be initiated on that one when an SSH starts at the other end + of the TCP socket. + </p> + <p>For a description of the options, see <seealso marker="#type-daemon_options">Daemon Options</seealso>. + </p> + <p>Please note that by historical reasons both the <c>HostAddress</c> argument and the + <seealso marker="kernel:gen_tcp#type-connect_option">gen_tcp connect_option() <c>{ip,Address}</c></seealso> + set the listening address. This is a source of possible inconsistent settings. + </p> + <p>The rules for handling the two address passing options are:</p> + <list> + <item>if <c>HostAddress</c> is an IP-address, that IP-address is the listening address. + An 'ip'-option will be discarded if present.</item> + + <item>if <c>HostAddress</c> is the atom <c>loopback</c>, the listening address + is <c>loopback</c> and an loopback address will be choosen by the underlying layers. + An 'ip'-option will be discarded if present.</item> + + <item>if <c>HostAddress</c> is the atom <c>any</c> and no 'ip'-option is present, the listening address is + <c>any</c> and the socket will listen to all addresses</item> + + <item>if <c>HostAddress</c> is <c>any</c> and an 'ip'-option is present, the listening address is + set to the value of the 'ip'-option</item> + </list> + </desc> + </func> +<!-- DAEMON_INFO/1 --> + <func> + <name name="daemon_info" arity="1"/> + <fsummary>Get info about a daemon</fsummary> + <desc> + <p>Returns a key-value list with information about the daemon.</p> + </desc> + </func> + + +<!-- DEFAULT_ALGORITHMS/0 --> <func> - <name>default_algorithms() -> algs_list()</name> + <name name="default_algorithms" arity="0"/> <fsummary>Get a list declaring the supported algorithms</fsummary> <desc> <p>Returns a key-value list, where the keys are the different types of algorithms and the values are the - algorithms themselves. An example:</p> - <code> -20> ssh:default_algorithms(). -[{kex,['diffie-hellman-group1-sha1']}, - {public_key,['ssh-rsa','ssh-dss']}, - {cipher,[{client2server,['aes128-ctr','aes128-cbc','3des-cbc']}, - {server2client,['aes128-ctr','aes128-cbc','3des-cbc']}]}, - {mac,[{client2server,['hmac-sha2-256','hmac-sha1']}, - {server2client,['hmac-sha2-256','hmac-sha1']}]}, - {compression,[{client2server,[none,zlib]}, - {server2client,[none,zlib]}]}] -21> -</code> + algorithms themselves.</p> + <p>See the <seealso marker="configure_algos#example_default_algorithms">User's Guide</seealso> for + an example.</p> </desc> </func> +<!-- SHELL/1,2,3 --> <func> - <name>shell(Host) -> </name> - <name>shell(Host, Option) -> </name> - <name>shell(Host, Port, Option) -> </name> - <name>shell(TcpSocket) -> _</name> - <fsummary>Starts an interactive shell over an SSH server.</fsummary> + <name>shell(Host | TcpSocket) -> Result </name> + <name>shell(Host | TcpSocket, Options) -> Result </name> + <name>shell(Host, Port, Options) -> Result </name> + <fsummary>Starts an interactive shell on a remote SSH server.</fsummary> <type> - <v>Host = string()</v> - <v>Port = integer()</v> - <v>Options - see ssh:connect/3</v> - <v>TcpSocket = port()</v> - <d>The socket is supposed to be from <seealso marker="kernel:gen_tcp#connect-3">gen_tcp:connect</seealso> or <seealso marker="kernel:gen_tcp#accept-1">gen_tcp:accept</seealso> with option <c>{active,false}</c></d> + <v>Host = <seealso marker="#type-host">host()</seealso></v> + <v>TcpSocket = <seealso marker="#type-open_socket">open_socket()</seealso></v> + <v>Port = <seealso marker="kernel:inet#type-port_number">inet:port_number()</seealso></v> + <v>Options = <seealso marker="#type-client_options">client_options()</seealso></v> + <v>Result = ok | {error, Reason::term()}</v> </type> <desc> - <p>Starts an interactive shell over an SSH server on the - given <c>Host</c>. The function waits for user input, - and does not return until the remote shell is ended (that is, + <p>Connects to an SSH server at <c>Host</c> and <c>Port</c> (defaults to 22) and starts an + interactive shell on that remote host. + </p> + <p>As an alternative, an already open TCP socket could be passed to the function in <c>TcpSocket</c>. + The SSH initiation and negotiation will be initiated on that one and finaly a shell will be started + on the host at the other end of the TCP socket. + </p> + <p>For a description of the options, see <seealso marker="#type-client_options">Client Options</seealso>.</p> + <p>The function waits for user input, and does not return until the remote shell is ended (that is, exit from the shell). </p> </desc> </func> <func> - <name>start() -> </name> - <name>start(Type) -> ok | {error, Reason}</name> + <name name="start" arity="0"/> + <name name="start" arity="1"/> <fsummary>Starts the SSH application.</fsummary> - <type> - <v>Type = permanent | transient | temporary</v> - <v>Reason = term() </v> - </type> <desc> <p>Utility function that starts the applications <c>crypto</c>, <c>public_key</c>, and <c>ssh</c>. Default type is <c>temporary</c>. @@ -985,11 +1218,8 @@ </func> <func> - <name>stop() -> ok | {error, Reason}</name> + <name name="stop" arity="0"/> <fsummary>Stops the <c>ssh</c> application.</fsummary> - <type> - <v>Reason = term()</v> - </type> <desc> <p>Stops the <c>ssh</c> application. For more information, see the <seealso marker="kernel:application">application(3)</seealso> @@ -998,34 +1228,22 @@ </func> <func> - <name>stop_daemon(DaemonRef) -> </name> - <name>stop_daemon(Address, Port) -> ok </name> - <fsummary>Stops the listener and all connections started by - the listener.</fsummary> - <type> - <v>DaemonRef = ssh_daemon_ref()</v> - <v>Address = ip_address()</v> - <v>Port = integer()</v> - </type> + <name name="stop_daemon" arity="1"/> + <name name="stop_daemon" arity="2"/> + <name name="stop_daemon" arity="3"/> + <fsummary>Stops the listener and all connections started by the listener.</fsummary> <desc> - <p>Stops the listener and all connections started by - the listener.</p> + <p>Stops the listener and all connections started by the listener.</p> </desc> </func> <func> - <name>stop_listener(DaemonRef) -> </name> - <name>stop_listener(Address, Port) -> ok </name> - <fsummary>Stops the listener, but leaves existing connections started - by the listener operational.</fsummary> - <type> - <v>DaemonRef = ssh_daemon_ref()</v> - <v>Address = ip_address()</v> - <v>Port = integer()</v> - </type> + <name name="stop_listener" arity="1"/> + <name name="stop_listener" arity="2"/> + <name name="stop_listener" arity="3"/> + <fsummary>Stops the listener, but leaves existing connections started by the listener operational.</fsummary> <desc> - <p>Stops the listener, but leaves existing connections started - by the listener operational.</p> + <p>Stops the listener, but leaves existing connections started by the listener operational.</p> </desc> </func> diff --git a/lib/ssh/doc/src/ssh_app.xml b/lib/ssh/doc/src/ssh_app.xml index 1cbbdfcf38..6d180a5272 100644 --- a/lib/ssh/doc/src/ssh_app.xml +++ b/lib/ssh/doc/src/ssh_app.xml @@ -330,10 +330,11 @@ <p/> </item> - <item><url href="https://tools.ietf.org/html/draft-ietf-curdle-rsa-sha2">Draft-ietf-curdle-rsa-sha2 (work in progress)</url>, Use of RSA Keys with SHA-2 256 and 512 in Secure Shell (SSH). + <item><url href="https://tools.ietf.org/html/rfc8332">RFC 8332</url>, Use of RSA Keys with SHA-256 and SHA-512 in the Secure Shell (SSH) Protocol. </item> - <item><url href="https://tools.ietf.org/html/draft-ietf-curdle-ssh-ext-info">Draft-ietf-curdle-ssh-ext-info (work in progress)</url>, Extension Negotiation in Secure Shell (SSH). + <item><marker id="supported-ext-info"/> + <url href="https://tools.ietf.org/html/rfc8308">RFC 8308</url>, Extension Negotiation in the Secure Shell (SSH) Protocol. <p>Implemented are:</p> <list type="bulleted"> <item>The Extension Negotiation Mechanism</item> diff --git a/lib/ssh/doc/src/ssh_channel.xml b/lib/ssh/doc/src/ssh_client_channel.xml index 7b598494f7..eed49beffa 100644 --- a/lib/ssh/doc/src/ssh_channel.xml +++ b/lib/ssh/doc/src/ssh_client_channel.xml @@ -23,21 +23,27 @@ The Initial Developer of the Original Code is Ericsson AB. </legalnotice> - <title>ssh_channel</title> + <title>ssh_client_channel</title> <prepared></prepared> <docno></docno> <date></date> <rev></rev> </header> - <module>ssh_channel</module> - <modulesummary>-behaviour(ssh_channel). + <module>ssh_client_channel</module> + <modulesummary>-behaviour(ssh_client_channel). (Replaces ssh_channel) </modulesummary> <description> + <note> + <p>This module replaces ssh_channel.</p> + <p>The old module is still available for compatibility, but should not be used for new programs. + The old module will not be maintained except for some error corrections + </p> + </note> <p>SSH services (clients and servers) are implemented as channels that are multiplexed over an SSH connection and communicates over the <url href="http://www.ietf.org/rfc/rfc4254.txt"> SSH Connection Protocol</url>. This module provides a callback API - that takes care of generic channel aspects, such as flow control + that takes care of generic channel aspects for clients, such as flow control and close messages. It lets the callback functions take care of the service (application) specific parts. This behavior also ensures that the channel process honors the principal of an OTP-process so @@ -46,41 +52,19 @@ the <c>ssh</c> applications supervisor tree. </p> - <note><p>When implementing an <c>ssh</c> subsystem, use - <c>-behaviour(ssh_daemon_channel)</c> instead of <c>-behaviour(ssh_channel)</c>. - The reason is that the only relevant callback functions for subsystems are - <c>init/1</c>, <c>handle_ssh_msg/2</c>, <c>handle_msg/2</c>, and <c>terminate/2</c>. - So, the <c>ssh_daemon_channel</c> behaviour is a limited version of the - <c>ssh_channel</c> behaviour. - </p></note> - </description> + <note><p>When implementing a <c>ssh</c> subsystem for daemons, use + <seealso marker="ssh_server_channel">-behaviour(ssh_server_channel)</seealso> (Replaces ssh_daemon_channel) + instead. + </p> + </note> - <section> - <title>DATA TYPES</title> + <dont> + <p>Functions in this module are not supposed to be called outside a module implementing this + behaviour! + </p> + </dont> - <p>Type definitions that are used more than once in this module, - or abstractions to indicate the intended use of the data - type, or both:</p> - - <taglist> - <tag><c>boolean() =</c></tag> - <item><p><c>true | false</c></p></item> - <tag><c>string() =</c></tag> - <item><p>list of ASCII characters</p></item> - <tag><c>timeout() =</c></tag> - <item><p><c>infinity | integer()</c> in milliseconds</p></item> - <tag><c>ssh_connection_ref() =</c></tag> - <item><p>opaque() -as returned by - <c>ssh:connect/3</c> or sent to an SSH channel process</p></item> - <tag><c>ssh_channel_id() =</c></tag> - <item><p><c>integer()</c></p></item> - <tag><c>ssh_data_type_code() =</c></tag> - <item><p><c>1</c> ("stderr") | <c>0</c> ("normal") are - the valid values, - see <url href="http://www.ietf.org/rfc/rfc4254.txt">RFC 4254</url> - Section 5.2</p></item> - </taglist> - </section> + </description> <funcs> <func> @@ -89,7 +73,7 @@ <fsummary>Makes a synchronous call to a channel.</fsummary> <type> <v>ChannelRef = pid() </v> - <d>As returned by <seealso marker = "#start_link-4">ssh_channel:start_link/4</seealso></d> + <d>As returned by <seealso marker = "#start_link-4">start_link/4</seealso></d> <v>Msg = term()</v> <v>Timeout = timeout()</v> <v>Reply = term()</v> @@ -113,7 +97,7 @@ ChannelRef and returns ok.</fsummary> <type> <v>ChannelRef = pid()</v> - <d>As returned by <seealso marker = "#start_link-4">ssh_channel:start_link/4</seealso></d> + <d>As returned by <seealso marker = "#start_link-4">start_link/4</seealso></d> <v>Msg = term()</v> </type> <desc> @@ -126,29 +110,29 @@ </desc> </func> - <func> + <func> <name>enter_loop(State) -> _ </name> - <fsummary>Makes an existing process an ssh_channel process.</fsummary> + <fsummary>Makes an existing process an ssh_client_channel (replaces ssh_channel) process.</fsummary> <type> <v>State = term()</v> - <d>as returned by <seealso marker = "#init-1">ssh_channel:init/1</seealso></d> + <d>as returned by <seealso marker = "#init-1">init/1</seealso></d> </type> <desc> - <p>Makes an existing process an <c>ssh_channel</c> + <p>Makes an existing process an <c>ssh_client_channel</c> (replaces ssh_channel) process. Does not return, instead the calling process - enters the <c>ssh_channel</c> process receive loop and become an - <c>ssh_channel process</c>. The process must have been started using + enters the <c>ssh_client_channel</c> (replaces ssh_channel) process receive loop and become an + <c>ssh_client_channel</c> process. The process must have been started using one of the start functions in <c>proc_lib</c>, see the <seealso marker="stdlib:proc_lib">proc_lib(3)</seealso> manual page in STDLIB. The user is responsible for any initialization of the process - and must call <seealso marker = "#init-1">ssh_channel:init/1</seealso>. + and must call <seealso marker = "#init-1">init/1</seealso>. </p> </desc> </func> <func> <name>init(Options) -> {ok, State} | {ok, State, Timeout} | {stop, Reason} </name> - <fsummary>Initiates an <c>ssh_channel</c> process.</fsummary> + <fsummary>Initiates an <c>ssh_client_channel</c> process.</fsummary> <type> <v>Options = [{Option, Value}]</v> <v>State = term()</v> @@ -160,18 +144,21 @@ The following options must be present: </p> <taglist> - <tag><c><![CDATA[{channel_cb, atom()}]]></c></tag> + <tag><c>{channel_cb, atom()}</c></tag> <item><p>The module that implements the channel behaviour.</p></item> - <tag><c><![CDATA[{init_args(), list()}]]></c></tag> + <tag><c>{init_args(), list()}</c></tag> <item><p>The list of arguments to the <c>init</c> function of the callback module.</p></item> - <tag><c><![CDATA[{cm, connection_ref()}]]></c></tag> - <item><p>Reference to the <c>ssh</c> connection as returned by <seealso - marker="ssh#connect-3">ssh:connect/3</seealso></p></item> + <tag><c>{cm, ssh:connection_ref()}</c></tag> + <item><p>Reference to the <c>ssh</c> connection as returned by + <seealso marker="ssh#connect-3">ssh:connect/3</seealso>. + </p></item> - <tag><c><![CDATA[{channel_id, channel_id()}]]></c></tag> - <item><p>Id of the <c>ssh</c> channel.</p></item> + <tag><c>{channel_id, ssh:channel_id()}</c></tag> + <item><p>Id of the <c>ssh</c> channel as returned by + <seealso marker="ssh_connection#session_channel/2">ssh_connection:session_channel/2,4</seealso>. + </p></item> </taglist> @@ -179,8 +166,8 @@ user. The user only needs to call if the channel process needs to be started with help of <c>proc_lib</c> instead of calling - <c>ssh_channel:start/4</c> or - <c>ssh_channel:start_link/4</c>.</p> + <c>start/4</c> or + <c>start_link/4</c>.</p> </note> </desc> </func> @@ -201,26 +188,31 @@ the callback function <c>handle_call/3</c>. <c>Reply</c> is an arbitrary term, which is given back to the client as the return value of - <seealso marker="#call-2">ssh_channel:call/[2,3].</seealso></p> + <seealso marker="#call-2">call/[2,3].</seealso></p> </desc> </func> - + <func> <name>start(SshConnection, ChannelId, ChannelCb, CbInitArgs) -> </name> <name>start_link(SshConnection, ChannelId, ChannelCb, CbInitArgs) -> {ok, ChannelRef} | {error, Reason}</name> <fsummary>Starts a process that handles an SSH channel.</fsummary> <type> - <v>SshConnection = ssh_connection_ref()</v> - <v>ChannelId = ssh_channel_id()</v> + <v>SshConnection = ssh:connection_ref()</v> + <d>As returned by <seealso marker="ssh#connect-3">ssh:connect/3</seealso></d> + + <v>ChannelId = <seealso marker="ssh#type-channel_id">ssh:channel_id()</seealso></v> <d>As returned by <seealso marker ="ssh_connection#session_channel/2"> ssh_connection:session_channel/[2,4]</seealso>.</d> + <v>ChannelCb = atom()</v> <d>Name of the module implementing the service-specific parts of the channel.</d> + <v>CbInitArgs = [term()]</v> <d>Argument list for the <c>init</c> function in the callback module.</d> + <v>ChannelRef = pid()</v> </type> <desc> @@ -235,13 +227,19 @@ </funcs> <section> + <title>Callback Functions</title> + <p> + The following functions are to be exported from a + <c>ssh_client_channel</c> callback module. + </p> <marker id="cb_timeouts"></marker> - <title>CALLBACK TIME-OUTS</title> - - <p>The time-out values that can be returned by the callback functions - have the same semantics as in a <seealso marker="stdlib:gen_server">gen_server</seealso>. - If the time-out occurs, <seealso marker="#Module:handle_msg-2">handle_msg/2</seealso> - is called as <c>handle_msg(timeout, State)</c>.</p> + <section> + <title>Callback timeouts</title> + <p>The timeout values that can be returned by the callback functions + have the same semantics as in a <seealso marker="stdlib:gen_server">gen_server</seealso>. + If the time-out occurs, <seealso marker="#Module:handle_msg-2">handle_msg/2</seealso> + is called as <c>handle_msg(timeout, State)</c>.</p> + </section> </section> <funcs> @@ -295,7 +293,7 @@ initial channel state if the initializations succeed.</fsummary> <type> <v>Args = term()</v> - <d>Last argument to <c>ssh_channel:start_link/4</c>.</d> + <d>Last argument to <c>start_link/4</c>.</d> <v>State = term()</v> <v>Reason = term()</v> </type> @@ -304,41 +302,41 @@ state if the initializations succeed. </p> <p>For more detailed information on time-outs, see Section - <seealso marker="#cb_timeouts">CALLBACK TIME-OUTS</seealso>. </p> + <seealso marker="#cb_timeouts">Callback timeouts</seealso>. </p> </desc> </func> <func> <name>Module:handle_call(Msg, From, State) -> Result</name> <fsummary>Handles messages sent by calling - <c>ssh_channel:call/[2,3]</c>.</fsummary> + <c>call/[2,3]</c>.</fsummary> <type> <v>Msg = term()</v> <v>From = opaque()</v> <d>Is to be used as argument to - <seealso marker="#reply-2">ssh_channel:reply/2</seealso></d> + <seealso marker="#reply-2">reply/2</seealso></d> <v>State = term()</v> <v>Result = {reply, Reply, NewState} | {reply, Reply, NewState, timeout()} | {noreply, NewState} | {noreply , NewState, timeout()} | {stop, Reason, Reply, NewState} | {stop, Reason, NewState} </v> <v>Reply = term()</v> - <d>Will be the return value of <seealso marker="#call-2">ssh_channel:call/[2,3]</seealso></d> + <d>Will be the return value of <seealso marker="#call-2">call/[2,3]</seealso></d> <v>NewState = term()</v> <v>Reason = term()</v> </type> <desc> <p>Handles messages sent by calling - <seealso marker="#call-2">ssh_channel:call/[2,3]</seealso> + <seealso marker="#call-2">call/[2,3]</seealso> </p> <p>For more detailed information on time-outs,, see Section - <seealso marker="#cb_timeouts">CALLBACK TIME-OUTS</seealso>.</p> + <seealso marker="#cb_timeouts">Callback timeouts</seealso>.</p> </desc> </func> <func> <name>Module:handle_cast(Msg, State) -> Result</name> <fsummary>Handles messages sent by calling - <c>ssh_channel:cact/2</c>.</fsummary> + <c>cast/2</c>.</fsummary> <type> <v>Msg = term()</v> <v>State = term()</v> @@ -349,10 +347,10 @@ </type> <desc> <p>Handles messages sent by calling - <c>ssh_channel:cast/2</c>. + <c>cast/2</c>. </p> <p>For more detailed information on time-outs, see Section - <seealso marker="#cb_timeouts">CALLBACK TIME-OUTS</seealso>.</p> + <seealso marker="#cb_timeouts">Callback timeouts</seealso>.</p> </desc> </func> @@ -364,7 +362,7 @@ call, or cast messages sent to the channel.</fsummary> <type> <v>Msg = timeout | term()</v> - <v>ChannelId = ssh_channel_id()</v> + <v>ChannelId = <seealso marker="ssh#type-channel_id">ssh:channel_id()</seealso></v> <v>State = term() </v> </type> <desc> @@ -376,11 +374,10 @@ function and all channels are to handle the following message.</p> <taglist> - <tag><c><![CDATA[{ssh_channel_up, ssh_channel_id(), - ssh_connection_ref()}]]></c></tag> + <tag><c>{ssh_channel_up, ssh:channel_id(), ssh:connection_ref()}</c></tag> <item><p>This is the first message that the channel receives. It is sent just before the <seealso - marker="#init-1">ssh_channel:init/1</seealso> function + marker="#init-1">init/1</seealso> function returns successfully. This is especially useful if the server wants to send a message to the client without first receiving a message from it. If the message is not @@ -397,7 +394,7 @@ <fsummary>Handles <c>ssh</c> connection protocol messages.</fsummary> <type> <v>Msg = ssh_connection:event()</v> - <v>ChannelId = ssh_channel_id()</v> + <v>ChannelId = <seealso marker="ssh#type-channel_id">ssh:channel_id()</seealso></v> <v>State = term()</v> </type> <desc> @@ -407,10 +404,10 @@ </p> <p>The following message is taken care of by the - <c>ssh_channel</c> behavior.</p> + <c>ssh_client_channel</c> behavior.</p> <taglist> - <tag><c><![CDATA[{closed, ssh_channel_id()}]]></c></tag> + <tag><c>{closed, ssh:channel_id()}</c></tag> <item><p>The channel behavior sends a close message to the other side, if such a message has not already been sent. Then it terminates the channel with reason <c>normal</c>.</p></item> diff --git a/lib/ssh/doc/src/ssh_client_key_api.xml b/lib/ssh/doc/src/ssh_client_key_api.xml index 98a1676ca4..9fc54341ed 100644 --- a/lib/ssh/doc/src/ssh_client_key_api.xml +++ b/lib/ssh/doc/src/ssh_client_key_api.xml @@ -41,7 +41,7 @@ see the <seealso marker="SSH_app"> ssh(6)</seealso> application manual.</p> </description> - <section> + <!-- section> <title>DATA TYPES</title> <p>Type definitions that are used more than once in this module, @@ -68,23 +68,34 @@ | 'rsa-sha2-256' | 'rsa-sha2-384' | 'rsa-sha2-512' | 'ecdsa-sha2-nistp256' | 'ecdsa-sha2-nistp384' | 'ecdsa-sha2-nistp521' </c></p></item> </taglist> - </section> + </section --> + + <datatypes> + <datatype> + <name name="client_key_cb_options"/> + <desc> + <p>Options provided to <seealso marker="ssh#connect-3">ssh:connect/[3,4]</seealso>. + </p> + <p>The option list given in the + <seealso marker="ssh#type-key_cb_common_option"><c>key_cb</c></seealso> + option is available with the key <c>key_cb_private</c>. + </p> + </desc> + </datatype> + </datatypes> <funcs> <func> - <name>Module:add_host_key(HostNames, Key, ConnectOptions) -> ok | {error, Reason}</name> + <name>Module:add_host_key(HostNames, PublicHostKey, ConnectOptions) -> ok | {error, Reason}</name> <fsummary>Adds a host key to the set of trusted host keys.</fsummary> <type> - <v>HostNames = string()</v> - <d>Description of the host that owns the <c>PublicKey</c>.</d> + <v>HostNames = string()</v> + <d>Description of the host that owns the <c>PublicHostKey</c>.</d> - <v>Key = public_key()</v> - <d>Normally an RSA, DSA or ECDSA public key, but handling of other public keys can be added.</d> + <v>PublicHostKey = <seealso marker="public_key:public_key#type-public_key">public_key:public_key()</seealso></v> + <d>Of ECDSA keys, only the Normally an RSA, DSA or ECDSA public key, but handling of other public keys can be added.</d> - <v>ConnectOptions = proplists:proplist()</v> - <d>Options provided to <seealso marker="ssh#connect-3">ssh:connect/[3,4]</seealso>. The option list given in - the <c>key_cb</c> option is available with the key <c>key_cb_private</c>.</d> - <v>Reason = term().</v> + <v>ConnectOptions = <seealso marker="#type-client_key_cb_options">client_key_cb_options()</seealso></v> </type> <desc> <p>Adds a host key to the set of trusted host keys.</p> @@ -95,18 +106,16 @@ <name>Module:is_host_key(Key, Host, Algorithm, ConnectOptions) -> Result</name> <fsummary>Checks if a host key is trusted.</fsummary> <type> - <v>Key = public_key() </v> + <v>Key = <seealso marker="public_key:public_key#type-public_key">public_key:public_key()</seealso></v> <d>Normally an RSA, DSA or ECDSA public key, but handling of other public keys can be added.</d> <v>Host = string()</v> <d>Description of the host.</d> - <v>Algorithm = public_key_algorithm()</v> + <v>Algorithm = <seealso marker="ssh#type-pubkey_alg">ssh:pubkey_alg()</seealso></v> <d>Host key algorithm.</d> - <v>ConnectOptions = proplists:proplist() </v> - <d>Options provided to <seealso marker="ssh#connect-3">ssh:connect/[3,4]</seealso>. The option list given in - the <c>key_cb</c> option is available with the key <c>key_cb_private</c>.</d> + <v>ConnectOptions = <seealso marker="#type-client_key_cb_options">client_key_cb_options()</seealso></v> <v>Result = boolean()</v> </type> @@ -120,14 +129,12 @@ {ok, PrivateKey} | {error, Reason}</name> <fsummary>Fetches the users <em>public key</em> matching the <c>Algorithm</c>.</fsummary> <type> - <v>Algorithm = public_key_algorithm()</v> + <v>Algorithm = <seealso marker="ssh#type-pubkey_alg">ssh:pubkey_alg()</seealso></v> <d>Host key algorithm.</d> - <v>ConnectOptions = proplists:proplist()</v> - <d>Options provided to <seealso marker="ssh#connect-3">ssh:connect/[3,4]</seealso>. The option list given in - the <c>key_cb</c> option is available with the key <c>key_cb_private</c>.</d> + <v>ConnectOptions = <seealso marker="#type-client_key_cb_options">client_key_cb_options()</seealso></v> - <v>PrivateKey = private_key()</v> + <v>PrivateKey = <seealso marker="public_key:public_key#type-private_key">public_key:private_key()</seealso></v> <d>Private key of the user matching the <c>Algorithm</c>.</d> <v>Reason = term()</v> diff --git a/lib/ssh/doc/src/ssh_connection.xml b/lib/ssh/doc/src/ssh_connection.xml index 150d46a9a2..821dfef93d 100644 --- a/lib/ssh/doc/src/ssh_connection.xml +++ b/lib/ssh/doc/src/ssh_connection.xml @@ -43,10 +43,10 @@ which are received as messages by the remote channel. If the receiving channel is an Erlang process, the messages have the format - <c><![CDATA[{ssh_cm, ssh_connection_ref(), ssh_event_msg()}]]></c>. - If the <seealso marker="ssh_channel">ssh_channel</seealso> behavior is used to + <c><![CDATA[{ssh_cm, connection_ref(), ssh_event_msg()}]]></c>. + If the <seealso marker="ssh_client_channel">ssh_client_channel</seealso> behavior is used to implement the channel process, these messages are handled by - <seealso marker="ssh_channel#Module:handle_ssh_msg-2">handle_ssh_msg/2</seealso>.</p> + <seealso marker="ssh_client_channel#Module:handle_ssh_msg-2">handle_ssh_msg/2</seealso>.</p> </description> <section> @@ -63,10 +63,10 @@ <item><p>list of ASCII characters</p></item> <tag><c>timeout() =</c></tag> <item><p><c>infinity | integer()</c> in milliseconds</p></item> - <tag><c>ssh_connection_ref() =</c></tag> + <tag><c>connection_ref() =</c></tag> <item><p>opaque() -as returned by <c>ssh:connect/3</c> or sent to an SSH channel processes</p></item> - <tag><c>ssh_channel_id() =</c></tag> + <tag><c>channel_id() =</c></tag> <item><p><c>integer()</c></p></item> <tag><c>ssh_data_type_code() =</c></tag> <item><p><c>1</c> ("stderr") | <c>0</c> ("normal") are @@ -75,7 +75,7 @@ <tag><c>ssh_request_status() =</c></tag> <item><p> <c>success | failure</c></p></item> <tag><c>event() =</c></tag> - <item><p><c>{ssh_cm, ssh_connection_ref(), ssh_event_msg()}</c></p></item> + <item><p><c>{ssh_cm, connection_ref(), ssh_event_msg()}</c></p></item> <tag><c>ssh_event_msg() =</c></tag> <item><p><c>data_events() | status_events() | terminal_events()</c></p></item> <tag><c>reason() =</c></tag> @@ -86,12 +86,12 @@ <tag><em>data_events()</em></tag> <item> <taglist> - <tag><c><![CDATA[{data, ssh_channel_id(), ssh_data_type_code(), Data :: binary()}]]></c></tag> + <tag><c><![CDATA[{data, channel_id(), ssh_data_type_code(), Data :: binary()}]]></c></tag> <item><p>Data has arrived on the channel. This event is sent as a result of calling <seealso marker="ssh_connection#send-3"> ssh_connection:send/[3,4,5]</seealso>.</p></item> - <tag><c><![CDATA[{eof, ssh_channel_id()}]]></c></tag> + <tag><c><![CDATA[{eof, channel_id()}]]></c></tag> <item><p>Indicates that the other side sends no more data. This event is sent as a result of calling <seealso marker="ssh_connection#send_eof-2"> ssh_connection:send_eof/2</seealso>. @@ -103,7 +103,7 @@ <item> <taglist> - <tag><c><![CDATA[{signal, ssh_channel_id(), ssh_signal()}]]></c></tag> + <tag><c><![CDATA[{signal, channel_id(), ssh_signal()}]]></c></tag> <item><p>A signal can be delivered to the remote process/service using the following message. Some systems do not support signals, in which case they are to ignore this message. There is @@ -111,7 +111,7 @@ referred to are on OS-level and not something generated by an Erlang program.</p></item> - <tag><c><![CDATA[{exit_signal, ssh_channel_id(), ExitSignal :: string(), ErrorMsg ::string(), + <tag><c><![CDATA[{exit_signal, channel_id(), ExitSignal :: string(), ErrorMsg ::string(), LanguageString :: string()}]]></c></tag> <item><p>A remote execution can terminate violently because of a signal. @@ -119,7 +119,7 @@ values, see <url href="http://www.ietf.org/rfc/rfc4254.txt">RFC 4254</url> Section 6.10, which shows a special case of these signals.</p></item> - <tag><c><![CDATA[{exit_status, ssh_channel_id(), ExitStatus :: integer()}]]></c></tag> + <tag><c><![CDATA[{exit_status, channel_id(), ExitStatus :: integer()}]]></c></tag> <item><p>When the command running at the other end terminates, the following message can be sent to return the exit status of the command. A zero <c>exit_status</c> usually means that the command @@ -127,11 +127,11 @@ <seealso marker="ssh_connection#exit_status-3"> ssh_connection:exit_status/3</seealso>.</p></item> - <tag><c><![CDATA[{closed, ssh_channel_id()}]]></c></tag> + <tag><c><![CDATA[{closed, channel_id()}]]></c></tag> <item><p>This event is sent as a result of calling <seealso marker="ssh_connection#close-2">ssh_connection:close/2</seealso>. Both the handling of this event and sending it are taken care of by the - <seealso marker="ssh_channel">ssh_channel</seealso> behavior.</p></item> + <seealso marker="ssh_client_channel">ssh_client_channel</seealso> behavior.</p></item> </taglist> </item> @@ -149,14 +149,14 @@ with the boolean value of <c>WantReply</c> as the second argument.</p> <taglist> - <tag><c><![CDATA[{env, ssh_channel_id(), WantReply :: boolean(), + <tag><c><![CDATA[{env, channel_id(), WantReply :: boolean(), Var ::string(), Value :: string()}]]></c></tag> <item><p>Environment variables can be passed to the shell/command to be started later. This event is sent as a result of calling <seealso marker="ssh_connection#setenv-5"> ssh_connection:setenv/5</seealso>. </p></item> - <tag><c><![CDATA[{pty, ssh_channel_id(), + <tag><c><![CDATA[{pty, channel_id(), WantReply :: boolean(), {Terminal :: string(), CharWidth :: integer(), RowHeight :: integer(), PixelWidth :: integer(), PixelHeight :: integer(), TerminalModes :: [{Opcode :: atom() | integer(), @@ -181,13 +181,13 @@ <seealso marker="ssh_connection#shell-2"> ssh_connection:shell/2</seealso>. </p></item> - <tag><c><![CDATA[{window_change, ssh_channel_id(), CharWidth() :: integer(), + <tag><c><![CDATA[{window_change, channel_id(), CharWidth() :: integer(), RowHeight :: integer(), PixWidth :: integer(), PixHeight :: integer()}]]></c></tag> <item><p>When the window (terminal) size changes on the client side, it <em>can</em> send a message to the server side to inform it of the new dimensions. No API function generates this event.</p></item> - <tag><c><![CDATA[{exec, ssh_channel_id(), + <tag><c><![CDATA[{exec, channel_id(), WantReply :: boolean(), Cmd :: string()}]]></c></tag> <item><p>This message requests that the server starts execution of the given command. This event is sent as a result of calling <seealso @@ -204,18 +204,18 @@ <name>adjust_window(ConnectionRef, ChannelId, NumOfBytes) -> ok</name> <fsummary>Adjusts the SSH flow control window.</fsummary> <type> - <v>ConnectionRef = ssh_connection_ref()</v> - <v>ChannelId = ssh_channel_id()</v> + <v>ConnectionRef = connection_ref()</v> + <v>ChannelId = channel_id()</v> <v>NumOfBytes = integer()</v> </type> <desc> <p>Adjusts the SSH flow control window. This is to be done by both the client- and server-side channel processes.</p> - <note><p>Channels implemented with the <seealso marker="ssh_channel"> ssh_channel</seealso> + <note><p>Channels implemented with the <seealso marker="ssh_client_channel"> ssh_client_channel</seealso> behavior do not normally need to call this function as flow control is handled by the behavior. The behavior adjusts the window every time - the callback <seealso marker="ssh_channel#Module:handle_ssh_msg-2"> + the callback <seealso marker="ssh_client_channel#Module:handle_ssh_msg-2"> handle_ssh_msg/2</seealso> returns after processing channel data.</p></note> </desc> </func> @@ -224,17 +224,17 @@ <name>close(ConnectionRef, ChannelId) -> ok</name> <fsummary>Sends a close message on the channel <c>ChannelId</c>.</fsummary> <type> - <v>ConnectionRef = ssh_connection_ref()</v> - <v>ChannelId = ssh_channel_id()</v> + <v>ConnectionRef = connection_ref()</v> + <v>ChannelId = channel_id()</v> </type> <desc> <p>A server- or client-channel process can choose to close their session by sending a close event. </p> - <note><p>This function is called by the <c>ssh_channel</c> + <note><p>This function is called by the <c>ssh_client_channel</c> behavior when the channel is terminated, see <seealso - marker="ssh_channel"> ssh_channel(3)</seealso>. Thus, channels implemented + marker="ssh_client_channel"> ssh_client_channel(3)</seealso>. Thus, channels implemented with the behavior are not to call this function explicitly.</p></note> </desc> </func> @@ -244,8 +244,8 @@ {error, reason()}</name> <fsummary>Requests that the server starts the execution of the given command.</fsummary> <type> - <v>ConnectionRef = ssh_connection_ref()</v> - <v>ChannelId = ssh_channel_id()</v> + <v>ConnectionRef = connection_ref()</v> + <v>ChannelId = channel_id()</v> <v>Command = string()</v> <v>Timeout = timeout()</v> </type> @@ -256,28 +256,28 @@ request is a one-time execution that closes the channel when it is done.</p> <taglist> - <tag><c>N x {ssh_cm, ssh_connection_ref(), - {data, ssh_channel_id(), ssh_data_type_code(), Data :: binary()}}</c></tag> + <tag><c>N x {ssh_cm, connection_ref(), + {data, channel_id(), ssh_data_type_code(), Data :: binary()}}</c></tag> <item><p>The result of executing the command can be only one line or thousands of lines depending on the command.</p></item> - <tag><c>0 or 1 x {ssh_cm, ssh_connection_ref(), {eof, ssh_channel_id()}}</c></tag> + <tag><c>0 or 1 x {ssh_cm, connection_ref(), {eof, channel_id()}}</c></tag> <item><p>Indicates that no more data is to be sent.</p></item> <tag><c>0 or 1 x {ssh_cm, - ssh_connection_ref(), {exit_signal, - ssh_channel_id(), ExitSignal :: string(), ErrorMsg :: string(), LanguageString :: string()}}</c></tag> + connection_ref(), {exit_signal, + channel_id(), ExitSignal :: string(), ErrorMsg :: string(), LanguageString :: string()}}</c></tag> <item><p>Not all systems send signals. For details on valid string values, see RFC 4254, Section 6.10</p></item> - <tag><c>0 or 1 x {ssh_cm, ssh_connection_ref(), {exit_status, - ssh_channel_id(), ExitStatus :: integer()}}</c></tag> + <tag><c>0 or 1 x {ssh_cm, connection_ref(), {exit_status, + channel_id(), ExitStatus :: integer()}}</c></tag> <item><p>It is recommended by the SSH Connection Protocol to send this message, but that is not always the case.</p></item> - <tag><c>1 x {ssh_cm, ssh_connection_ref(), - {closed, ssh_channel_id()}}</c></tag> - <item><p>Indicates that the <c>ssh_channel</c> started for the + <tag><c>1 x {ssh_cm, connection_ref(), + {closed, channel_id()}}</c></tag> + <item><p>Indicates that the <c>ssh_client_channel</c> started for the execution of the command has now been shut down.</p></item> </taglist> </desc> @@ -287,8 +287,8 @@ <name>exit_status(ConnectionRef, ChannelId, Status) -> ok</name> <fsummary>Sends the exit status of a command to the client.</fsummary> <type> - <v>ConnectionRef = ssh_connection_ref() </v> - <v>ChannelId = ssh_channel_id()</v> + <v>ConnectionRef = connection_ref() </v> + <v>ChannelId = channel_id()</v> <v>Status = integer()</v> </type> <desc> @@ -304,8 +304,8 @@ <fsummary>Sends an SSH Connection Protocol <c>pty_req</c>, to allocate a pseudo-terminal.</fsummary> <type> - <v>ConnectionRef = ssh_connection_ref()</v> - <v>ChannelId = ssh_channel_id()</v> + <v>ConnectionRef = connection_ref()</v> + <v>ChannelId = channel_id()</v> <v>Options = proplists:proplist()</v> </type> <desc> @@ -342,10 +342,10 @@ <name>reply_request(ConnectionRef, WantReply, Status, ChannelId) -> ok</name> <fsummary>Sends status replies to requests that want such replies.</fsummary> <type> - <v>ConnectionRef = ssh_connection_ref()</v> + <v>ConnectionRef = connection_ref()</v> <v>WantReply = boolean()</v> <v>Status = ssh_request_status()</v> - <v>ChannelId = ssh_channel_id()</v> + <v>ChannelId = channel_id()</v> </type> <desc> <p>Sends status replies to requests where the requester has @@ -364,8 +364,8 @@ ok | {error, timeout} | {error, closed}</name> <fsummary>Sends channel data.</fsummary> <type> - <v>ConnectionRef = ssh_connection_ref()</v> - <v>ChannelId = ssh_channel_id()</v> + <v>ConnectionRef = connection_ref()</v> + <v>ChannelId = channel_id()</v> <v>Data = binary()</v> <v>Type = ssh_data_type_code()</v> <v>Timeout = timeout()</v> @@ -383,8 +383,8 @@ <name>send_eof(ConnectionRef, ChannelId) -> ok | {error, closed}</name> <fsummary>Sends EOF on channel <c>ChannelId</c>.</fsummary> <type> - <v>ConnectionRef = ssh_connection_ref()</v> - <v>ChannelId = ssh_channel_id()</v> + <v>ConnectionRef = connection_ref()</v> + <v>ChannelId = channel_id()</v> </type> <desc> <p>Sends EOF on channel <c>ChannelId</c>.</p> @@ -394,10 +394,10 @@ <func> <name>session_channel(ConnectionRef, Timeout) -></name> <name>session_channel(ConnectionRef, InitialWindowSize, - MaxPacketSize, Timeout) -> {ok, ssh_channel_id()} | {error, reason()}</name> + MaxPacketSize, Timeout) -> {ok, channel_id()} | {error, reason()}</name> <fsummary>Opens a channel for an SSH session.</fsummary> <type> - <v>ConnectionRef = ssh_connection_ref()</v> + <v>ConnectionRef = connection_ref()</v> <v>InitialWindowSize = integer()</v> <v>MaxPacketSize = integer()</v> <v>Timeout = timeout()</v> @@ -415,8 +415,8 @@ <fsummary>Environment variables can be passed to the shell/command to be started later.</fsummary> <type> - <v>ConnectionRef = ssh_connection_ref()</v> - <v>ChannelId = ssh_channel_id()</v> + <v>ConnectionRef = connection_ref()</v> + <v>ChannelId = channel_id()</v> <v>Var = string()</v> <v>Value = string()</v> <v>Timeout = timeout()</v> @@ -428,18 +428,22 @@ </func> <func> - <name>shell(ConnectionRef, ChannelId) -> ssh_request_status() | {error, closed} + <name>shell(ConnectionRef, ChannelId) -> ok | failure | {error, closed} </name> <fsummary>Requests that the user default shell (typically defined in /etc/passwd in Unix systems) is to be executed at the server end.</fsummary> <type> - <v>ConnectionRef = ssh_connection_ref()</v> - <v>ChannelId = ssh_channel_id()</v> + <v>ConnectionRef = connection_ref()</v> + <v>ChannelId = channel_id()</v> </type> <desc> <p>Is to be called by a client channel process to request that the user default shell (typically defined in /etc/passwd in Unix systems) is executed at the server end.</p> + <p>Note: the return value is <c>ok</c> instead of <c>success</c> unlike in other + functions in this module. This is a fault that was introduced so long ago that + any change would break a large number of existing software. + </p> </desc> </func> @@ -448,8 +452,8 @@ {error, reason()}</name> <fsummary>Requests to execute a predefined subsystem on the server.</fsummary> <type> - <v>ConnectionRef = ssh_connection_ref()</v> - <v>ChannelId = ssh_channel_id()</v> + <v>ConnectionRef = connection_ref()</v> + <v>ChannelId = channel_id()</v> <v>Subsystem = string()</v> <v>Timeout = timeout()</v> </type> diff --git a/lib/ssh/doc/src/ssh_protocol.xml b/lib/ssh/doc/src/ssh_protocol.xml index a0032ab449..53f0524b97 100644 --- a/lib/ssh/doc/src/ssh_protocol.xml +++ b/lib/ssh/doc/src/ssh_protocol.xml @@ -87,8 +87,10 @@ connection, and all channels are flow-controlled. Typically an SSH client will open a channel, send data/commands, receive data/"control information" and when it is done close the - channel. The <seealso - marker="ssh_channel">ssh_channel</seealso> behaviour makes it easy to + channel. The + <seealso marker="ssh_client_channel">ssh_client_channel</seealso> / + <seealso marker="ssh_server_channel">ssh_server_channel</seealso> (Replaces ssh_daemon_channel) + behaviours makes it easy to write your own SSH client/server processes that use flow control. It handles generic parts of SSH channel management and lets you focus on the application logic. diff --git a/lib/ssh/doc/src/ssh_server_channel.xml b/lib/ssh/doc/src/ssh_server_channel.xml new file mode 100644 index 0000000000..af51ec470b --- /dev/null +++ b/lib/ssh/doc/src/ssh_server_channel.xml @@ -0,0 +1,176 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>2009</year> + <year>2016</year> + <holder>Ericsson AB, All Rights Reserved</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + The Initial Developer of the Original Code is Ericsson AB. + </legalnotice> + <title>ssh_server_channel</title> + <prepared></prepared> + <docno></docno> + <date></date> + <rev></rev> + </header> + <module>ssh_server_channel</module> + <modulesummary>-behaviour(ssh_server_channel). (Replaces ssh_daemon_channel) + </modulesummary> + <description> + <note> + <p>This module replaces ssh_daemon_channel.</p> + <p>The old module is still available for compatibility, but should not be used for new programs. + The old module will not be maintained except for some error corrections + </p> + </note> + + <p>SSH services (clients and servers) are implemented as channels + that are multiplexed over an SSH connection and communicates over + the <url href="http://www.ietf.org/rfc/rfc4254.txt"> SSH + Connection Protocol</url>. This module provides a callback API + that takes care of generic channel aspects for daemons, such as flow control + and close messages. It lets the callback functions take care of + the service (application) specific parts. This behavior also ensures + that the channel process honors the principal of an OTP-process so + that it can be part of a supervisor tree. This is a requirement of + channel processes implementing a subsystem that will be added to + the <c>ssh</c> applications supervisor tree. + </p> + + <note><p>When implementing a client subsystem handler, use + <seealso marker="ssh_client_channel">-behaviour(ssh_client_channel)</seealso> instead. + </p> + </note> + + </description> + + <section> + <title>Callback Functions</title> + <p> + The following functions are to be exported from a + <c>ssh_server_channel</c> callback module. + </p> + </section> + + <funcs> + <func> + <name>Module:init(Args) -> {ok, State} | {ok, State, timeout()} | + {stop, Reason}</name> + <fsummary>Makes necessary initializations and returns the + initial channel state if the initializations succeed.</fsummary> + <type> + <v>Args = term()</v> + <d>Last argument to <c>start_link/4</c>.</d> + <v>State = term()</v> + <v>Reason = term()</v> + </type> + <desc> + <p>Makes necessary initializations and returns the initial channel + state if the initializations succeed. + </p> + <p>The time-out values that can be returned + have the same semantics as in a <seealso marker="stdlib:gen_server">gen_server</seealso>. + If the time-out occurs, <seealso marker="#Module:handle_msg-2">handle_msg/2</seealso> + is called as <c>handle_msg(timeout, State)</c>. + </p> + </desc> + </func> + + <func> + <name>Module:handle_msg(Msg, State) -> {ok, State} | + {stop, ChannelId, State}</name> + + <fsummary>Handles other messages than SSH connection protocol, + call, or cast messages sent to the channel.</fsummary> + <type> + <v>Msg = timeout | term()</v> + <v>ChannelId = <seealso marker="ssh#type-channel_id">ssh:channel_id()</seealso></v> + <v>State = term() </v> + </type> + <desc> + <p>Handles other messages than SSH Connection Protocol, call, or + cast messages sent to the channel. + </p> + + <p>Possible Erlang 'EXIT' messages is to be handled by this + function and all channels are to handle the following message.</p> + + <taglist> + <tag><c>{ssh_channel_up, ssh:channel_id(), ssh:connection_ref()}</c></tag> + <item><p>This is the first message that the channel receives. + This is especially useful if the + server wants to send a message to the client without first + receiving a message from it. If the message is not + useful for your particular scenario, ignore it by + immediately returning <c>{ok, State}</c>. + </p></item> + </taglist> + </desc> + </func> + + <func> + <name>Module:handle_ssh_msg(Msg, State) -> {ok, State} | {stop, + ChannelId, State}</name> + <fsummary>Handles <c>ssh</c> connection protocol messages.</fsummary> + <type> + <v>Msg = ssh_connection:event()</v> + <v>ChannelId = <seealso marker="ssh#type-channel_id">ssh:channel_id()</seealso></v> + <v>State = term()</v> + </type> + <desc> + <p>Handles SSH Connection Protocol messages that may need + service-specific attention. For details, + see <seealso marker="ssh_connection"> ssh_connection:event()</seealso>. + </p> + + <p>The following message is taken care of by the + <c>ssh_server_channel</c> behavior.</p> + + <taglist> + <tag><c>{closed, ssh:channel_id()}</c></tag> + <item><p>The channel behavior sends a close message to the + other side, if such a message has not already been sent. + Then it terminates the channel with reason <c>normal</c>.</p></item> + </taglist> + </desc> + </func> + + <func> + <name>Module:terminate(Reason, State) -> _</name> + <fsummary>Does cleaning up before channel process termination. + </fsummary> + <type> + <v>Reason = term()</v> + <v>State = term()</v> + </type> + <desc> + <p>This function is called by a channel process when it is + about to terminate. Before this function is called, <seealso + marker="ssh_connection#close-2"> ssh_connection:close/2 + </seealso> is called, if it has not been called earlier. + This function does any necessary cleaning + up. When it returns, the channel process terminates with + reason <c>Reason</c>. The return value is ignored. + </p> + </desc> + </func> + + </funcs> + +</erlref> diff --git a/lib/ssh/doc/src/ssh_server_key_api.xml b/lib/ssh/doc/src/ssh_server_key_api.xml index c6808b95d1..cf3b1d319f 100644 --- a/lib/ssh/doc/src/ssh_server_key_api.xml +++ b/lib/ssh/doc/src/ssh_server_key_api.xml @@ -41,7 +41,7 @@ see the <seealso marker="SSH_app"> ssh(6)</seealso> application manual.</p> </description> - <section> + <!-- section> <title>DATA TYPES</title> <p>Type definitions that are used more than once in this module, @@ -69,22 +69,40 @@ | 'rsa-sha2-256' | 'rsa-sha2-384' | 'rsa-sha2-512' | 'ecdsa-sha2-nistp256' | 'ecdsa-sha2-nistp384' | 'ecdsa-sha2-nistp521' </c></p></item> </taglist> - </section> + </section --> + <datatypes> + <datatype> + <name name="daemon_key_cb_options"/> + <desc> + <p>Options provided to <seealso marker="ssh#daemon-2">ssh:daemon/2,3</seealso>. + </p> + <p>The option list given in the + <seealso marker="ssh#type-key_cb_common_option"><c>key_cb</c></seealso> + option is available with the key <c>key_cb_private</c>. + </p> + </desc> + </datatype> + </datatypes> + <funcs> <func> <name>Module:host_key(Algorithm, DaemonOptions) -> {ok, Key} | {error, Reason}</name> <fsummary>Fetches the host’s private key.</fsummary> <type> - <v>Algorithm = public_key_algorithm()</v> + <v>Algorithm = <seealso marker="ssh#type-pubkey_alg">ssh:pubkey_alg()</seealso></v> <d>Host key algorithm.</d> - <v>DaemonOptions = proplists:proplist()</v> - <d>Options provided to <seealso marker="ssh#daemon-2">ssh:daemon/[2,3]</seealso>. The option list given in - the <c>key_cb</c> option is available with the key <c>key_cb_private</c>.</d> - <v>Key = private_key() | crypto:engine_key_ref()</v> + + <v>DaemonOptions = <seealso marker="#type-daemon_key_cb_options">daemon_key_cb_options()</seealso></v> + + <v>PrivateKey = <seealso marker="public_key:public_key#type-private_key">public_key:private_key()</seealso> + | <seealso marker="crypto:crypto#type-engine_key_ref">crypto:engine_key_ref()</seealso> + </v> + <d>Private key of the host matching the <c>Algorithm</c>. It may be a reference to a 'ssh-rsa', rsa-sha2-* or 'ssh-dss' (NOT ecdsa) key stored in a loaded Engine.</d> + <v>Reason = term()</v> </type> <desc> @@ -93,16 +111,17 @@ </func> <func> - <name>Module:is_auth_key(Key, User, DaemonOptions) -> Result</name> + <name>Module:is_auth_key(PublicUserKey, User, DaemonOptions) -> Result</name> <fsummary>Checks if the user key is authorized.</fsummary> <type> - <v>Key = public_key()</v> + <v>PublicUserKey = <seealso marker="public_key:public_key#type-public_key">public_key:public_key()</seealso></v> <d>Normally an RSA, DSA or ECDSA public key, but handling of other public keys can be added</d> + <v>User = string()</v> <d>User owning the public key.</d> - <v>DaemonOptions = proplists:proplist()</v> - <d>Options provided to <seealso marker="ssh#daemon-2">ssh:daemon/[2,3]</seealso>. The option list given in - the <c>key_cb</c> option is available with the key <c>key_cb_private</c>.</d> + + <v>DaemonOptions = <seealso marker="#type-daemon_key_cb_options">daemon_key_cb_options()</seealso></v> + <v>Result = boolean()</v> </type> <desc> diff --git a/lib/ssh/doc/src/ssh_sftp.xml b/lib/ssh/doc/src/ssh_sftp.xml index 129426a6d5..60f643d052 100644 --- a/lib/ssh/doc/src/ssh_sftp.xml +++ b/lib/ssh/doc/src/ssh_sftp.xml @@ -59,7 +59,7 @@ </p> </item> - <tag><c>ssh_connection_ref() =</c></tag> + <tag><c>connection_ref() =</c></tag> <item><p><c>opaque()</c> - as returned by <seealso marker="ssh#connect-3"><c>ssh:connect/3</c></seealso></p></item> @@ -546,7 +546,7 @@ <fsummary>Starts an SFTP client.</fsummary> <type> <v>Host = string()</v> - <v>ConnectionRef = ssh_connection_ref()</v> + <v>ConnectionRef = connection_ref()</v> <v>Port = integer()</v> <v>TcpSocket = port()</v> <d>The socket is supposed to be from <seealso marker="kernel:gen_tcp#connect-3">gen_tcp:connect</seealso> or <seealso marker="kernel:gen_tcp#accept-1">gen_tcp:accept</seealso> with option <c>{active,false}</c></d> diff --git a/lib/ssh/doc/src/ssh_sftpd.xml b/lib/ssh/doc/src/ssh_sftpd.xml index 1be29b3b29..a25ce123b3 100644 --- a/lib/ssh/doc/src/ssh_sftpd.xml +++ b/lib/ssh/doc/src/ssh_sftpd.xml @@ -44,8 +44,7 @@ <item><p><c>"sftp"</c></p></item> <tag><c>channel_callback() =</c></tag> <item><p><c>atom()</c> - Name of the Erlang module implementing the subsystem using the - <c>ssh_channel</c> behavior, see the - <seealso marker="ssh_channel">ssh_channel(3)</seealso> manual page.</p></item> + <seealso marker="ssh_server_channel">ssh_server_channel</seealso> (replaces ssh_daemon_channel) behaviour.</p></item> <tag><c>channel_init_args() =</c></tag> <item><p><c>list()</c> - The one given as argument to function <c>subsystem_spec/1</c>.</p></item> </taglist> diff --git a/lib/ssh/doc/src/using_ssh.xml b/lib/ssh/doc/src/using_ssh.xml index ab307624e6..efd2a997f5 100644 --- a/lib/ssh/doc/src/using_ssh.xml +++ b/lib/ssh/doc/src/using_ssh.xml @@ -298,6 +298,7 @@ ok = erl_tar:close(HandleRead), </section> <section> + <marker id="usersguide_creating_a_subsystem"/> <title>Creating a Subsystem</title> <p>A small <c>ssh</c> subsystem that echoes N bytes can be implemented as shown @@ -305,7 +306,7 @@ ok = erl_tar:close(HandleRead), <code type="erl" > -module(ssh_echo_server). --behaviour(ssh_daemon_channel). +-behaviour(ssh_server_channel). % replaces ssh_daemon_channel -record(state, { n, id, @@ -383,7 +384,7 @@ terminate(_Reason, _State) -> {ssh_msg, <0.57.0>, {closed, 0}} 7> {error, closed} = ssh_connection:send(ConnectionRef, ChannelId, "10", infinity). </code> -<p>See also <seealso marker="ssh_channel"> ssh_channel(3)</seealso>.</p> +<p>See also <seealso marker="ssh_client_channel">ssh_client_channel(3)</seealso> (replaces ssh_channel(3)).</p> </section> diff --git a/lib/ssh/src/Makefile b/lib/ssh/src/Makefile index 9e8d80c71f..5e4efb6b99 100644 --- a/lib/ssh/src/Makefile +++ b/lib/ssh/src/Makefile @@ -40,42 +40,44 @@ RELSYSDIR = $(RELEASE_PATH)/lib/ssh-$(VSN) # Behaviour (api) modules are first so they are compiled when # the compiler reaches a callback module using them. BEHAVIOUR_MODULES= \ - ssh_sftpd_file_api \ - ssh_channel \ - ssh_daemon_channel \ ssh_client_key_api \ - ssh_server_key_api + ssh_daemon_channel \ + ssh_server_channel \ + ssh_server_key_api \ + ssh_sftpd_file_api \ + ssh_channel \ + ssh_client_channel MODULES= \ ssh \ - ssh_sup \ - sshc_sup \ - sshd_sup \ - ssh_options \ - ssh_connection_sup \ - ssh_connection \ - ssh_connection_handler \ - ssh_dbg \ - ssh_shell \ - ssh_system_sup \ - ssh_subsystem_sup \ - ssh_channel_sup \ - ssh_acceptor_sup \ ssh_acceptor \ + ssh_acceptor_sup \ ssh_app \ ssh_auth\ ssh_bits \ ssh_cli \ + ssh_connection \ + ssh_connection_handler \ + ssh_connection_sup \ + ssh_dbg \ ssh_file \ - ssh_io \ ssh_info \ + ssh_io \ ssh_message \ ssh_no_io \ + ssh_options \ + ssh_server_channel_sup \ ssh_sftp \ ssh_sftpd \ ssh_sftpd_file\ + ssh_shell \ + ssh_subsystem_sup \ + ssh_sup \ + ssh_system_sup \ ssh_transport \ - ssh_xfer + ssh_xfer \ + sshc_sup \ + sshd_sup HRL_FILES = @@ -97,7 +99,7 @@ APP_TARGET= $(EBIN)/$(APP_FILE) APPUP_SRC= $(APPUP_FILE).src APPUP_TARGET= $(EBIN)/$(APPUP_FILE) -INTERNAL_HRL_FILES = ssh_auth.hrl ssh_connect.hrl ssh_transport.hrl ssh.hrl ssh_userauth.hrl ssh_xfer.hrl ssh_dbg.hrl +INTERNAL_HRL_FILES = ssh_auth.hrl ssh_connect.hrl ssh_transport.hrl ssh.hrl ssh_userauth.hrl ssh_xfer.hrl # ---------------------------------------------------- # FLAGS @@ -169,7 +171,7 @@ $(EBIN)/ssh_connection_handler.$(EMULATOR): ssh_connection_handler.erl ssh.hrl \ $(EBIN)/ssh_shell.$(EMULATOR): ssh_shell.erl ssh_connect.hrl $(EBIN)/ssh_system_sup.$(EMULATOR): ssh_system_sup.erl ssh.hrl $(EBIN)/ssh_subsystem_sup.$(EMULATOR): ssh_subsystem_sup.erl -$(EBIN)/ssh_channel_sup.$(EMULATOR): ssh_channel_sup.erl +$(EBIN)/ssh_server_channel_sup.$(EMULATOR): ssh_server_channel_sup.erl $(EBIN)/ssh_acceptor_sup.$(EMULATOR): ssh_acceptor_sup.erl ssh.hrl $(EBIN)/ssh_acceptor.$(EMULATOR): ssh_acceptor.erl ssh.hrl $(EBIN)/ssh_app.$(EMULATOR): ssh_app.erl @@ -208,8 +210,10 @@ $(EBIN)/ssh_transport.$(EMULATOR): ssh_transport.erl \ ssh_transport.hrl ssh.hrl $(EBIN)/ssh_xfer.$(EMULATOR): ssh_xfer.erl ssh.hrl ssh_xfer.hrl $(EBIN)/ssh_sftpd_file_api.$(EMULATOR): ssh_sftpd_file_api.erl +$(EBIN)/ssh_client_channel.$(EMULATOR): ssh_client_channel.erl ssh_connect.hrl $(EBIN)/ssh_channel.$(EMULATOR): ssh_channel.erl ssh_connect.hrl $(EBIN)/ssh_daemon_channel.$(EMULATOR): ssh_daemon_channel.erl +$(EBIN)/ssh_server_channel.$(EMULATOR): ssh_server_channel.erl $(EBIN)/ssh_client_key_api.$(EMULATOR): ssh_client_key_api.erl \ ../../public_key/include/public_key.hrl \ ../../public_key/include/OTP-PUB-KEY.hrl \ diff --git a/lib/ssh/src/ssh.app.src b/lib/ssh/src/ssh.app.src index 4a22322333..410061cded 100644 --- a/lib/ssh/src/ssh.app.src +++ b/lib/ssh/src/ssh.app.src @@ -12,9 +12,9 @@ ssh_message, ssh_bits, ssh_cli, + ssh_client_channel, ssh_client_key_api, ssh_channel, - ssh_channel_sup, ssh_connection, ssh_connection_handler, ssh_connection_sup, @@ -27,6 +27,8 @@ ssh_io, ssh_info, ssh_no_io, + ssh_server_channel, + ssh_server_channel_sup, ssh_server_key_api, ssh_sftp, ssh_sftpd, diff --git a/lib/ssh/src/ssh.erl b/lib/ssh/src/ssh.erl index 25d537c624..7ddb1ca5be 100644 --- a/lib/ssh/src/ssh.erl +++ b/lib/ssh/src/ssh.erl @@ -41,35 +41,51 @@ shell/1, shell/2, shell/3 ]). +%%% "Deprecated" types export: +-export_type([ssh_daemon_ref/0, ssh_connection_ref/0, ssh_channel_id/0]). +-opaque ssh_daemon_ref() :: daemon_ref(). +-opaque ssh_connection_ref() :: connection_ref(). +-opaque ssh_channel_id() :: channel_id(). + + %%% Type exports --export_type([ssh_daemon_ref/0, - ssh_connection_ref/0, - ssh_channel_id/0, +-export_type([daemon_ref/0, + connection_ref/0, + channel_id/0, + client_options/0, client_option/0, + daemon_options/0, daemon_option/0, + common_options/0, role/0, subsystem_spec/0, - subsystem_name/0, - channel_callback/0, - channel_init_args/0, algs_list/0, + double_algs/1, + modify_algs_list/0, alg_entry/0, - simple_algs/0, - double_algs/0 + kex_alg/0, + pubkey_alg/0, + cipher_alg/0, + mac_alg/0, + compression_alg/0, + ip_port/0 ]). --opaque ssh_daemon_ref() :: daemon_ref() . --opaque ssh_connection_ref() :: connection_ref() . --opaque ssh_channel_id() :: channel_id(). + +-opaque daemon_ref() :: pid() . +-opaque channel_id() :: non_neg_integer(). +-type connection_ref() :: pid(). % should be -opaque, but that gives problems %%-------------------------------------------------------------------- --spec start() -> ok | {error, term()}. --spec start(permanent | transient | temporary) -> ok | {error, term()}. -%% %% Description: Starts the ssh application. Default type %% is temporary. see application(3) %%-------------------------------------------------------------------- +-spec start() -> ok | {error, term()}. + start() -> start(temporary). +-spec start(Type) -> ok | {error, term()} when + Type :: permanent | transient | temporary . + start(Type) -> case application:ensure_all_started(ssh, Type) of {ok, _} -> @@ -79,30 +95,32 @@ start(Type) -> end. %%-------------------------------------------------------------------- --spec stop() -> ok | {error, term()}. -%% %% Description: Stops the ssh application. %%-------------------------------------------------------------------- +-spec stop() -> ok | {error, term()}. + stop() -> application:stop(ssh). %%-------------------------------------------------------------------- --spec connect(inet:socket(), proplists:proplist()) -> ok_error(connection_ref()). +%% Description: Starts an ssh connection. +%%-------------------------------------------------------------------- +-spec connect(OpenTcpSocket, Options) -> {ok,connection_ref()} | {error,term()} when + OpenTcpSocket :: open_socket(), + Options :: client_options(). --spec connect(inet:socket(), proplists:proplist(), timeout()) -> ok_error(connection_ref()) - ; (string(), inet:port_number(), proplists:proplist()) -> ok_error(connection_ref()). +connect(OpenTcpSocket, Options) when is_port(OpenTcpSocket), + is_list(Options) -> + connect(OpenTcpSocket, Options, infinity). --spec connect(string(), inet:port_number(), proplists:proplist(), timeout()) -> ok_error(connection_ref()). -%% -%% Description: Starts an ssh connection. -%%-------------------------------------------------------------------- -connect(Socket, UserOptions) when is_port(Socket), - is_list(UserOptions) -> - connect(Socket, UserOptions, infinity). +-spec connect(open_socket(), client_options(), timeout()) -> + {ok,connection_ref()} | {error,term()} + ; (host(), inet:port_number(), client_options()) -> + {ok,connection_ref()} | {error,term()}. -connect(Socket, UserOptions, Timeout) when is_port(Socket), - is_list(UserOptions) -> +connect(Socket, UserOptions, NegotiationTimeout) when is_port(Socket), + is_list(UserOptions) -> case ssh_options:handle_options(client, UserOptions) of {error, Error} -> {error, Error}; @@ -111,16 +129,23 @@ connect(Socket, UserOptions, Timeout) when is_port(Socket), ok -> {ok, {Host,_Port}} = inet:sockname(Socket), Opts = ?PUT_INTERNAL_OPT([{user_pid,self()}, {host,Host}], Options), - ssh_connection_handler:start_connection(client, Socket, Opts, Timeout); + ssh_connection_handler:start_connection(client, Socket, Opts, NegotiationTimeout); {error,SockError} -> {error,SockError} end end; -connect(Host, Port, UserOptions) when is_integer(Port), - Port>0, - is_list(UserOptions) -> - connect(Host, Port, UserOptions, infinity). +connect(Host, Port, Options) when is_integer(Port), + Port>0, + is_list(Options) -> + connect(Host, Port, Options, infinity). + + +-spec connect(Host, Port, Options, NegotiationTimeout) -> {ok,connection_ref()} | {error,term()} when + Host :: host(), + Port :: inet:port_number(), + Options :: client_options(), + NegotiationTimeout :: timeout(). connect(Host0, Port, UserOptions, Timeout) when is_integer(Port), Port>0, @@ -148,7 +173,8 @@ connect(Host0, Port, UserOptions, Timeout) when is_integer(Port), end. %%-------------------------------------------------------------------- --spec close(pid()) -> ok. +-spec close(ConnectionRef) -> ok | {error,term()} when + ConnectionRef :: connection_ref() . %% %% Description: Closes an ssh connection. %%-------------------------------------------------------------------- @@ -156,15 +182,25 @@ close(ConnectionRef) -> ssh_connection_handler:stop(ConnectionRef). %%-------------------------------------------------------------------- --spec connection_info(pid(), [atom()]) -> [{atom(), term()}]. -%% %% Description: Retrieves information about a connection. %%-------------------------------------------------------------------- -connection_info(ConnectionRef, Options) -> - ssh_connection_handler:connection_info(ConnectionRef, Options). +-spec connection_info(ConnectionRef, Keys) -> ConnectionInfo when + ConnectionRef :: connection_ref(), + Keys :: [client_version | server_version | user | peer | sockname], + ConnectionInfo :: [{client_version, Version} + | {server_version, Version} + | {user,string()} + | {peer, {inet:hostname(), ip_port()}} + | {sockname, ip_port()} + ], + Version :: {ProtocolVersion, VersionString::string()}, + ProtocolVersion :: {Major::pos_integer(), Minor::non_neg_integer()} . + +connection_info(Connection, Options) -> + ssh_connection_handler:connection_info(Connection, Options). %%-------------------------------------------------------------------- --spec channel_info(pid(), channel_id(), [atom()]) -> [{atom(), term()}]. +-spec channel_info(connection_ref(), channel_id(), [atom()]) -> proplists:proplist(). %% %% Description: Retrieves information about a connection. %%-------------------------------------------------------------------- @@ -172,18 +208,17 @@ channel_info(ConnectionRef, ChannelId, Options) -> ssh_connection_handler:channel_info(ConnectionRef, ChannelId, Options). %%-------------------------------------------------------------------- --spec daemon(inet:port_number()) -> ok_error(daemon_ref()). --spec daemon(inet:port_number()|inet:socket(), proplists:proplist()) -> ok_error(daemon_ref()). --spec daemon(any | inet:ip_address(), inet:port_number(), proplists:proplist()) -> ok_error(daemon_ref()) - ;(socket, inet:socket(), proplists:proplist()) -> ok_error(daemon_ref()) - . - %% Description: Starts a server listening for SSH connections %% on the given port. %%-------------------------------------------------------------------- +-spec daemon(inet:port_number()) -> {ok,daemon_ref()} | {error,term()}. + daemon(Port) -> daemon(Port, []). + +-spec daemon(inet:port_number()|open_socket(), daemon_options()) -> {ok,daemon_ref()} | {error,term()}. + daemon(Socket, UserOptions) when is_port(Socket) -> try #{} = Options = ssh_options:handle_options(server, UserOptions), @@ -226,6 +261,10 @@ daemon(Port, UserOptions) when 0 =< Port, Port =< 65535 -> daemon(any, Port, UserOptions). +-spec daemon(any | inet:ip_address(), inet:port_number(), daemon_options()) -> {ok,daemon_ref()} | {error,term()} + ;(socket, open_socket(), daemon_options()) -> {ok,daemon_ref()} | {error,term()} + . + daemon(Host0, Port0, UserOptions0) when 0 =< Port0, Port0 =< 65535, Host0 == any ; Host0 == loopback ; is_tuple(Host0) -> try @@ -267,7 +306,12 @@ daemon(_, _, _) -> {error, badarg}. %%-------------------------------------------------------------------- --spec daemon_info(daemon_ref()) -> ok_error( [{atom(), term()}] ). +-spec daemon_info(Daemon) -> {ok, DaemonInfo} | {error,term()} when + Daemon :: daemon_ref(), + DaemonInfo :: [ {ip, inet:ip_address()} + | {port, inet:port_number()} + | {profile, term()} + ]. daemon_info(Pid) -> case catch ssh_system_sup:acceptor_supervisor(Pid) of @@ -290,16 +334,23 @@ daemon_info(Pid) -> end. %%-------------------------------------------------------------------- --spec stop_listener(daemon_ref()) -> ok. --spec stop_listener(inet:ip_address(), inet:port_number()) -> ok. -%% %% Description: Stops the listener, but leaves %% existing connections started by the listener up and running. %%-------------------------------------------------------------------- +-spec stop_listener(daemon_ref()) -> ok. + stop_listener(SysSup) -> ssh_system_sup:stop_listener(SysSup). + + +-spec stop_listener(inet:ip_address(), inet:port_number()) -> ok. + stop_listener(Address, Port) -> stop_listener(Address, Port, ?DEFAULT_PROFILE). + + +-spec stop_listener(any|inet:ip_address(), inet:port_number(), term()) -> ok. + stop_listener(any, Port, Profile) -> map_ip(fun(IP) -> ssh_system_sup:stop_listener(IP, Port, Profile) @@ -310,17 +361,23 @@ stop_listener(Address, Port, Profile) -> end, {address,Address}). %%-------------------------------------------------------------------- --spec stop_daemon(daemon_ref()) -> ok. --spec stop_daemon(inet:ip_address(), inet:port_number()) -> ok. --spec stop_daemon(inet:ip_address(), inet:port_number(), atom()) -> ok. -%% %% Description: Stops the listener and all connections started by %% the listener. %%-------------------------------------------------------------------- +-spec stop_daemon(DaemonRef::daemon_ref()) -> ok. + stop_daemon(SysSup) -> ssh_system_sup:stop_system(SysSup). + + +-spec stop_daemon(inet:ip_address(), inet:port_number()) -> ok. + stop_daemon(Address, Port) -> stop_daemon(Address, Port, ?DEFAULT_PROFILE). + + +-spec stop_daemon(any|inet:ip_address(), inet:port_number(), atom()) -> ok. + stop_daemon(any, Port, Profile) -> map_ip(fun(IP) -> ssh_system_sup:stop_system(IP, Port, Profile) @@ -331,33 +388,37 @@ stop_daemon(Address, Port, Profile) -> end, {address,Address}). %%-------------------------------------------------------------------- --spec shell(inet:socket() | string()) -> _. --spec shell(inet:socket() | string(), proplists:proplist()) -> _. --spec shell(string(), inet:port_number(), proplists:proplist()) -> _. - -%% Host = string() -%% Port = integer() -%% Options = [{Option, Value}] -%% %% Description: Starts an interactive shell to an SSH server on the %% given <Host>. The function waits for user input, %% and will not return until the remote shell is ended.(e.g. on %% exit from the shell) %%-------------------------------------------------------------------- +-spec shell(open_socket() | host()) -> _. + shell(Socket) when is_port(Socket) -> shell(Socket, []); shell(Host) -> shell(Host, ?SSH_DEFAULT_PORT, []). + +-spec shell(open_socket() | host(), client_options()) -> _. + shell(Socket, Options) when is_port(Socket) -> start_shell( connect(Socket, Options) ); shell(Host, Options) -> shell(Host, ?SSH_DEFAULT_PORT, Options). + +-spec shell(Host, Port, Options) -> _ when + Host :: host(), + Port :: inet:port_number(), + Options :: client_options() . + shell(Host, Port, Options) -> start_shell( connect(Host, Port, Options) ). + start_shell({ok, ConnectionRef}) -> case ssh_connection:session_channel(ConnectionRef, infinity) of {ok,ChannelId} -> @@ -365,11 +426,17 @@ start_shell({ok, ConnectionRef}) -> Args = [{channel_cb, ssh_shell}, {init_args,[ConnectionRef, ChannelId]}, {cm, ConnectionRef}, {channel_id, ChannelId}], - {ok, State} = ssh_channel:init([Args]), - ssh_channel:enter_loop(State); + {ok, State} = ssh_client_channel:init([Args]), + try + ssh_client_channel:enter_loop(State) + catch + exit:normal -> + ok + end; Error -> Error end; + start_shell(Error) -> Error. @@ -380,7 +447,7 @@ default_algorithms() -> ssh_transport:default_algorithms(). %%-------------------------------------------------------------------- --spec chk_algos_opts(list(any())) -> algs_list() . +-spec chk_algos_opts(client_options()|daemon_options()) -> internal_options() | {error,term()}. %%-------------------------------------------------------------------- chk_algos_opts(Opts) -> case lists:foldl( diff --git a/lib/ssh/src/ssh.hrl b/lib/ssh/src/ssh.hrl index 4711f54fb5..a3d9a1b1cb 100644 --- a/lib/ssh/src/ssh.hrl +++ b/lib/ssh/src/ssh.hrl @@ -98,35 +98,267 @@ %% Types --type role() :: client | server . --type ok_error(SuccessType) :: {ok, SuccessType} | {error, any()} . --type daemon_ref() :: pid() . +-type role() :: client | server . + +-type host() :: string() | inet:ip_address() | loopback . +-type open_socket() :: gen_tcp:socket(). + +-type subsystem_spec() :: {Name::string(), mod_args()} . + +-type algs_list() :: list( alg_entry() ). +-type alg_entry() :: {kex, [kex_alg()]} + | {public_key, [pubkey_alg()]} + | {cipher, double_algs(cipher_alg())} + | {mac, double_algs(mac_alg())} + | {compression, double_algs(compression_alg())} . + +-type kex_alg() :: 'diffie-hellman-group-exchange-sha1' | + 'diffie-hellman-group-exchange-sha256' | + 'diffie-hellman-group1-sha1' | + 'diffie-hellman-group14-sha1' | + 'diffie-hellman-group14-sha256' | + 'diffie-hellman-group16-sha512' | + 'diffie-hellman-group18-sha512' | + 'ecdh-sha2-nistp256' | + 'ecdh-sha2-nistp384' | + 'ecdh-sha2-nistp521' + . + +-type pubkey_alg() :: 'ecdsa-sha2-nistp256' | + 'ecdsa-sha2-nistp384' | + 'ecdsa-sha2-nistp521' | + 'rsa-sha2-256' | + 'rsa-sha2-512' | + 'ssh-dss' | + 'ssh-rsa' + . + +-type cipher_alg() :: '3des-cbc' | + 'AEAD_AES_128_GCM' | + 'AEAD_AES_256_GCM' | + 'aes128-cbc' | + 'aes128-ctr' | + '[email protected]' | + 'aes192-ctr' | + 'aes256-ctr' | + . + +-type mac_alg() :: 'AEAD_AES_128_GCM' | + 'AEAD_AES_256_GCM' | + 'hmac-sha1' | + 'hmac-sha2-256' | + 'hmac-sha2-512' + . + +-type compression_alg() :: 'none' | + 'zlib' | + . + +-type double_algs(AlgType) :: list( {client2server,[AlgType]} | {server2client,[AlgType]} ) + | [AlgType]. + +-type modify_algs_list() :: list( {append,algs_list()} | {prepend,algs_list()} | {rm,algs_list()} ) . + +-type internal_options() :: ssh_options:private_options(). +-type socket_options() :: [gen_tcp:connect_option() | gen_tcp:listen_option()]. + +-type client_options() :: [ client_option() ] . +-type daemon_options() :: [ daemon_option() ]. + + +-type common_options() :: [ common_option() ]. +-type common_option() :: + user_dir_common_option() + | profile_common_option() + | max_idle_time_common_option() + | key_cb_common_option() + | disconnectfun_common_option() + | unexpectedfun_common_option() + | ssh_msg_debug_fun_common_option() + | rekey_limit_common_option() + | id_string_common_option() + | preferred_algorithms_common_option() + | modify_algorithms_common_option() + | auth_methods_common_option() + | inet_common_option() + | fd_common_option() + . + +-define(COMMON_OPTION, common_option()). + + +-type user_dir_common_option() :: {user_dir, false | string()}. +-type profile_common_option() :: {profile, atom() }. +-type max_idle_time_common_option() :: {idle_time, timeout()}. +-type rekey_limit_common_option() :: {rekey_limit, non_neg_integer() }. + +-type key_cb_common_option() :: {key_cb, Module::atom() | {Module::atom(),Opts::[term()]} } . +-type disconnectfun_common_option() :: + {disconnectfun, fun((Reason::term()) -> void | any()) }. +-type unexpectedfun_common_option() :: + {unexpectedfun, fun((Message::term(),{Host::term(),Port::term()}) -> report | skip ) }. +-type ssh_msg_debug_fun_common_option() :: + {ssh_msg_debug_fun, fun((ssh:connection_ref(),AlwaysDisplay::boolean(),Msg::binary(),LanguageTag::binary()) -> any()) } . + +-type id_string_common_option() :: {id_string, string() | random | {random,Nmin::pos_integer(),Nmax::pos_integer()} }. +-type preferred_algorithms_common_option():: {preferred_algorithms, algs_list()}. +-type modify_algorithms_common_option() :: {modify_algorithms, modify_algs_list()}. +-type auth_methods_common_option() :: {auth_methods, string() }. + +-type inet_common_option() :: {inet, inet | inet6} . +-type fd_common_option() :: {fd, gen_tcp:socket()} . + + +-type opaque_common_options() :: + {transport, {atom(),atom(),atom()} } + | {vsn, {non_neg_integer(),non_neg_integer()} } + | {tstflg, list(term())} + | {user_dir_fun, fun()} + | {max_random_length_padding, non_neg_integer()} . + + + +-type client_option() :: + pref_public_key_algs_client_option() + | pubkey_passphrase_client_options() + | host_accepting_client_options() + | authentication_client_options() + | diffie_hellman_group_exchange_client_option() + | connect_timeout_client_option() + | recv_ext_info_client_option() + | opaque_client_options() + | gen_tcp:connect_option() + | ?COMMON_OPTION . + +-type opaque_client_options() :: + {keyboard_interact_fun, fun((term(),term(),term()) -> term())} + | opaque_common_options(). + +-type pref_public_key_algs_client_option() :: {pref_public_key_algs, [pubkey_alg()] } . + +-type pubkey_passphrase_client_options() :: {dsa_pass_phrase, string()} + | {rsa_pass_phrase, string()} + | {ecdsa_pass_phrase, string()} . + +-type host_accepting_client_options() :: + {silently_accept_hosts, accept_hosts()} + | {user_interaction, boolean()} + | {save_accepted_host, boolean()} + | {quiet_mode, boolean()} . + +-type accept_hosts() :: boolean() + | accept_callback() + | {HashAlgoSpec::fp_digest_alg(), accept_callback()}. + +-type fp_digest_alg() :: 'md5' | + 'sha' | + 'sha224' | + 'sha256' | + 'sha384' | + 'sha512' + . + +-type accept_callback() :: fun((PeerName::string(), fingerprint() ) -> boolean()) . +-type fingerprint() :: string() | [string()]. + +-type authentication_client_options() :: + {user, string()} + | {password, string()} . + +-type diffie_hellman_group_exchange_client_option() :: + {dh_gex_limits, {Min::pos_integer(), I::pos_integer(), Max::pos_integer()} } . + +-type connect_timeout_client_option() :: {connect_timeout, timeout()} . + +-type recv_ext_info_client_option() :: {recv_ext_info, boolean()} . + + + +-type daemon_option() :: + subsystem_daemon_option() + | shell_daemon_option() + | exec_daemon_option() + | ssh_cli_daemon_option() + | authentication_daemon_options() + | diffie_hellman_group_exchange_daemon_option() + | negotiation_timeout_daemon_option() + | hardening_daemon_options() + | callbacks_daemon_options() + | send_ext_info_daemon_option() + | opaque_daemon_options() + | gen_tcp:listen_option() + | ?COMMON_OPTION . --type subsystem_spec() :: {subsystem_name(), {channel_callback(), channel_init_args()}} . --type subsystem_name() :: string() . --type channel_callback() :: atom() . --type channel_init_args() :: list() . +-type subsystem_daemon_option() :: {subsystems, subsystem_spec()}. --type algs_list() :: list( alg_entry() ). --type alg_entry() :: {kex, simple_algs()} - | {public_key, simple_algs()} - | {cipher, double_algs()} - | {mac, double_algs()} - | {compression, double_algs()} . --type simple_algs() :: list( atom() ) . --type double_algs() :: list( {client2serverlist,simple_algs()} | {server2client,simple_algs()} ) - | simple_algs() . +-type shell_daemon_option() :: {shell, mod_fun_args() | 'shell_fun/1'() | 'shell_fun/2'() }. +-type 'shell_fun/1'() :: fun((User::string()) -> pid()) . +-type 'shell_fun/2'() :: fun((User::string(), PeerAddr::inet:ip_address()) -> pid()). --type options() :: #{socket_options := socket_options(), - internal_options := internal_options(), - option_key() => any() - }. +-type exec_daemon_option() :: {exec, 'exec_fun/1'() | 'exec_fun/2'() | 'exec_fun/3'() }. --type socket_options() :: proplists:proplist(). --type internal_options() :: #{option_key() => any()}. +-type 'exec_fun/1'() :: fun((Cmd::string()) -> exec_result()) . +-type 'exec_fun/2'() :: fun((Cmd::string(), User::string()) -> exec_result()) . +-type 'exec_fun/3'() :: fun((Cmd::string(), User::string(), ClientAddr::ip_port()) -> exec_result()) . +-type exec_result() :: {ok,Result::term()} | {error,Reason::term()} . --type option_key() :: atom(). +-type ssh_cli_daemon_option() :: {ssh_cli, mod_args() | no_cli }. +-type send_ext_info_daemon_option() :: {send_ext_info, boolean()} . + +-type authentication_daemon_options() :: + {system_dir, string()} + | {auth_method_kb_interactive_data, prompt_texts() } + | {user_passwords, [{UserName::string(),Pwd::string()}]} + | {password, string()} + | {pwdfun, pwdfun_2() | pwdfun_4()} . + +-type prompt_texts() :: + kb_int_tuple() + | kb_int_fun_3() + . + +-type kb_int_fun_3() :: fun((Peer::ip_port(), User::string(), Service::string()) -> kb_int_tuple()). +-type kb_int_tuple() :: {Name::string(), Instruction::string(), Prompt::string(), Echo::boolean()}. + +-type pwdfun_2() :: fun((User::string(), Password::string()) -> boolean()) . +-type pwdfun_4() :: fun((User::string(), + Password::string(), + PeerAddress::ip_port(), + State::any()) -> + boolean() | disconnect | {boolean(),NewState::any()} + ) . + +-type diffie_hellman_group_exchange_daemon_option() :: + {dh_gex_groups, [explicit_group()] | explicit_group_file() | ssh_moduli_file()} + | {dh_gex_limits, {Min::pos_integer(), Max::pos_integer()} } . + +-type explicit_group() :: {Size::pos_integer(),G::pos_integer(),P::pos_integer()} . +-type explicit_group_file() :: {file,string()} . +-type ssh_moduli_file() :: {ssh_moduli_file,string()}. + +-type negotiation_timeout_daemon_option() :: {negotiation_timeout, timeout()} . + +-type hardening_daemon_options() :: + {max_sessions, pos_integer()} + | {max_channels, pos_integer()} + | {parallel_login, boolean()} + | {minimal_remote_max_packet_size, pos_integer()}. + +-type callbacks_daemon_options() :: + {failfun, fun((User::string(), PeerAddress::inet:ip_address(), Reason::term()) -> _)} + | {connectfun, fun((User::string(), PeerAddress::inet:ip_address(), Method::string()) ->_)} . + +-type opaque_daemon_options() :: + {infofun, fun()} + | opaque_common_options(). + +-type ip_port() :: {inet:ip_address(), inet:port_number()} . + +-type mod_args() :: {Module::atom(), Args::list()} . +-type mod_fun_args() :: {Module::atom(), Function::atom(), Args::list()} . %% Records @@ -134,8 +366,9 @@ { role :: client | role(), peer :: undefined | - {inet:hostname(), - {inet:ip_address(),inet:port_number()}}, %% string version of peer address + {inet:hostname(),ip_port()}, %% string version of peer address + + local, %% Local sockname. Need this AFTER a socket is closed by i.e. a crash c_vsn, %% client version {Major,Minor} s_vsn, %% server version {Major,Minor} @@ -151,8 +384,6 @@ algorithms, %% #alg{} - kex, %% key exchange algorithm - hkey, %% host key algorithm key_cb, %% Private/Public key callback module io_cb, %% Interaction callback module @@ -248,4 +479,13 @@ _ -> exit(Reason) end). + +%% dbg help macros +-define(wr_record(N,BlackList), + wr_record(R=#N{}) -> ssh_dbg:wr_record(R, record_info(fields,N), BlackList) + ). + +-define(wr_record(N), ?wr_record(N, [])). + + -endif. % SSH_HRL defined diff --git a/lib/ssh/src/ssh_acceptor.erl b/lib/ssh/src/ssh_acceptor.erl index 27d4242dd4..516a9febaa 100644 --- a/lib/ssh/src/ssh_acceptor.erl +++ b/lib/ssh/src/ssh_acceptor.erl @@ -33,6 +33,8 @@ %% spawn export -export([acceptor_init/5, acceptor_loop/6]). +-export([dbg_trace/3]). + -define(SLEEP_TIME, 200). %%==================================================================== @@ -195,3 +197,33 @@ handle_error(Reason) -> error_logger:error_report(String), exit({accept_failed, String}). +%%%################################################################ +%%%# +%%%# Tracing +%%%# + +dbg_trace(points, _, _) -> [connections]; + +dbg_trace(flags, connections, _) -> [c]; +dbg_trace(on, connections, _) -> dbg:tp(?MODULE, acceptor_init, 5, x), + dbg:tpl(?MODULE, handle_connection, 5, x); +dbg_trace(off, connections, _) -> dbg:ctp(?MODULE, acceptor_init, 5), + dbg:ctp(?MODULE, handle_connection, 5); +dbg_trace(format, connections, {call, {?MODULE,acceptor_init, + [_Parent, Port, Address, _Opts, _AcceptTimeout]}}) -> + [io_lib:format("Starting LISTENER on ~s:~p\n", [ntoa(Address),Port]) + ]; +dbg_trace(format, connections, {return_from, {?MODULE,handle_connection,5}, {error,Error}}) -> + ["Starting connection to server failed:\n", + io_lib:format("Error = ~p", [Error]) + ]. + + + +ntoa(A) -> + try inet:ntoa(A) + catch + _:_ when is_list(A) -> A; + _:_ -> io_lib:format('~p',[A]) + end. + diff --git a/lib/ssh/src/ssh_acceptor_sup.erl b/lib/ssh/src/ssh_acceptor_sup.erl index fc564a359b..10fd4452bf 100644 --- a/lib/ssh/src/ssh_acceptor_sup.erl +++ b/lib/ssh/src/ssh_acceptor_sup.erl @@ -36,8 +36,6 @@ -define(DEFAULT_TIMEOUT, 50000). --spec init( [term()] ) -> {ok,{supervisor:sup_flags(),[supervisor:child_spec()]}} | ignore . - %%%========================================================================= %%% API %%%========================================================================= diff --git a/lib/ssh/src/ssh_auth.erl b/lib/ssh/src/ssh_auth.erl index 03d264745b..bf3f5a68e4 100644 --- a/lib/ssh/src/ssh_auth.erl +++ b/lib/ssh/src/ssh_auth.erl @@ -40,15 +40,12 @@ %%-------------------------------------------------------------------- %%%---------------------------------------------------------------- userauth_request_msg(#ssh{userauth_methods = ServerMethods, - userauth_supported_methods = UserPrefMethods, % Note: this is not documented as supported for clients + userauth_supported_methods = UserPrefMethods, userauth_preference = ClientMethods0 } = Ssh0) -> case sort_select_mthds(ClientMethods0, UserPrefMethods, ServerMethods) of [] -> - Msg = #ssh_msg_disconnect{code = ?SSH_DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE, - description = "Unable to connect using the available authentication methods", - language = "en"}, - {disconnect, Msg, ssh_transport:ssh_packet(Msg, Ssh0)}; + {send_disconnect, ?SSH_DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE, Ssh0}; [{Pref,Module,Function,Args} | Prefs] -> Ssh = case Pref of @@ -196,11 +193,8 @@ init_userauth_request_msg(#ssh{opts = Opts} = Ssh) -> %% Client side case ?GET_OPT(user, Opts) of undefined -> - ErrStr = "Could not determine the users name", - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{code = ?SSH_DISCONNECT_ILLEGAL_USER_NAME, - description = ErrStr}); - + ?DISCONNECT(?SSH_DISCONNECT_ILLEGAL_USER_NAME, + "Could not determine the users name"); User -> ssh_transport:ssh_packet( #ssh_msg_userauth_request{user = User, @@ -451,11 +445,8 @@ handle_userauth_info_response({extra,#ssh_msg_userauth_info_response{}}, handle_userauth_info_response(#ssh_msg_userauth_info_response{}, _Auth) -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{code = ?SSH_DISCONNECT_SERVICE_NOT_AVAILABLE, - description = "Server does not support keyboard-interactive" - }). - + ?DISCONNECT(?SSH_DISCONNECT_SERVICE_NOT_AVAILABLE, + "Server does not support keyboard-interactive"). %%-------------------------------------------------------------------- %%% Internal functions @@ -492,10 +483,8 @@ check_password(User, Password, Opts, Ssh) -> {false,NewState} -> {false, Ssh#ssh{pwdfun_user_state=NewState}}; disconnect -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{code = ?SSH_DISCONNECT_SERVICE_NOT_AVAILABLE, - description = "Unable to connect using the available authentication methods" - }) + ?DISCONNECT(?SSH_DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE, + "") end end. @@ -591,16 +580,12 @@ keyboard_interact_fun(KbdInteractFun, Name, Instr, PromptInfos, NumPrompts) -> case KbdInteractFun(Name, Instr, Prompts) of Rs when length(Rs) == NumPrompts -> Rs; - Rs -> - throw({mismatching_number_of_responses, - {got,Rs}, - {expected, NumPrompts}, - #ssh_msg_disconnect{code = ?SSH_DISCONNECT_SERVICE_NOT_AVAILABLE, - description = "User interaction failed", - language = "en"}}) + _Rs -> + nok end. key_alg('rsa-sha2-256') -> 'ssh-rsa'; key_alg('rsa-sha2-512') -> 'ssh-rsa'; key_alg(Alg) -> Alg. + diff --git a/lib/ssh/src/ssh_channel.erl b/lib/ssh/src/ssh_channel.erl index 85b31f3669..81c495a815 100644 --- a/lib/ssh/src/ssh_channel.erl +++ b/lib/ssh/src/ssh_channel.erl @@ -22,6 +22,7 @@ -module(ssh_channel). +-include("ssh.hrl"). -include("ssh_connect.hrl"). -callback init(Args :: term()) -> @@ -49,331 +50,43 @@ {ok, NewState :: term()} | {error, Reason :: term()}. -callback handle_msg(Msg ::term(), State :: term()) -> - {ok, State::term()} | {stop, ChannelId::integer(), State::term()}. + {ok, State::term()} | {stop, ChannelId::ssh:channel_id(), State::term()}. --callback handle_ssh_msg({ssh_cm, ConnectionRef::term(), SshMsg::term()}, +-callback handle_ssh_msg({ssh_cm, ConnectionRef::ssh:connection_ref(), SshMsg::term()}, State::term()) -> {ok, State::term()} | - {stop, ChannelId::integer(), + {stop, ChannelId::ssh:channel_id(), State::term()}. --behaviour(gen_server). - %%% API -export([start/4, start/5, start_link/4, start_link/5, call/2, call/3, cast/2, reply/2, enter_loop/1]). -%% gen_server callbacks --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - -%% Internal application API --export([cache_create/0, cache_lookup/2, cache_update/2, - cache_delete/1, cache_delete/2, cache_foldl/3, - cache_info/2, cache_find/2, - get_print_info/1]). - --record(state, { - cm, - channel_cb, - channel_state, - channel_id, - close_sent = false - }). - %%==================================================================== %% API %%==================================================================== call(ChannelPid, Msg) -> - call(ChannelPid, Msg, infinity). + ssh_client_channel:call(ChannelPid, Msg). call(ChannelPid, Msg, TimeOute) -> - try gen_server:call(ChannelPid, Msg, TimeOute) of - Result -> - Result - catch - exit:{noproc, _} -> - {error, closed}; - exit:{normal, _} -> - {error, closed}; - exit:{shutdown, _} -> - {error, closed}; - exit:{{shutdown, _}, _} -> - {error, closed}; - exit:{timeout, _} -> - {error, timeout} - end. + ssh_client_channel:call(ChannelPid, Msg, TimeOute). cast(ChannelPid, Msg) -> - gen_server:cast(ChannelPid, Msg). - + ssh_client_channel:cast(ChannelPid, Msg). reply(From, Msg) -> - gen_server:reply(From, Msg). + ssh_client_channel:reply(From, Msg). -%%==================================================================== -%% Internal application API -%%==================================================================== - -%%-------------------------------------------------------------------- -%% Function: start_link() -> {ok,Pid} | ignore | {error,Error} -%% Description: Starts the server -%%-------------------------------------------------------------------- start(ConnectionManager, ChannelId, CallBack, CbInitArgs) -> - start(ConnectionManager, ChannelId, CallBack, CbInitArgs, undefined). + ssh_client_channel:start(ConnectionManager, ChannelId, CallBack, CbInitArgs). start(ConnectionManager, ChannelId, CallBack, CbInitArgs, Exec) -> - Options = [{channel_cb, CallBack}, - {channel_id, ChannelId}, - {init_args, CbInitArgs}, - {cm, ConnectionManager}, - {exec, Exec}], - gen_server:start(?MODULE, [Options], []). + ssh_client_channel:start(ConnectionManager, ChannelId, CallBack, CbInitArgs, Exec). start_link(ConnectionManager, ChannelId, CallBack, CbInitArgs) -> - start_link(ConnectionManager, ChannelId, CallBack, CbInitArgs, undefined). + ssh_client_channel:start_link(ConnectionManager, ChannelId, CallBack, CbInitArgs). start_link(ConnectionManager, ChannelId, CallBack, CbInitArgs, Exec) -> - Options = [{channel_cb, CallBack}, - {channel_id, ChannelId}, - {init_args, CbInitArgs}, - {cm, ConnectionManager}, - {exec, Exec}], - gen_server:start_link(?MODULE, [Options], []). + ssh_client_channel:start_link(ConnectionManager, ChannelId, CallBack, CbInitArgs, Exec). enter_loop(State) -> - gen_server:enter_loop(?MODULE, [], State). - -%%==================================================================== -%% gen_server callbacks -%%==================================================================== - -%%-------------------------------------------------------------------- -%% Function: init(Args) -> {ok, State} | -%% {ok, State, Timeout} | -%% ignore | -%% {stop, Reason} -%% Description: Initiates the server -%%-------------------------------------------------------------------- -init([Options]) -> - Cb = proplists:get_value(channel_cb, Options), - ConnectionManager = proplists:get_value(cm, Options), - ChannelId = proplists:get_value(channel_id, Options), - process_flag(trap_exit, true), - InitArgs = - case proplists:get_value(exec, Options) of - undefined -> - proplists:get_value(init_args, Options); - Exec -> - proplists:get_value(init_args, Options) ++ [Exec] - end, - try Cb:init(InitArgs) of - {ok, ChannelState} -> - State = #state{cm = ConnectionManager, - channel_cb = Cb, - channel_id = ChannelId, - channel_state = ChannelState}, - self() ! {ssh_channel_up, ChannelId, ConnectionManager}, - {ok, State}; - {ok, ChannelState, Timeout} -> - State = #state{cm = ConnectionManager, - channel_cb = Cb, - channel_id = ChannelId, - channel_state = ChannelState}, - self() ! {ssh_channel_up, ChannelId, ConnectionManager}, - {ok, State, Timeout}; - {stop, Why} -> - {stop, Why} - catch - _:Reason -> - {stop, Reason} - end. - -%%-------------------------------------------------------------------- -%% Function: %% handle_call(Request, From, State) -> {reply, Reply, State} | -%% {reply, Reply, State, Timeout} | -%% {noreply, State} | -%% {noreply, State, Timeout} | -%% {stop, Reason, Reply, State} | -%% {stop, Reason, State} -%% Description: Handling call messages -%%-------------------------------------------------------------------- -handle_call(get_print_info, _From, State) -> - Reply = - {{State#state.cm, - State#state.channel_id}, - io_lib:format('CB=~p',[State#state.channel_cb]) - }, - {reply, Reply, State}; - -handle_call(Request, From, #state{channel_cb = Module, - channel_state = ChannelState} = State) -> - try Module:handle_call(Request, From, ChannelState) of - Result -> - handle_cb_result(Result, State) - catch - error:{undef, _} -> - {noreply, State} - end. - - -%%-------------------------------------------------------------------- -%% Function: handle_cast(Msg, State) -> {noreply, State} | -%% {noreply, State, Timeout} | -%% {stop, Reason, State} -%% Description: Handling cast messages -%%-------------------------------------------------------------------- -handle_cast(Msg, #state{channel_cb = Module, - channel_state = ChannelState} = State) -> - - try Module:handle_cast(Msg, ChannelState) of - Result -> - handle_cb_result(Result, State) - catch - error:{undef, _} -> - {noreply, State} - end. - -%%-------------------------------------------------------------------- -%% Function: handle_info(Info, State) -> {noreply, State} | -%% {noreply, State, Timeout} | -%% {stop, Reason, State} -%% Description: Handling all non call/cast messages -%%-------------------------------------------------------------------- -handle_info({ssh_cm, ConnectionManager, {closed, _ChannelId}}, - #state{cm = ConnectionManager, - close_sent = true} = State) -> - {stop, normal, State}; -handle_info({ssh_cm, ConnectionManager, {closed, ChannelId}}, - #state{cm = ConnectionManager, - close_sent = false} = State) -> - %% To be on the safe side, i.e. the manager has already been terminated. - (catch ssh_connection:close(ConnectionManager, ChannelId)), - {stop, normal, State#state{close_sent = true}}; - -handle_info({ssh_cm, _, _} = Msg, #state{cm = ConnectionManager, - channel_cb = Module, - channel_state = ChannelState0} = State) -> - case Module:handle_ssh_msg(Msg, ChannelState0) of - {ok, ChannelState} -> - adjust_window(Msg), - {noreply, State#state{channel_state = ChannelState}}; - {ok, ChannelState, Timeout} -> - adjust_window(Msg), - {noreply, State#state{channel_state = ChannelState}, Timeout}; - {stop, ChannelId, ChannelState} -> - catch ssh_connection:close(ConnectionManager, ChannelId), - {stop, normal, State#state{close_sent = true, - channel_state = ChannelState}} - end; - -handle_info(Msg, #state{cm = ConnectionManager, channel_cb = Module, - channel_state = ChannelState0} = State) -> - case Module:handle_msg(Msg, ChannelState0) of - {ok, ChannelState} -> - {noreply, State#state{channel_state = ChannelState}}; - {ok, ChannelState, Timeout} -> - {noreply, State#state{channel_state = ChannelState}, Timeout}; - {stop, Reason, ChannelState} when is_atom(Reason)-> - {stop, Reason, State#state{close_sent = true, - channel_state = ChannelState}}; - {stop, ChannelId, ChannelState} -> - Reason = - case Msg of - {'EXIT', _Pid, shutdown} -> - shutdown; - _ -> - normal - end, - (catch ssh_connection:close(ConnectionManager, ChannelId)), - {stop, Reason, State#state{close_sent = true, - channel_state = ChannelState}} - end. - -%%-------------------------------------------------------------------- -%% Function: terminate(Reason, State) -> void() -%% Description: This function is called by a gen_server when it is about to -%% terminate. It should be the opposite of Module:init/1 and do any necessary -%% cleaning up. When it returns, the gen_server terminates with Reason. -%% The return value is ignored. -%%-------------------------------------------------------------------- -terminate(Reason, #state{cm = ConnectionManager, - channel_id = ChannelId, - close_sent = false} = State) -> - catch ssh_connection:close(ConnectionManager, ChannelId), - terminate(Reason, State#state{close_sent = true}); -terminate(_, #state{channel_cb = Cb, channel_state = ChannelState}) -> - catch Cb:terminate(Cb, ChannelState), - ok. - -%%-------------------------------------------------------------------- -%% Func: code_change(OldVsn, State, Extra) -> {ok, NewState} -%% Description: Convert process state when code is changed -%%-------------------------------------------------------------------- -code_change(OldVsn, #state{channel_cb = Module, - channel_state = ChannelState0} = State, Extra) -> - {ok, ChannelState} = Module:code_change(OldVsn, ChannelState0, Extra), - {ok, State#state{channel_state = ChannelState}}. - -%%==================================================================== -%% Internal application API -%%==================================================================== -cache_create() -> - ets:new(cm_tab, [set,{keypos, #channel.local_id}]). - -cache_lookup(Cache, Key) -> - case ets:lookup(Cache, Key) of - [Channel] -> - Channel; - [] -> - undefined - end. - -cache_update(Cache, #channel{local_id = Id} = Entry) when Id =/= undefined -> - ets:insert(Cache, Entry). - -cache_delete(Cache, Key) -> - ets:delete(Cache, Key). - -cache_delete(Cache) -> - ets:delete(Cache). - -cache_foldl(Fun, Acc, Cache) -> - ets:foldl(Fun, Acc, Cache). - -cache_info(num_entries, Cache) -> - proplists:get_value(size, ets:info(Cache)). - -cache_find(ChannelPid, Cache) -> - case ets:match_object(Cache, #channel{user = ChannelPid}) of - [] -> - undefined; - [Channel] -> - Channel - end. - -get_print_info(Pid) -> - call(Pid, get_print_info, 1000). - -%%-------------------------------------------------------------------- -%%% Internal functions -%%-------------------------------------------------------------------- -handle_cb_result({reply, Reply, ChannelState}, State) -> - {reply, Reply, State#state{channel_state = ChannelState}}; -handle_cb_result({reply, Reply, ChannelState, Timeout}, State) -> - {reply, Reply,State#state{channel_state = ChannelState}, Timeout}; -handle_cb_result({noreply, ChannelState}, State) -> - {noreply, State#state{channel_state = ChannelState}}; -handle_cb_result({noreply, ChannelState, Timeout}, State) -> - {noreply, State#state{channel_state = ChannelState}, Timeout}; -handle_cb_result({stop, Reason, Reply, ChannelState}, State) -> - {stop, Reason, Reply, State#state{channel_state = ChannelState}}; -handle_cb_result({stop, Reason, ChannelState}, State) -> - {stop, Reason, State#state{channel_state = ChannelState}}. - -adjust_window({ssh_cm, ConnectionManager, - {data, ChannelId, _, Data}}) -> - ssh_connection:adjust_window(ConnectionManager, ChannelId, size(Data)); -adjust_window(_) -> - ok. - - + ssh_client_channel:enter_loop(State). diff --git a/lib/ssh/src/ssh_cli.erl b/lib/ssh/src/ssh_cli.erl index 958c342f5f..fcc1d3d59f 100644 --- a/lib/ssh/src/ssh_cli.erl +++ b/lib/ssh/src/ssh_cli.erl @@ -25,14 +25,16 @@ -module(ssh_cli). --behaviour(ssh_daemon_channel). +-behaviour(ssh_server_channel). -include("ssh.hrl"). -include("ssh_connect.hrl"). -%% ssh_channel callbacks +%% ssh_server_channel callbacks -export([init/1, handle_ssh_msg/2, handle_msg/2, terminate/2]). +-export([dbg_trace/3]). + %% state -record(state, { cm, @@ -45,23 +47,8 @@ }). %%==================================================================== -%% ssh_channel callbacks +%% ssh_server_channel callbacks %%==================================================================== --spec init(Args :: term()) -> - {ok, State :: term()} | {ok, State :: term(), timeout() | hibernate} | - {stop, Reason :: term()} | ignore. - --spec terminate(Reason :: (normal | shutdown | {shutdown, term()} | - term()), - State :: term()) -> - term(). - --spec handle_msg(Msg ::term(), State :: term()) -> - {ok, State::term()} | {stop, ChannelId::integer(), State::term()}. --spec handle_ssh_msg({ssh_cm, ConnectionRef::term(), SshMsg::term()}, - State::term()) -> {ok, State::term()} | - {stop, ChannelId::integer(), - State::term()}. %%-------------------------------------------------------------------- %% Function: init(Args) -> {ok, State} @@ -118,42 +105,53 @@ handle_ssh_msg({ssh_cm, ConnectionHandler, write_chars(ConnectionHandler, ChannelId, Chars), {ok, State#state{pty = Pty, buf = NewBuf}}; -handle_ssh_msg({ssh_cm, ConnectionHandler, - {shell, ChannelId, WantReply}}, State) -> +handle_ssh_msg({ssh_cm, ConnectionHandler, {shell, ChannelId, WantReply}}, State) -> NewState = start_shell(ConnectionHandler, State), - ssh_connection:reply_request(ConnectionHandler, WantReply, - success, ChannelId), - {ok, NewState#state{channel = ChannelId, - cm = ConnectionHandler}}; - -handle_ssh_msg({ssh_cm, ConnectionHandler, - {exec, ChannelId, WantReply, Cmd}}, #state{exec=undefined, - shell=?DEFAULT_SHELL} = State) -> - {Reply, Status} = exec(Cmd), - write_chars(ConnectionHandler, - ChannelId, io_lib:format("~p\n", [Reply])), - ssh_connection:reply_request(ConnectionHandler, WantReply, - success, ChannelId), - ssh_connection:exit_status(ConnectionHandler, ChannelId, Status), - ssh_connection:send_eof(ConnectionHandler, ChannelId), - {stop, ChannelId, State#state{channel = ChannelId, cm = ConnectionHandler}}; - -handle_ssh_msg({ssh_cm, ConnectionHandler, - {exec, ChannelId, WantReply, _Cmd}}, #state{exec = undefined} = State) -> - write_chars(ConnectionHandler, ChannelId, 1, "Prohibited.\n"), ssh_connection:reply_request(ConnectionHandler, WantReply, success, ChannelId), - ssh_connection:exit_status(ConnectionHandler, ChannelId, 255), - ssh_connection:send_eof(ConnectionHandler, ChannelId), - {stop, ChannelId, State#state{channel = ChannelId, cm = ConnectionHandler}}; - -handle_ssh_msg({ssh_cm, ConnectionHandler, - {exec, ChannelId, WantReply, Cmd}}, State) -> - NewState = start_shell(ConnectionHandler, Cmd, State), - ssh_connection:reply_request(ConnectionHandler, WantReply, - success, ChannelId), {ok, NewState#state{channel = ChannelId, cm = ConnectionHandler}}; +handle_ssh_msg({ssh_cm, ConnectionHandler, {exec, ChannelId, WantReply, Cmd}}, S0) -> + case + case S0#state.exec of + {direct,F} -> + %% Exec called and a Fun or MFA is defined to use. The F returns the + %% value to return. + exec_direct(ConnectionHandler, F, Cmd); + + undefined when S0#state.shell == ?DEFAULT_SHELL -> + %% Exec called and the shell is the default shell (= Erlang shell). + %% To be exact, eval the term as an Erlang term (but not using the + %% ?DEFAULT_SHELL directly). This disables banner, prompts and such. + exec_in_erlang_default_shell(Cmd); + + undefined -> + %% Exec called, but the a shell other than the default shell is defined. + %% No new exec shell is defined, so don't execute! + %% We don't know if it is intended to use the new shell or not. + {"Prohibited.", 255, 1}; + + _ -> + %% Exec called and a Fun or MFA is defined to use. The F communicates via + %% standard io:write/read. + %% Kept for compatibility. + S1 = start_exec_shell(ConnectionHandler, Cmd, S0), + ssh_connection:reply_request(ConnectionHandler, WantReply, success, ChannelId), + {ok, S1} + end + of + {Reply, Status, Type} -> + write_chars(ConnectionHandler, ChannelId, Type, Reply), + ssh_connection:reply_request(ConnectionHandler, WantReply, success, ChannelId), + ssh_connection:exit_status(ConnectionHandler, ChannelId, Status), + ssh_connection:send_eof(ConnectionHandler, ChannelId), + {stop, ChannelId, S0#state{channel = ChannelId, cm = ConnectionHandler}}; + + {ok, S} -> + {ok, S#state{channel = ChannelId, + cm = ConnectionHandler}} + end; + handle_ssh_msg({ssh_cm, _ConnectionHandler, {eof, _ChannelId}}, State) -> {ok, State}; @@ -259,35 +257,7 @@ to_group(Data, Group) -> end, to_group(Tail, Group). -exec(Cmd) -> - case eval(parse(scan(Cmd))) of - {error, _} -> - {Cmd, 0}; %% This should be an external call - Term -> - Term - end. - -scan(Cmd) -> - erl_scan:string(Cmd). - -parse({ok, Tokens, _}) -> - erl_parse:parse_exprs(Tokens); -parse(Error) -> - Error. - -eval({ok, Expr_list}) -> - case (catch erl_eval:exprs(Expr_list, - erl_eval:new_bindings())) of - {value, Value, _NewBindings} -> - {Value, 0}; - {'EXIT', {Error, _}} -> - {Error, -1}; - Error -> - {Error, -1} - end; -eval(Error) -> - {Error, -1}. - +%%-------------------------------------------------------------------- %%% io_request, handle io requests from the user process, %%% Note, this is not the real I/O-protocol, but the mockup version %%% used between edlin and a user_driver. The protocol tags are @@ -506,53 +476,130 @@ bin_to_list(L) when is_list(L) -> bin_to_list(I) when is_integer(I) -> I. + +%%-------------------------------------------------------------------- start_shell(ConnectionHandler, State) -> - Shell = State#state.shell, - ConnectionInfo = ssh_connection_handler:connection_info(ConnectionHandler, - [peer, user]), - ShellFun = case is_function(Shell) of - true -> - User = proplists:get_value(user, ConnectionInfo), - case erlang:fun_info(Shell, arity) of - {arity, 1} -> - fun() -> Shell(User) end; - {arity, 2} -> - {_, PeerAddr} = proplists:get_value(peer, ConnectionInfo), - fun() -> Shell(User, PeerAddr) end; - _ -> - Shell - end; - _ -> - Shell - end, - Echo = get_echo(State#state.pty), - Group = group:start(self(), ShellFun, [{echo, Echo}]), - State#state{group = Group, buf = empty_buf()}. - -start_shell(_ConnectionHandler, Cmd, #state{exec={M, F, A}} = State) -> - Group = group:start(self(), {M, F, A++[Cmd]}, [{echo, false}]), - State#state{group = Group, buf = empty_buf()}; -start_shell(ConnectionHandler, Cmd, #state{exec=Shell} = State) when is_function(Shell) -> - - ConnectionInfo = ssh_connection_handler:connection_info(ConnectionHandler, - [peer, user]), - User = proplists:get_value(user, ConnectionInfo), - ShellFun = - case erlang:fun_info(Shell, arity) of - {arity, 1} -> - fun() -> Shell(Cmd) end; - {arity, 2} -> - fun() -> Shell(Cmd, User) end; - {arity, 3} -> - {_, PeerAddr} = proplists:get_value(peer, ConnectionInfo), - fun() -> Shell(Cmd, User, PeerAddr) end; - _ -> - Shell - end, - Echo = get_echo(State#state.pty), - Group = group:start(self(), ShellFun, [{echo,Echo}]), - State#state{group = Group, buf = empty_buf()}. + ShellSpawner = + case State#state.shell of + Shell when is_function(Shell, 1) -> + [{user,User}] = ssh_connection_handler:connection_info(ConnectionHandler, [user]), + fun() -> Shell(User) end; + Shell when is_function(Shell, 2) -> + ConnectionInfo = + ssh_connection_handler:connection_info(ConnectionHandler, [peer, user]), + User = proplists:get_value(user, ConnectionInfo), + {_, PeerAddr} = proplists:get_value(peer, ConnectionInfo), + fun() -> Shell(User, PeerAddr) end; + {_,_,_} = Shell -> + Shell + end, + State#state{group = group:start(self(), ShellSpawner, [{echo, get_echo(State#state.pty)}]), + buf = empty_buf()}. + +%%-------------------------------------------------------------------- +start_exec_shell(ConnectionHandler, Cmd, State) -> + ExecShellSpawner = + case State#state.exec of + ExecShell when is_function(ExecShell, 1) -> + fun() -> ExecShell(Cmd) end; + ExecShell when is_function(ExecShell, 2) -> + [{user,User}] = ssh_connection_handler:connection_info(ConnectionHandler, [user]), + fun() -> ExecShell(Cmd, User) end; + ExecShell when is_function(ExecShell, 3) -> + ConnectionInfo = + ssh_connection_handler:connection_info(ConnectionHandler, [peer, user]), + User = proplists:get_value(user, ConnectionInfo), + {_, PeerAddr} = proplists:get_value(peer, ConnectionInfo), + fun() -> ExecShell(Cmd, User, PeerAddr) end; + {M,F,A} -> + {M, F, A++[Cmd]} + end, + State#state{group = group:start(self(), ExecShellSpawner, [{echo,false}]), + buf = empty_buf()}. + +%%-------------------------------------------------------------------- +exec_in_erlang_default_shell(Cmd) -> + case eval(parse(scan(Cmd))) of + {ok, Term} -> + {io_lib:format("~p\n", [Term]), 0, 0}; + {error, Error} when is_atom(Error) -> + {io_lib:format("Error in ~p: ~p\n", [Cmd,Error]), -1, 1}; + _ -> + {io_lib:format("Error: ~p\n", [Cmd]), -1, 1} + end. + +scan(Cmd) -> + erl_scan:string(Cmd). + +parse({ok, Tokens, _}) -> + erl_parse:parse_exprs(Tokens); +parse(Error) -> + Error. +eval({ok, Expr_list}) -> + case (catch erl_eval:exprs(Expr_list, + erl_eval:new_bindings())) of + {value, Value, _NewBindings} -> + {ok, Value}; + {'EXIT', {Error, _}} -> + {error, Error}; + {error, Error} -> + {error, Error}; + Error -> + {error, Error} + end; +eval({error,Error}) -> + {error, Error}; +eval(Error) -> + {error, Error}. + +%%-------------------------------------------------------------------- +exec_direct(ConnectionHandler, ExecSpec, Cmd) -> + try + case ExecSpec of + _ when is_function(ExecSpec, 1) -> + ExecSpec(Cmd); + _ when is_function(ExecSpec, 2) -> + [{user,User}] = ssh_connection_handler:connection_info(ConnectionHandler, [user]), + ExecSpec(Cmd, User); + _ when is_function(ExecSpec, 3) -> + ConnectionInfo = + ssh_connection_handler:connection_info(ConnectionHandler, [peer, user]), + User = proplists:get_value(user, ConnectionInfo), + {_, PeerAddr} = proplists:get_value(peer, ConnectionInfo), + ExecSpec(Cmd, User, PeerAddr) + end + of + Reply -> + return_direct_exec_reply(Reply, Cmd) + catch + C:Error -> + {io_lib:format("Error in \"~s\": ~p ~p~n", [Cmd,C,Error]), -1, 1} + end. + + + +return_direct_exec_reply(Reply, Cmd) -> + case fmt_exec_repl(Reply) of + {ok,S} -> + {S, 0, 0}; + {error,S} -> + {io_lib:format("Error in \"~s\": ~s~n", [Cmd,S]), -1, 1} + end. + +fmt_exec_repl({T,A}) when T==ok ; T==error -> + try + {T, io_lib:format("~s",[A])} + catch + error:badarg -> + {T, io_lib:format("~p", [A])}; + C:Err -> + {error, io_lib:format("~p:~p~n",[C,Err])} + end; +fmt_exec_repl(Other) -> + {error, io_lib:format("Bad exec-plugin return: ~p",[Other])}. + +%%-------------------------------------------------------------------- % Pty can be undefined if the client never sets any pty options before % starting the shell. get_echo(undefined) -> @@ -578,3 +625,19 @@ not_zero(0, B) -> not_zero(A, _) -> A. +%%%################################################################ +%%%# +%%%# Tracing +%%%# + +dbg_trace(points, _, _) -> [terminate]; + +dbg_trace(flags, terminate, _) -> [c]; +dbg_trace(on, terminate, _) -> dbg:tp(?MODULE, terminate, 2, x); +dbg_trace(off, terminate, _) -> dbg:ctpg(?MODULE, terminate, 2); +dbg_trace(format, terminate, {call, {?MODULE,terminate, [Reason, State]}}) -> + ["Cli Terminating:\n", + io_lib:format("Reason: ~p,~nState:~n~s", [Reason, wr_record(State)]) + ]. + +?wr_record(state). diff --git a/lib/ssh/src/ssh_client_channel.erl b/lib/ssh/src/ssh_client_channel.erl new file mode 100644 index 0000000000..f20007baaf --- /dev/null +++ b/lib/ssh/src/ssh_client_channel.erl @@ -0,0 +1,456 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2008-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% + +-module(ssh_client_channel). + +-include("ssh.hrl"). +-include("ssh_connect.hrl"). + +-callback init(Args :: term()) -> + {ok, State :: term()} | {ok, State :: term(), timeout() | hibernate} | + {stop, Reason :: term()} | ignore. +-callback handle_call(Request :: term(), From :: {pid(), Tag :: term()}, + State :: term()) -> + {reply, Reply :: term(), NewState :: term()} | + {reply, Reply :: term(), NewState :: term(), timeout() | hibernate} | + {noreply, NewState :: term()} | + {noreply, NewState :: term(), timeout() | hibernate} | + {stop, Reason :: term(), Reply :: term(), NewState :: term()} | + {stop, Reason :: term(), NewState :: term()}. +-callback handle_cast(Request :: term(), State :: term()) -> + {noreply, NewState :: term()} | + {noreply, NewState :: term(), timeout() | hibernate} | + {stop, Reason :: term(), NewState :: term()}. + +-callback terminate(Reason :: (normal | shutdown | {shutdown, term()} | + term()), + State :: term()) -> + term(). +-callback code_change(OldVsn :: (term() | {down, term()}), State :: term(), + Extra :: term()) -> + {ok, NewState :: term()} | {error, Reason :: term()}. + +-callback handle_msg(Msg ::term(), State :: term()) -> + {ok, State::term()} | {stop, ChannelId::ssh:channel_id(), State::term()}. + +-callback handle_ssh_msg({ssh_cm, ConnectionRef::ssh:connection_ref(), SshMsg::term()}, + State::term()) -> {ok, State::term()} | + {stop, ChannelId::ssh:channel_id(), + State::term()}. +-behaviour(gen_server). + +%%% API +-export([start/4, start/5, start_link/4, start_link/5, call/2, call/3, + cast/2, reply/2, enter_loop/1]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +%% Internal application API +-export([cache_create/0, cache_lookup/2, cache_update/2, + cache_delete/1, cache_delete/2, cache_foldl/3, + cache_info/2, cache_find/2, + get_print_info/1]). + +-export([dbg_trace/3]). + +-record(state, { + cm, + channel_cb, + channel_state, + channel_id, + close_sent = false + }). + +%%==================================================================== +%% API +%%==================================================================== + +call(ChannelPid, Msg) -> + call(ChannelPid, Msg, infinity). + +call(ChannelPid, Msg, TimeOute) -> + try gen_server:call(ChannelPid, Msg, TimeOute) of + Result -> + Result + catch + exit:{noproc, _} -> + {error, closed}; + exit:{normal, _} -> + {error, closed}; + exit:{shutdown, _} -> + {error, closed}; + exit:{{shutdown, _}, _} -> + {error, closed}; + exit:{timeout, _} -> + {error, timeout} + end. + +cast(ChannelPid, Msg) -> + gen_server:cast(ChannelPid, Msg). + + +reply(From, Msg) -> + gen_server:reply(From, Msg). + +%%==================================================================== +%% Internal application API +%%==================================================================== + +%%-------------------------------------------------------------------- +%% Function: start_link() -> {ok,Pid} | ignore | {error,Error} +%% Description: Starts the server +%%-------------------------------------------------------------------- +start(ConnectionManager, ChannelId, CallBack, CbInitArgs) -> + start(ConnectionManager, ChannelId, CallBack, CbInitArgs, undefined). + +start(ConnectionManager, ChannelId, CallBack, CbInitArgs, Exec) -> + Options = [{channel_cb, CallBack}, + {channel_id, ChannelId}, + {init_args, CbInitArgs}, + {cm, ConnectionManager}, + {exec, Exec}], + gen_server:start(?MODULE, [Options], []). + +start_link(ConnectionManager, ChannelId, CallBack, CbInitArgs) -> + start_link(ConnectionManager, ChannelId, CallBack, CbInitArgs, undefined). + +start_link(ConnectionManager, ChannelId, CallBack, CbInitArgs, Exec) -> + Options = [{channel_cb, CallBack}, + {channel_id, ChannelId}, + {init_args, CbInitArgs}, + {cm, ConnectionManager}, + {exec, Exec}], + gen_server:start_link(?MODULE, [Options], []). + +enter_loop(State) -> + gen_server:enter_loop(?MODULE, [], State). + +%%==================================================================== +%% gen_server callbacks +%%==================================================================== + +%%-------------------------------------------------------------------- +%% Function: init(Args) -> {ok, State} | +%% {ok, State, Timeout} | +%% ignore | +%% {stop, Reason} +%% Description: Initiates the server +%%-------------------------------------------------------------------- +init([Options]) -> + Cb = proplists:get_value(channel_cb, Options), + ConnectionManager = proplists:get_value(cm, Options), + ChannelId = proplists:get_value(channel_id, Options), + process_flag(trap_exit, true), + try Cb:init(channel_cb_init_args(Options)) of + {ok, ChannelState} -> + State = #state{cm = ConnectionManager, + channel_cb = Cb, + channel_id = ChannelId, + channel_state = ChannelState}, + self() ! {ssh_channel_up, ChannelId, ConnectionManager}, + {ok, State}; + {ok, ChannelState, Timeout} -> + State = #state{cm = ConnectionManager, + channel_cb = Cb, + channel_id = ChannelId, + channel_state = ChannelState}, + self() ! {ssh_channel_up, ChannelId, ConnectionManager}, + {ok, State, Timeout}; + {stop, Why} -> + {stop, Why} + catch + _:Reason -> + {stop, Reason} + end. + +channel_cb_init_args(Options) -> + case proplists:get_value(exec, Options) of + undefined -> + proplists:get_value(init_args, Options); + Exec -> + proplists:get_value(init_args, Options) ++ [Exec] + end. + +%%-------------------------------------------------------------------- +%% Function: %% handle_call(Request, From, State) -> {reply, Reply, State} | +%% {reply, Reply, State, Timeout} | +%% {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, Reply, State} | +%% {stop, Reason, State} +%% Description: Handling call messages +%%-------------------------------------------------------------------- +handle_call(get_print_info, _From, State) -> + Reply = + {{State#state.cm, + State#state.channel_id}, + io_lib:format('CB=~p',[State#state.channel_cb]) + }, + {reply, Reply, State}; + +handle_call(Request, From, #state{channel_cb = Module, + channel_state = ChannelState} = State) -> + try Module:handle_call(Request, From, ChannelState) of + Result -> + handle_cb_result(Result, State) + catch + error:{undef, _} -> + {noreply, State} + end. + + +%%-------------------------------------------------------------------- +%% Function: handle_cast(Msg, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% Description: Handling cast messages +%%-------------------------------------------------------------------- +handle_cast(Msg, #state{channel_cb = Module, + channel_state = ChannelState} = State) -> + + try Module:handle_cast(Msg, ChannelState) of + Result -> + handle_cb_result(Result, State) + catch + error:{undef, _} -> + {noreply, State} + end. + +%%-------------------------------------------------------------------- +%% Function: handle_info(Info, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% Description: Handling all non call/cast messages +%%-------------------------------------------------------------------- +handle_info({ssh_cm, ConnectionManager, {closed, _ChannelId}}, + #state{cm = ConnectionManager, + close_sent = true} = State) -> + {stop, normal, State}; +handle_info({ssh_cm, ConnectionManager, {closed, ChannelId}}, + #state{cm = ConnectionManager, + close_sent = false} = State) -> + %% To be on the safe side, i.e. the manager has already been terminated. + (catch ssh_connection:close(ConnectionManager, ChannelId)), + {stop, normal, State#state{close_sent = true}}; + +handle_info({ssh_cm, _, _} = Msg, #state{cm = ConnectionManager, + channel_cb = Module, + channel_state = ChannelState0} = State) -> + case Module:handle_ssh_msg(Msg, ChannelState0) of + {ok, ChannelState} -> + adjust_window(Msg), + {noreply, State#state{channel_state = ChannelState}}; + {ok, ChannelState, Timeout} -> + adjust_window(Msg), + {noreply, State#state{channel_state = ChannelState}, Timeout}; + {stop, ChannelId, ChannelState} -> + catch ssh_connection:close(ConnectionManager, ChannelId), + {stop, normal, State#state{close_sent = true, + channel_state = ChannelState}} + end; + +handle_info(Msg, #state{cm = ConnectionManager, channel_cb = Module, + channel_state = ChannelState0} = State) -> + case Module:handle_msg(Msg, ChannelState0) of + {ok, ChannelState} -> + {noreply, State#state{channel_state = ChannelState}}; + {ok, ChannelState, Timeout} -> + {noreply, State#state{channel_state = ChannelState}, Timeout}; + {stop, Reason, ChannelState} when is_atom(Reason)-> + {stop, Reason, State#state{close_sent = true, + channel_state = ChannelState}}; + {stop, ChannelId, ChannelState} -> + Reason = + case Msg of + {'EXIT', _Pid, shutdown} -> + shutdown; + _ -> + normal + end, + (catch ssh_connection:close(ConnectionManager, ChannelId)), + {stop, Reason, State#state{close_sent = true, + channel_state = ChannelState}} + end. + +%%-------------------------------------------------------------------- +%% Function: terminate(Reason, State) -> void() +%% Description: This function is called by a gen_server when it is about to +%% terminate. It should be the opposite of Module:init/1 and do any necessary +%% cleaning up. When it returns, the gen_server terminates with Reason. +%% The return value is ignored. +%%-------------------------------------------------------------------- +terminate(Reason, #state{cm = ConnectionManager, + channel_id = ChannelId, + close_sent = false} = State) -> + catch ssh_connection:close(ConnectionManager, ChannelId), + terminate(Reason, State#state{close_sent = true}); +terminate(_, #state{channel_cb = Cb, channel_state = ChannelState}) -> + catch Cb:terminate(Cb, ChannelState), + ok. + +%%-------------------------------------------------------------------- +%% Func: code_change(OldVsn, State, Extra) -> {ok, NewState} +%% Description: Convert process state when code is changed +%%-------------------------------------------------------------------- +code_change(OldVsn, #state{channel_cb = Module, + channel_state = ChannelState0} = State, Extra) -> + {ok, ChannelState} = Module:code_change(OldVsn, ChannelState0, Extra), + {ok, State#state{channel_state = ChannelState}}. + +%%==================================================================== +%% Internal application API +%%==================================================================== +cache_create() -> + ets:new(cm_tab, [set,{keypos, #channel.local_id}]). + +cache_lookup(Cache, Key) -> + case ets:lookup(Cache, Key) of + [Channel] -> + Channel; + [] -> + undefined + end. + +cache_update(Cache, #channel{local_id = Id} = Entry) when Id =/= undefined -> + ets:insert(Cache, Entry). + +cache_delete(Cache, Key) -> + ets:delete(Cache, Key). + +cache_delete(Cache) -> + ets:delete(Cache). + +cache_foldl(Fun, Acc, Cache) -> + ets:foldl(Fun, Acc, Cache). + +cache_info(num_entries, Cache) -> + proplists:get_value(size, ets:info(Cache)). + +cache_find(ChannelPid, Cache) -> + case ets:match_object(Cache, #channel{user = ChannelPid}) of + [] -> + undefined; + [Channel] -> + Channel + end. + +get_print_info(Pid) -> + call(Pid, get_print_info, 1000). + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- +handle_cb_result({reply, Reply, ChannelState}, State) -> + {reply, Reply, State#state{channel_state = ChannelState}}; +handle_cb_result({reply, Reply, ChannelState, Timeout}, State) -> + {reply, Reply,State#state{channel_state = ChannelState}, Timeout}; +handle_cb_result({noreply, ChannelState}, State) -> + {noreply, State#state{channel_state = ChannelState}}; +handle_cb_result({noreply, ChannelState, Timeout}, State) -> + {noreply, State#state{channel_state = ChannelState}, Timeout}; +handle_cb_result({stop, Reason, Reply, ChannelState}, State) -> + {stop, Reason, Reply, State#state{channel_state = ChannelState}}; +handle_cb_result({stop, Reason, ChannelState}, State) -> + {stop, Reason, State#state{channel_state = ChannelState}}. + +adjust_window({ssh_cm, ConnectionManager, + {data, ChannelId, _, Data}}) -> + ssh_connection:adjust_window(ConnectionManager, ChannelId, size(Data)); +adjust_window(_) -> + ok. + + +%%%################################################################ +%%%# +%%%# Tracing +%%%# + +dbg_trace(points, _, _) -> [terminate, channels, channel_events]; + + +dbg_trace(flags, channels, A) -> [c] ++ dbg_trace(flags, terminate, A); +dbg_trace(on, channels, A) -> dbg:tp(?MODULE, init, 1, x), + dbg_trace(on, terminate, A); +dbg_trace(off, channels, A) -> dbg:ctpg(?MODULE, init, 1), + dbg_trace(off, terminate, A); +dbg_trace(format, channels, {call, {?MODULE,init, [[KVs]]}}) -> + ["Server Channel Starting:\n", + io_lib:format("Connection: ~p, ChannelId: ~p, CallBack: ~p\nCallBack init args = ~p", + [proplists:get_value(K,KVs) || K <- [cm, channel_id, channel_cb]] + ++ [channel_cb_init_args(KVs)]) + ]; +dbg_trace(format, channels, {return_from, {?MODULE,init,1}, {stop,Reason}}) -> + ["Server Channel Start FAILED!\n", + io_lib:format("Reason = ~p", [Reason]) + ]; +dbg_trace(format, channels, F) -> + dbg_trace(format, terminate, F); + + +dbg_trace(flags, terminate, _) -> [c]; +dbg_trace(on, terminate, _) -> dbg:tp(?MODULE, terminate, 2, x); +dbg_trace(off, terminate, _) -> dbg:ctpg(?MODULE, terminate, 2); +dbg_trace(format, terminate, {call, {?MODULE,terminate, [Reason, State]}}) -> + ["Server Channel Terminating:\n", + io_lib:format("Reason: ~p,~nState:~n~s", [Reason, wr_record(State)]) + ]; + +dbg_trace(flags, channel_events, _) -> [c]; +dbg_trace(on, channel_events, _) -> dbg:tp(?MODULE, handle_call, 3, x), + dbg:tp(?MODULE, handle_cast, 2, x), + dbg:tp(?MODULE, handle_info, 2, x); +dbg_trace(off, channel_events, _) -> dbg:ctpg(?MODULE, handle_call, 3), + dbg:ctpg(?MODULE, handle_cast, 2), + dbg:ctpg(?MODULE, handle_info, 2); +dbg_trace(format, channel_events, {call, {?MODULE,handle_call, [Call,From,State]}}) -> + [hdr("is called", State), + io_lib:format("From: ~p~nCall: ~p~n", [From, Call]) + ]; +dbg_trace(format, channel_events, {return_from, {?MODULE,handle_call,3}, Ret}) -> + ["Server Channel call returned:\n", + io_lib:format("~p~n", [ssh_dbg:reduce_state(Ret)]) + ]; +dbg_trace(format, channel_events, {call, {?MODULE,handle_cast, [Cast,State]}}) -> + [hdr("got cast", State), + io_lib:format("Cast: ~p~n", [Cast]) + ]; +dbg_trace(format, channel_events, {return_from, {?MODULE,handle_cast,2}, Ret}) -> + ["Server Channel cast returned:\n", + io_lib:format("~p~n", [ssh_dbg:reduce_state(Ret)]) + ]; +dbg_trace(format, channel_events, {call, {?MODULE,handle_info, [Info,State]}}) -> + [hdr("got info", State), + io_lib:format("Info: ~p~n", [Info]) + ]; +dbg_trace(format, channel_events, {return_from, {?MODULE,handle_info,2}, Ret}) -> + ["Server Channel info returned:\n", + io_lib:format("~p~n", [ssh_dbg:reduce_state(Ret)]) + ]. + +hdr(Title, S) -> + io_lib:format("Server Channel (Id=~p, CB=~p) ~s:\n", [S#state.channel_id, S#state.channel_cb, Title]). + +?wr_record(state). + + diff --git a/lib/ssh/src/ssh_client_key_api.erl b/lib/ssh/src/ssh_client_key_api.erl index 6e994ff292..d0d8ab25d6 100644 --- a/lib/ssh/src/ssh_client_key_api.erl +++ b/lib/ssh/src/ssh_client_key_api.erl @@ -23,26 +23,25 @@ -include_lib("public_key/include/public_key.hrl"). -include("ssh.hrl"). --export_type([algorithm/0]). - --type algorithm() :: 'ssh-rsa' - | 'ssh-dss' - | 'ecdsa-sha2-nistp256' - | 'ecdsa-sha2-nistp384' - | 'ecdsa-sha2-nistp521' - . - --callback is_host_key(PublicKey :: public_key:public_key(), - Host :: string(), - Algorithm :: algorithm(), - ConnectOptions :: proplists:proplist()) -> +-export_type([client_key_cb_options/0]). + +-type client_key_cb_options() :: [{key_cb_private,term()} | ssh:client_option()]. + +-callback is_host_key(Key :: public_key:public_key(), + Host :: string(), + Algorithm :: ssh:pubkey_alg(), + Options :: client_key_cb_options() + ) -> boolean(). --callback user_key(Algorithm :: algorithm(), - ConnectOptions :: proplists:proplist()) -> - {ok, PrivateKey::public_key:private_key()} | {error, term()}. +-callback user_key(Algorithm :: ssh:pubkey_alg(), + Options :: client_key_cb_options() + ) -> + {ok, PrivateKey :: public_key:private_key()} | {error, string()}. --callback add_host_key(Host :: string(), PublicKey :: public_key:public_key(), - Options :: proplists:proplist()) -> +-callback add_host_key(Host :: string(), + PublicKey :: public_key:public_key(), + Options :: client_key_cb_options() + ) -> ok | {error, Error::term()}. diff --git a/lib/ssh/src/ssh_connect.hrl b/lib/ssh/src/ssh_connect.hrl index a8de5f9a2f..3c61638285 100644 --- a/lib/ssh/src/ssh_connect.hrl +++ b/lib/ssh/src/ssh_connect.hrl @@ -22,10 +22,6 @@ %%% Description : SSH connection protocol --type channel_id() :: pos_integer(). --type connection_ref() :: pid(). - - -define(DEFAULT_PACKET_SIZE, 65536). -define(DEFAULT_WINDOW_SIZE, 10*?DEFAULT_PACKET_SIZE). diff --git a/lib/ssh/src/ssh_connection.erl b/lib/ssh/src/ssh_connection.erl index 946ae2967b..ed03b4e2ed 100644 --- a/lib/ssh/src/ssh_connection.erl +++ b/lib/ssh/src/ssh_connection.erl @@ -40,40 +40,56 @@ -export([window_change/4, window_change/6, signal/3, exit_status/3]). -%% Internal application API --export([channel_data/5, handle_msg/3, channel_eof_msg/1, - channel_close_msg/1, channel_success_msg/1, channel_failure_msg/1, +%% Internal SSH application API +-export([channel_data/5, + handle_msg/3, + handle_stop/1, + + channel_adjust_window_msg/2, + channel_close_msg/1, + channel_open_failure_msg/4, + channel_open_msg/5, channel_status_msg/1, - channel_adjust_window_msg/2, channel_data_msg/3, - channel_open_msg/5, channel_open_confirmation_msg/4, - channel_open_failure_msg/4, channel_request_msg/4, + channel_data_msg/3, + channel_eof_msg/1, + channel_failure_msg/1, + channel_open_confirmation_msg/4, + channel_request_msg/4, + channel_success_msg/1, + request_failure_msg/0, - request_success_msg/1, bind/4, unbind/3, unbind_channel/2, - bound_channel/3, encode_ip/1]). + request_success_msg/1, + + bind/4, unbind/3, unbind_channel/2, + bound_channel/3, encode_ip/1 + ]). + +-type connection_ref() :: ssh:connection_ref(). +-type channel_id() :: ssh:channel_id(). %%-------------------------------------------------------------------- %%% API %%-------------------------------------------------------------------- %%-------------------------------------------------------------------- --spec session_channel(connection_ref(), timeout()) -> {ok, channel_id()} | {error, timeout | closed}. --spec session_channel(connection_ref(), integer(), integer(), timeout()) -> {ok, channel_id()} | {error, timeout | closed}. - %% Description: Opens a channel for a ssh session. A session is a %% remote execution of a program. The program may be a shell, an %% application, a system command, or some built-in subsystem. %% -------------------------------------------------------------------- +-spec session_channel(connection_ref(), timeout()) -> + {ok, channel_id()} | {error, timeout | closed}. + session_channel(ConnectionHandler, Timeout) -> - session_channel(ConnectionHandler, - ?DEFAULT_WINDOW_SIZE, ?DEFAULT_PACKET_SIZE, - Timeout). + session_channel(ConnectionHandler, ?DEFAULT_WINDOW_SIZE, ?DEFAULT_PACKET_SIZE, Timeout). -session_channel(ConnectionHandler, InitialWindowSize, - MaxPacketSize, Timeout) -> +-spec session_channel(connection_ref(), integer(), integer(), timeout()) -> + {ok, channel_id()} | {error, timeout | closed}. + +session_channel(ConnectionHandler, InitialWindowSize, MaxPacketSize, Timeout) -> case ssh_connection_handler:open_channel(ConnectionHandler, "session", <<>>, - InitialWindowSize, - MaxPacketSize, Timeout) of + InitialWindowSize, + MaxPacketSize, Timeout) of {open, Channel} -> {ok, Channel}; Error -> @@ -81,55 +97,63 @@ session_channel(ConnectionHandler, InitialWindowSize, end. %%-------------------------------------------------------------------- --spec exec(connection_ref(), channel_id(), string(), timeout()) -> - success | failure | {error, timeout | closed}. - %% Description: Will request that the server start the %% execution of the given command. %%-------------------------------------------------------------------- +-spec exec(connection_ref(), channel_id(), string(), timeout()) -> + success | failure | {error, timeout | closed}. + exec(ConnectionHandler, ChannelId, Command, TimeOut) -> ssh_connection_handler:request(ConnectionHandler, self(), ChannelId, "exec", true, [?string(Command)], TimeOut). %%-------------------------------------------------------------------- --spec shell(connection_ref(), channel_id()) -> _. - %% Description: Will request that the user's default shell (typically %% defined in /etc/passwd in UNIX systems) be started at the other %% end. %%-------------------------------------------------------------------- +-spec shell(connection_ref(), channel_id()) -> + ok | success | failure | {error, timeout}. + shell(ConnectionHandler, ChannelId) -> ssh_connection_handler:request(ConnectionHandler, self(), ChannelId, "shell", false, <<>>, 0). %%-------------------------------------------------------------------- --spec subsystem(connection_ref(), channel_id(), string(), timeout()) -> - success | failure | {error, timeout | closed}. %% %% Description: Executes a predefined subsystem. %%-------------------------------------------------------------------- +-spec subsystem(connection_ref(), channel_id(), string(), timeout()) -> + success | failure | {error, timeout | closed}. + subsystem(ConnectionHandler, ChannelId, SubSystem, TimeOut) -> ssh_connection_handler:request(ConnectionHandler, self(), ChannelId, "subsystem", true, [?string(SubSystem)], TimeOut). %%-------------------------------------------------------------------- --spec send(connection_ref(), channel_id(), iodata()) -> - ok | {error, closed}. --spec send(connection_ref(), channel_id(), integer()| iodata(), timeout() | iodata()) -> - ok | {error, timeout} | {error, closed}. --spec send(connection_ref(), channel_id(), integer(), iodata(), timeout()) -> - ok | {error, timeout} | {error, closed}. -%% -%% %% Description: Sends channel data. %%-------------------------------------------------------------------- +-spec send(connection_ref(), channel_id(), iodata()) -> + ok | {error, timeout | closed}. send(ConnectionHandler, ChannelId, Data) -> send(ConnectionHandler, ChannelId, 0, Data, infinity). + + +-spec send(connection_ref(), channel_id(), integer()| iodata(), timeout() | iodata()) -> + ok | {error, timeout | closed}. + send(ConnectionHandler, ChannelId, Data, TimeOut) when is_integer(TimeOut) -> send(ConnectionHandler, ChannelId, 0, Data, TimeOut); + send(ConnectionHandler, ChannelId, Data, infinity) -> send(ConnectionHandler, ChannelId, 0, Data, infinity); + send(ConnectionHandler, ChannelId, Type, Data) -> send(ConnectionHandler, ChannelId, Type, Data, infinity). + + +-spec send(connection_ref(), channel_id(), integer(), iodata(), timeout()) -> + ok | {error, timeout | closed}. + send(ConnectionHandler, ChannelId, Type, Data, TimeOut) -> ssh_connection_handler:send(ConnectionHandler, ChannelId, Type, Data, TimeOut). @@ -143,7 +167,7 @@ send_eof(ConnectionHandler, Channel) -> ssh_connection_handler:send_eof(ConnectionHandler, Channel). %%-------------------------------------------------------------------- --spec adjust_window(connection_ref(), channel_id(), integer()) -> ok | {error, closed}. +-spec adjust_window(connection_ref(), channel_id(), integer()) -> ok. %% %% %% Description: Adjusts the ssh flowcontrol window. @@ -185,17 +209,18 @@ reply_request(_,false, _, _) -> ok. %%-------------------------------------------------------------------- --spec ptty_alloc(connection_ref(), channel_id(), proplists:proplist()) -> - success | failiure | {error, closed}. --spec ptty_alloc(connection_ref(), channel_id(), proplists:proplist(), timeout()) -> - success | failiure | {error, timeout} | {error, closed}. - -%% -%% %% Description: Sends a ssh connection protocol pty_req. %%-------------------------------------------------------------------- +-spec ptty_alloc(connection_ref(), channel_id(), proplists:proplist()) -> + success | failure | {error, timeout}. + ptty_alloc(ConnectionHandler, Channel, Options) -> ptty_alloc(ConnectionHandler, Channel, Options, infinity). + + +-spec ptty_alloc(connection_ref(), channel_id(), proplists:proplist(), timeout()) -> + success | failure | {error, timeout | closed}. + ptty_alloc(ConnectionHandler, Channel, Options0, TimeOut) -> TermData = backwards_compatible(Options0, []), % FIXME {Width, PixWidth} = pty_default_dimensions(width, TermData), @@ -232,33 +257,21 @@ exit_status(ConnectionHandler, Channel, Status) -> "exit-status", false, [?uint32(Status)], 0). %%-------------------------------------------------------------------- -%%% Internal API +%%% Internal, that is, ssh application internal API %%-------------------------------------------------------------------- -l2b(L) when is_integer(hd(L)) -> - try list_to_binary(L) - of - B -> B - catch - _:_ -> - unicode:characters_to_binary(L) - end; -l2b([H|T]) -> - << (l2b(H))/binary, (l2b(T))/binary >>; -l2b(B) when is_binary(B) -> - B; -l2b([]) -> - <<>>. - +%%%---------------------------------------------------------------- +%%% Send data on a channel/connection as result of for example +%%% ssh_connection:send (executed in the ssh_connection_state machine) +%%% -channel_data(ChannelId, DataType, Data, Connection, From) - when is_list(Data)-> +channel_data(ChannelId, DataType, Data, Connection, From) when is_list(Data)-> channel_data(ChannelId, DataType, l2b(Data), Connection, From); channel_data(ChannelId, DataType, Data, #connection{channel_cache = Cache} = Connection, From) -> - case ssh_channel:cache_lookup(Cache, ChannelId) of + case ssh_client_channel:cache_lookup(Cache, ChannelId) of #channel{remote_id = Id, sent_close = false} = Channel0 -> {SendList, Channel} = update_send_window(Channel0#channel{flow_control = From}, DataType, @@ -271,11 +284,18 @@ channel_data(ChannelId, DataType, Data, SendData)} end, SendList), FlowCtrlMsgs = flow_control(Replies, Channel, Cache), - {{replies, Replies ++ FlowCtrlMsgs}, Connection}; + {Replies ++ FlowCtrlMsgs, Connection}; _ -> - {{replies,[{channel_request_reply,From,{error,closed}}]}, Connection} + {[{channel_request_reply,From,{error,closed}}], Connection} end. +%%%---------------------------------------------------------------- +%%% Handle the channel messages on behalf of the ssh_connection_handler +%%% state machine. +%%% +%%% Replies {Reply, UpdatedConnection} +%%% + handle_msg(#ssh_msg_channel_open_confirmation{recipient_channel = ChannelId, sender_channel = RemoteId, initial_window_size = WindowSz, @@ -283,63 +303,42 @@ handle_msg(#ssh_msg_channel_open_confirmation{recipient_channel = ChannelId, #connection{channel_cache = Cache} = Connection0, _) -> #channel{remote_id = undefined} = Channel = - ssh_channel:cache_lookup(Cache, ChannelId), + ssh_client_channel:cache_lookup(Cache, ChannelId), - ssh_channel:cache_update(Cache, Channel#channel{ + ssh_client_channel:cache_update(Cache, Channel#channel{ remote_id = RemoteId, recv_packet_size = max(32768, % rfc4254/5.2 min(PacketSz, Channel#channel.recv_packet_size) ), send_window_size = WindowSz, send_packet_size = PacketSz}), - {Reply, Connection} = reply_msg(Channel, Connection0, {open, ChannelId}), - {{replies, [Reply]}, Connection}; + reply_msg(Channel, Connection0, {open, ChannelId}); handle_msg(#ssh_msg_channel_open_failure{recipient_channel = ChannelId, reason = Reason, description = Descr, lang = Lang}, #connection{channel_cache = Cache} = Connection0, _) -> - Channel = ssh_channel:cache_lookup(Cache, ChannelId), - ssh_channel:cache_delete(Cache, ChannelId), - {Reply, Connection} = - reply_msg(Channel, Connection0, {open_error, Reason, Descr, Lang}), - {{replies, [Reply]}, Connection}; + Channel = ssh_client_channel:cache_lookup(Cache, ChannelId), + ssh_client_channel:cache_delete(Cache, ChannelId), + reply_msg(Channel, Connection0, {open_error, Reason, Descr, Lang}); -handle_msg(#ssh_msg_channel_success{recipient_channel = ChannelId}, - #connection{channel_cache = Cache} = Connection0, _) -> - Channel = ssh_channel:cache_lookup(Cache, ChannelId), - case reply_msg(Channel, Connection0, success) of - {[], Connection} -> - {noreply, Connection}; - {Reply, Connection} -> - {{replies, [Reply]}, Connection} - end; +handle_msg(#ssh_msg_channel_success{recipient_channel = ChannelId}, Connection, _) -> + reply_msg(ChannelId, Connection, success); -handle_msg(#ssh_msg_channel_failure{recipient_channel = ChannelId}, - #connection{channel_cache = Cache} = Connection0, _) -> - Channel = ssh_channel:cache_lookup(Cache, ChannelId), - case reply_msg(Channel, Connection0, failure) of - {[], Connection} -> - {noreply, Connection}; - {Reply, Connection} -> - {{replies, [Reply]}, Connection} - end; +handle_msg(#ssh_msg_channel_failure{recipient_channel = ChannelId}, Connection, _) -> + reply_msg(ChannelId, Connection, failure); - -handle_msg(#ssh_msg_channel_eof{recipient_channel = ChannelId}, - #connection{channel_cache = Cache} = Connection0, _) -> - Channel = ssh_channel:cache_lookup(Cache, ChannelId), - {Reply, Connection} = reply_msg(Channel, Connection0, {eof, ChannelId}), - {{replies, [Reply]}, Connection}; +handle_msg(#ssh_msg_channel_eof{recipient_channel = ChannelId}, Connection, _) -> + reply_msg(ChannelId, Connection, {eof, ChannelId}); handle_msg(#ssh_msg_channel_close{recipient_channel = ChannelId}, #connection{channel_cache = Cache} = Connection0, _) -> - case ssh_channel:cache_lookup(Cache, ChannelId) of + case ssh_client_channel:cache_lookup(Cache, ChannelId) of #channel{sent_close = Closed, remote_id = RemoteId, flow_control = FlowControl} = Channel -> - ssh_channel:cache_delete(Cache, ChannelId), + ssh_client_channel:cache_delete(Cache, ChannelId), {CloseMsg, Connection} = reply_msg(Channel, Connection0, {closed, ChannelId}), ConnReplyMsgs = @@ -358,48 +357,29 @@ handle_msg(#ssh_msg_channel_close{recipient_channel = ChannelId}, [{flow_control, From, {error, closed}}] end, - Replies = ConnReplyMsgs ++ [CloseMsg] ++ SendReplyMsgs, - {{replies, Replies}, Connection}; + Replies = ConnReplyMsgs ++ CloseMsg ++ SendReplyMsgs, + {Replies, Connection}; undefined -> - {{replies, []}, Connection0} + {[], Connection0} end; handle_msg(#ssh_msg_channel_data{recipient_channel = ChannelId, data = Data}, - #connection{channel_cache = Cache} = Connection0, _) -> - - case ssh_channel:cache_lookup(Cache, ChannelId) of - #channel{recv_window_size = Size} = Channel -> - WantedSize = Size - size(Data), - ssh_channel:cache_update(Cache, Channel#channel{ - recv_window_size = WantedSize}), - {Replies, Connection} = - channel_data_reply(Cache, Channel, Connection0, 0, Data), - {{replies, Replies}, Connection}; - undefined -> - {noreply, Connection0} - end; + Connection, _) -> + channel_data_reply_msg(ChannelId, Connection, 0, Data); handle_msg(#ssh_msg_channel_extended_data{recipient_channel = ChannelId, data_type_code = DataType, data = Data}, - #connection{channel_cache = Cache} = Connection0, _) -> - - #channel{recv_window_size = Size} = Channel = - ssh_channel:cache_lookup(Cache, ChannelId), - WantedSize = Size - size(Data), - ssh_channel:cache_update(Cache, Channel#channel{ - recv_window_size = WantedSize}), - {Replies, Connection} = - channel_data_reply(Cache, Channel, Connection0, DataType, Data), - {{replies, Replies}, Connection}; + Connection, _) -> + channel_data_reply_msg(ChannelId, Connection, DataType, Data); handle_msg(#ssh_msg_channel_window_adjust{recipient_channel = ChannelId, bytes_to_add = Add}, #connection{channel_cache = Cache} = Connection, _) -> #channel{send_window_size = Size, remote_id = RemoteId} = - Channel0 = ssh_channel:cache_lookup(Cache, ChannelId), + Channel0 = ssh_client_channel:cache_lookup(Cache, ChannelId), {SendList, Channel} = %% TODO: Datatype 0 ? update_send_window(Channel0#channel{send_window_size = Size + Add}, @@ -409,7 +389,7 @@ handle_msg(#ssh_msg_channel_window_adjust{recipient_channel = ChannelId, {connection_reply, channel_data_msg(RemoteId, Type, Data)} end, SendList), FlowCtrlMsgs = flow_control(Channel, Cache), - {{replies, Replies ++ FlowCtrlMsgs}, Connection}; + {Replies ++ FlowCtrlMsgs, Connection}; handle_msg(#ssh_msg_channel_open{channel_type = "session" = Type, sender_channel = RemoteId, @@ -430,8 +410,7 @@ handle_msg(#ssh_msg_channel_open{channel_type = "session" = Type, FailMsg = channel_open_failure_msg(RemoteId, ?SSH_OPEN_CONNECT_FAILED, "Connection refused", "en"), - {{replies, [{connection_reply, FailMsg}]}, - Connection0} + {[{connection_reply, FailMsg}], Connection0} end; MinAcceptedPackSz > PacketSz -> @@ -439,7 +418,7 @@ handle_msg(#ssh_msg_channel_open{channel_type = "session" = Type, ?SSH_OPEN_ADMINISTRATIVELY_PROHIBITED, lists:concat(["Maximum packet size below ",MinAcceptedPackSz, " not supported"]), "en"), - {{replies, [{connection_reply, FailMsg}]}, Connection0} + {[{connection_reply, FailMsg}], Connection0} end; handle_msg(#ssh_msg_channel_open{channel_type = "session", @@ -452,35 +431,31 @@ handle_msg(#ssh_msg_channel_open{channel_type = "session", FailMsg = channel_open_failure_msg(RemoteId, ?SSH_OPEN_CONNECT_FAILED, "Connection refused", "en"), - {{replies, [{connection_reply, FailMsg}]}, - Connection}; + {[{connection_reply, FailMsg}], Connection}; handle_msg(#ssh_msg_channel_open{sender_channel = RemoteId}, Connection, _) -> FailMsg = channel_open_failure_msg(RemoteId, ?SSH_OPEN_ADMINISTRATIVELY_PROHIBITED, "Not allowed", "en"), - {{replies, [{connection_reply, FailMsg}]}, Connection}; + {[{connection_reply, FailMsg}], Connection}; handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, request_type = "exit-status", data = Data}, - #connection{channel_cache = Cache} = Connection, _) -> + Connection, _) -> <<?UINT32(Status)>> = Data, - Channel = ssh_channel:cache_lookup(Cache, ChannelId), - {Reply, Connection} = - reply_msg(Channel, Connection, {exit_status, ChannelId, Status}), - {{replies, [Reply]}, Connection}; + reply_msg(ChannelId, Connection, {exit_status, ChannelId, Status}); handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, request_type = "exit-signal", want_reply = false, data = Data}, - #connection{channel_cache = Cache} = Connection0, _) -> - <<?UINT32(SigLen), SigName:SigLen/binary, - ?BOOLEAN(_Core), - ?UINT32(ErrLen), Err:ErrLen/binary, - ?UINT32(LangLen), Lang:LangLen/binary>> = Data, - Channel = ssh_channel:cache_lookup(Cache, ChannelId), + #connection{channel_cache = Cache} = Connection0, _) -> + <<?DEC_BIN(SigName, _SigLen), + ?BOOLEAN(_Core), + ?DEC_BIN(Err, _ErrLen), + ?DEC_BIN(Lang, _LangLen)>> = Data, + Channel = ssh_client_channel:cache_lookup(Cache, ChannelId), RemoteId = Channel#channel.remote_id, {Reply, Connection} = reply_msg(Channel, Connection0, {exit_signal, ChannelId, @@ -488,55 +463,44 @@ handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, binary_to_list(Err), binary_to_list(Lang)}), CloseMsg = channel_close_msg(RemoteId), - {{replies, [{connection_reply, CloseMsg}, Reply]}, - Connection}; + {[{connection_reply, CloseMsg}|Reply], Connection}; handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, request_type = "xon-xoff", want_reply = false, data = Data}, - #connection{channel_cache = Cache} = Connection, _) -> + Connection, _) -> <<?BOOLEAN(CDo)>> = Data, - Channel = ssh_channel:cache_lookup(Cache, ChannelId), - {Reply, Connection} = - reply_msg(Channel, Connection, {xon_xoff, ChannelId, CDo=/= 0}), - {{replies, [Reply]}, Connection}; + reply_msg(ChannelId, Connection, {xon_xoff, ChannelId, CDo=/= 0}); handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, request_type = "window-change", want_reply = false, data = Data}, - #connection{channel_cache = Cache} = Connection0, _) -> + Connection0, _) -> <<?UINT32(Width),?UINT32(Height), - ?UINT32(PixWidth), ?UINT32(PixHeight)>> = Data, - Channel = ssh_channel:cache_lookup(Cache, ChannelId), - {Reply, Connection} = - reply_msg(Channel, Connection0, {window_change, ChannelId, - Width, Height, - PixWidth, PixHeight}), - {{replies, [Reply]}, Connection}; + ?UINT32(PixWidth), ?UINT32(PixHeight)>> = Data, + reply_msg(ChannelId, Connection0, {window_change, ChannelId, + Width, Height, + PixWidth, PixHeight}); handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, request_type = "signal", data = Data}, - #connection{channel_cache = Cache} = Connection0, _) -> - <<?UINT32(SigLen), SigName:SigLen/binary>> = Data, - - Channel = ssh_channel:cache_lookup(Cache, ChannelId), - {Reply, Connection} = - reply_msg(Channel, Connection0, {signal, ChannelId, - binary_to_list(SigName)}), - {{replies, [Reply]}, Connection}; + Connection0, _) -> + <<?DEC_BIN(SigName, _SigLen)>> = Data, + reply_msg(ChannelId, Connection0, {signal, ChannelId, + binary_to_list(SigName)}); handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, request_type = "subsystem", want_reply = WantReply, data = Data}, #connection{channel_cache = Cache} = Connection, server) -> - <<?UINT32(SsLen), SsName:SsLen/binary>> = Data, + <<?DEC_BIN(SsName,_SsLen)>> = Data, #channel{remote_id = RemoteId} = Channel0 = - ssh_channel:cache_lookup(Cache, ChannelId), + ssh_client_channel:cache_lookup(Cache, ChannelId), ReplyMsg = {subsystem, ChannelId, WantReply, binary_to_list(SsName)}, @@ -544,111 +508,95 @@ handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, {ok, Pid} = start_subsystem(SsName, Connection, Channel0, ReplyMsg), erlang:monitor(process, Pid), Channel = Channel0#channel{user = Pid}, - ssh_channel:cache_update(Cache, Channel), + ssh_client_channel:cache_update(Cache, Channel), Reply = {connection_reply, channel_success_msg(RemoteId)}, - {{replies, [Reply]}, Connection} + {[Reply], Connection} catch _:_ -> - ErrorReply = {connection_reply, - channel_failure_msg(RemoteId)}, - {{replies, [ErrorReply]}, Connection} + ErrorReply = {connection_reply, channel_failure_msg(RemoteId)}, + {[ErrorReply], Connection} end; handle_msg(#ssh_msg_channel_request{request_type = "subsystem"}, Connection, client) -> %% The client SHOULD ignore subsystem requests. See RFC 4254 6.5. - {{replies, []}, Connection}; + {[], Connection}; handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, request_type = "pty-req", want_reply = WantReply, data = Data}, - #connection{channel_cache = Cache} = Connection, server) -> - <<?UINT32(TermLen), BTermName:TermLen/binary, - ?UINT32(Width),?UINT32(Height), - ?UINT32(PixWidth), ?UINT32(PixHeight), - Modes/binary>> = Data, + Connection, server) -> + <<?DEC_BIN(BTermName,_TermLen), + ?UINT32(Width),?UINT32(Height), + ?UINT32(PixWidth), ?UINT32(PixHeight), + Modes/binary>> = Data, TermName = binary_to_list(BTermName), - PtyRequest = {TermName, Width, Height, PixWidth, PixHeight, decode_pty_opts(Modes)}, - - Channel = ssh_channel:cache_lookup(Cache, ChannelId), - handle_cli_msg(Connection, Channel, + handle_cli_msg(Connection, ChannelId, {pty, ChannelId, WantReply, PtyRequest}); handle_msg(#ssh_msg_channel_request{request_type = "pty-req"}, Connection, client) -> %% The client SHOULD ignore pty requests. See RFC 4254 6.2. - {{replies, []}, Connection}; + {[], Connection}; handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, request_type = "shell", want_reply = WantReply}, - #connection{channel_cache = Cache} = Connection, server) -> - - Channel = ssh_channel:cache_lookup(Cache, ChannelId), - - handle_cli_msg(Connection, Channel, + Connection, server) -> + handle_cli_msg(Connection, ChannelId, {shell, ChannelId, WantReply}); handle_msg(#ssh_msg_channel_request{request_type = "shell"}, Connection, client) -> %% The client SHOULD ignore shell requests. See RFC 4254 6.5. - {{replies, []}, Connection}; + {[], Connection}; handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, request_type = "exec", want_reply = WantReply, data = Data}, - #connection{channel_cache = Cache} = Connection, server) -> - <<?UINT32(Len), Command:Len/binary>> = Data, - - Channel = ssh_channel:cache_lookup(Cache, ChannelId), - - handle_cli_msg(Connection, Channel, + Connection, server) -> + <<?DEC_BIN(Command, _Len)>> = Data, + handle_cli_msg(Connection, ChannelId, {exec, ChannelId, WantReply, binary_to_list(Command)}); handle_msg(#ssh_msg_channel_request{request_type = "exec"}, Connection, client) -> %% The client SHOULD ignore exec requests. See RFC 4254 6.5. - {{replies, []}, Connection}; + {[], Connection}; handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, request_type = "env", want_reply = WantReply, data = Data}, - #connection{channel_cache = Cache} = Connection, server) -> - - <<?UINT32(VarLen), - Var:VarLen/binary, ?UINT32(ValueLen), Value:ValueLen/binary>> = Data, - - Channel = ssh_channel:cache_lookup(Cache, ChannelId), - - handle_cli_msg(Connection, Channel, + Connection, server) -> + <<?DEC_BIN(Var,_VarLen), ?DEC_BIN(Value,_ValLen)>> = Data, + handle_cli_msg(Connection, ChannelId, {env, ChannelId, WantReply, Var, Value}); handle_msg(#ssh_msg_channel_request{request_type = "env"}, Connection, client) -> %% The client SHOULD ignore env requests. - {{replies, []}, Connection}; + {[], Connection}; handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, request_type = _Other, want_reply = WantReply}, #connection{channel_cache = Cache} = Connection, _) -> if WantReply == true -> - case ssh_channel:cache_lookup(Cache, ChannelId) of + case ssh_client_channel:cache_lookup(Cache, ChannelId) of #channel{remote_id = RemoteId} -> FailMsg = channel_failure_msg(RemoteId), - {{replies, [{connection_reply, FailMsg}]}, - Connection}; + {[{connection_reply, FailMsg}], Connection}; undefined -> %% Chanel has been closed - {noreply, Connection} + {[], Connection} end; true -> - {noreply, Connection} + {[], Connection} end; handle_msg(#ssh_msg_global_request{name = _Type, @@ -656,79 +604,54 @@ handle_msg(#ssh_msg_global_request{name = _Type, data = _Data}, Connection, _) -> if WantReply == true -> FailMsg = request_failure_msg(), - {{replies, [{connection_reply, FailMsg}]}, - Connection}; + {[{connection_reply, FailMsg}], Connection}; true -> - {noreply, Connection} + {[], Connection} end; handle_msg(#ssh_msg_request_failure{}, #connection{requests = [{_, From} | Rest]} = Connection, _) -> - {{replies, [{channel_request_reply, From, {failure, <<>>}}]}, + {[{channel_request_reply, From, {failure, <<>>}}], Connection#connection{requests = Rest}}; + handle_msg(#ssh_msg_request_success{data = Data}, #connection{requests = [{_, From} | Rest]} = Connection, _) -> - {{replies, [{channel_request_reply, From, {success, Data}}]}, + {[{channel_request_reply, From, {success, Data}}], Connection#connection{requests = Rest}}; handle_msg(#ssh_msg_disconnect{code = Code, - description = Description, - language = _Lang }, - #connection{channel_cache = Cache} = Connection0, _) -> - {Connection, Replies} = - ssh_channel:cache_foldl(fun(Channel, {Connection1, Acc}) -> - {Reply, Connection2} = - reply_msg(Channel, - Connection1, - {closed, Channel#channel.local_id}), - {Connection2, [Reply | Acc]} - end, {Connection0, []}, Cache), - - ssh_channel:cache_delete(Cache), - {disconnect, {Code, Description}, {{replies, Replies}, Connection}}. - -handle_cli_msg(#connection{channel_cache = Cache} = Connection, - #channel{user = undefined, - remote_id = RemoteId, - local_id = ChannelId} = Channel0, Reply0) -> - case (catch start_cli(Connection, ChannelId)) of - {ok, Pid} -> - erlang:monitor(process, Pid), - Channel = Channel0#channel{user = Pid}, - ssh_channel:cache_update(Cache, Channel), - {Reply, Connection1} = reply_msg(Channel, Connection, Reply0), - {{replies, [Reply]}, Connection1}; - _Other -> - Reply = {connection_reply, - channel_failure_msg(RemoteId)}, - {{replies, [Reply]}, Connection} - end; + description = Description}, + Connection, _) -> + {disconnect, {Code, Description}, handle_stop(Connection)}. -handle_cli_msg(Connection0, Channel, Reply0) -> - {Reply, Connection} = reply_msg(Channel, Connection0, Reply0), - {{replies, [Reply]}, Connection}. - -channel_eof_msg(ChannelId) -> - #ssh_msg_channel_eof{recipient_channel = ChannelId}. - -channel_close_msg(ChannelId) -> - #ssh_msg_channel_close {recipient_channel = ChannelId}. - -channel_status_msg({success, ChannelId}) -> - channel_success_msg(ChannelId); -channel_status_msg({failure, ChannelId}) -> - channel_failure_msg(ChannelId). - -channel_success_msg(ChannelId) -> - #ssh_msg_channel_success{recipient_channel = ChannelId}. - -channel_failure_msg(ChannelId) -> - #ssh_msg_channel_failure{recipient_channel = ChannelId}. +%%%---------------------------------------------------------------- +%%% Returns pending responses to be delivered to the peer when a +%%% Channel/Connection closes +%%% +handle_stop(#connection{channel_cache = Cache} = Connection0) -> + {Connection, Replies} = + ssh_client_channel:cache_foldl( + fun(Channel, {Connection1, Acc}) -> + {Reply, Connection2} = + reply_msg(Channel, Connection1, + {closed, Channel#channel.local_id}), + {Connection2, Reply ++ Acc} + end, {Connection0, []}, Cache), + ssh_client_channel:cache_delete(Cache), + {Replies, Connection}. + +%%%---------------------------------------------------------------- +%%% channel_*_msg(...) +%%% Returns a #ssh_msg_....{} for channel operations. +%%% channel_adjust_window_msg(ChannelId, Bytes) -> #ssh_msg_channel_window_adjust{recipient_channel = ChannelId, bytes_to_add = Bytes}. +channel_close_msg(ChannelId) -> + #ssh_msg_channel_close {recipient_channel = ChannelId}. + channel_data_msg(ChannelId, 0, Data) -> #ssh_msg_channel_data{recipient_channel = ChannelId, data = Data}; @@ -737,6 +660,12 @@ channel_data_msg(ChannelId, Type, Data) -> data_type_code = Type, data = Data}. +channel_eof_msg(ChannelId) -> + #ssh_msg_channel_eof{recipient_channel = ChannelId}. + +channel_failure_msg(ChannelId) -> + #ssh_msg_channel_failure{recipient_channel = ChannelId}. + channel_open_msg(Type, ChannelId, WindowSize, MaxPacketSize, Data) -> #ssh_msg_channel_open{channel_type = Type, sender_channel = ChannelId, @@ -757,18 +686,34 @@ channel_open_failure_msg(RemoteId, Reason, Description, Lang) -> description = Description, lang = Lang}. +channel_status_msg({success, ChannelId}) -> + channel_success_msg(ChannelId); + +channel_status_msg({failure, ChannelId}) -> + channel_failure_msg(ChannelId). + channel_request_msg(ChannelId, Type, WantReply, Data) -> #ssh_msg_channel_request{recipient_channel = ChannelId, request_type = Type, want_reply = WantReply, data = Data}. +channel_success_msg(ChannelId) -> + #ssh_msg_channel_success{recipient_channel = ChannelId}. + +%%%---------------------------------------------------------------- +%%% request_*_msg(...) +%%% Returns a #ssh_msg_....{} for request responses. +%%% request_failure_msg() -> #ssh_msg_request_failure{}. request_success_msg(Data) -> #ssh_msg_request_success{data = Data}. +%%%---------------------------------------------------------------- +%%% +%%% bind(IP, Port, ChannelPid, Connection) -> Binds = [{{IP, Port}, ChannelPid} | lists:keydelete({IP, Port}, 1, @@ -808,6 +753,68 @@ encode_ip(Addr) when is_list(Addr) -> end end. +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% +%%% Internal functions +%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +%%%---------------------------------------------------------------- +%%% Create the channel data when an ssh_msg_open_channel message +%%% of "session" typ is handled +%%% +setup_session(#connection{channel_cache = Cache, + channel_id_seed = NewChannelID + } = C, + RemoteId, Type, WindowSize, PacketSize) -> + NextChannelID = NewChannelID + 1, + Channel = + #channel{type = Type, + sys = "ssh", + local_id = NewChannelID, + recv_window_size = ?DEFAULT_WINDOW_SIZE, + recv_packet_size = ?DEFAULT_PACKET_SIZE, + send_window_size = WindowSize, + send_packet_size = PacketSize, + send_buf = queue:new(), + remote_id = RemoteId + }, + ssh_client_channel:cache_update(Cache, Channel), + OpenConfMsg = channel_open_confirmation_msg(RemoteId, NewChannelID, + ?DEFAULT_WINDOW_SIZE, + ?DEFAULT_PACKET_SIZE), + Reply = {connection_reply, OpenConfMsg}, + {[Reply], C#connection{channel_id_seed = NextChannelID}}. + + +%%%---------------------------------------------------------------- +%%% Start a cli or subsystem +%%% +start_cli(#connection{options = Options, + cli_spec = CliSpec, + exec = Exec, + sub_system_supervisor = SubSysSup}, ChannelId) -> + case CliSpec of + no_cli -> + {error, cli_disabled}; + {CbModule, Args} -> + start_channel(CbModule, ChannelId, Args, SubSysSup, Exec, Options) + end. + + +start_subsystem(BinName, #connection{options = Options, + sub_system_supervisor = SubSysSup}, + #channel{local_id = ChannelId}, _ReplyMsg) -> + Name = binary_to_list(BinName), + case check_subsystem(Name, Options) of + {Callback, Opts} when is_atom(Callback), Callback =/= none -> + start_channel(Callback, ChannelId, Opts, SubSysSup, Options); + {Other, _} when Other =/= none -> + {error, legacy_option_not_supported} + end. + + +%%% Helpers for starting cli/subsystems start_channel(Cb, Id, Args, SubSysSup, Opts) -> start_channel(Cb, Id, Args, SubSysSup, undefined, Opts). @@ -815,45 +822,18 @@ start_channel(Cb, Id, Args, SubSysSup, Exec, Opts) -> ChannelSup = ssh_subsystem_sup:channel_supervisor(SubSysSup), case max_num_channels_not_exceeded(ChannelSup, Opts) of true -> - ssh_channel_sup:start_child(ChannelSup, Cb, Id, Args, Exec); + ssh_server_channel_sup:start_child(ChannelSup, Cb, Id, Args, Exec); false -> throw(max_num_channels_exceeded) end. max_num_channels_not_exceeded(ChannelSup, Opts) -> MaxNumChannels = ?GET_OPT(max_channels, Opts), - NumChannels = length([x || {_,_,worker,[ssh_channel]} <- + NumChannels = length([x || {_,_,worker,[ssh_server_channel]} <- supervisor:which_children(ChannelSup)]), %% Note that NumChannels is BEFORE starting a new one NumChannels < MaxNumChannels. -%%-------------------------------------------------------------------- -%%% Internal functions -%%-------------------------------------------------------------------- -setup_session(#connection{channel_cache = Cache - } = Connection0, - RemoteId, - Type, WindowSize, PacketSize) -> - {ChannelId, Connection} = new_channel_id(Connection0), - - Channel = #channel{type = Type, - sys = "ssh", - local_id = ChannelId, - recv_window_size = ?DEFAULT_WINDOW_SIZE, - recv_packet_size = ?DEFAULT_PACKET_SIZE, - send_window_size = WindowSize, - send_packet_size = PacketSize, - send_buf = queue:new(), - remote_id = RemoteId - }, - ssh_channel:cache_update(Cache, Channel), - OpenConfMsg = channel_open_confirmation_msg(RemoteId, ChannelId, - ?DEFAULT_WINDOW_SIZE, - ?DEFAULT_PACKET_SIZE), - - {{replies, [{connection_reply, OpenConfMsg}]}, Connection}. - - check_subsystem("sftp"= SsName, Options) -> case ?GET_OPT(subsystems, Options) of no_subsys -> % FIXME: Can 'no_subsys' ever be matched? @@ -872,64 +852,10 @@ check_subsystem(SsName, Options) -> Value end. -start_cli(#connection{cli_spec = no_cli}, _) -> - {error, cli_disabled}; -start_cli(#connection{options = Options, - cli_spec = {CbModule, Args}, - exec = Exec, - sub_system_supervisor = SubSysSup}, ChannelId) -> - start_channel(CbModule, ChannelId, Args, SubSysSup, Exec, Options). - -start_subsystem(BinName, #connection{options = Options, - sub_system_supervisor = SubSysSup}, - #channel{local_id = ChannelId}, _ReplyMsg) -> - Name = binary_to_list(BinName), - case check_subsystem(Name, Options) of - {Callback, Opts} when is_atom(Callback), Callback =/= none -> - start_channel(Callback, ChannelId, Opts, SubSysSup, Options); - {Other, _} when Other =/= none -> - {error, legacy_option_not_supported} - end. - -channel_data_reply(_, #channel{local_id = ChannelId} = Channel, - Connection0, DataType, Data) -> - {Reply, Connection} = - reply_msg(Channel, Connection0, {data, ChannelId, DataType, Data}), - {[Reply], Connection}. - -new_channel_id(Connection) -> - ID = Connection#connection.channel_id_seed, - {ID, Connection#connection{channel_id_seed = ID + 1}}. - -reply_msg(Channel, Connection, {open, _} = Reply) -> - request_reply_or_data(Channel, Connection, Reply); -reply_msg(Channel, Connection, {open_error, _, _, _} = Reply) -> - request_reply_or_data(Channel, Connection, Reply); -reply_msg(Channel, Connection, success = Reply) -> - request_reply_or_data(Channel, Connection, Reply); -reply_msg(Channel, Connection, failure = Reply) -> - request_reply_or_data(Channel, Connection, Reply); -reply_msg(Channel, Connection, {closed, _} = Reply) -> - request_reply_or_data(Channel, Connection, Reply); -reply_msg(undefined, Connection, _Reply) -> - {noreply, Connection}; -reply_msg(#channel{user = ChannelPid}, Connection, Reply) -> - {{channel_data, ChannelPid, Reply}, Connection}. - - -request_reply_or_data(#channel{local_id = ChannelId, user = ChannelPid}, - #connection{requests = Requests} = - Connection, Reply) -> - case lists:keysearch(ChannelId, 1, Requests) of - {value, {ChannelId, From}} -> - {{channel_request_reply, From, Reply}, - Connection#connection{requests = - lists:keydelete(ChannelId, 1, Requests)}}; - false when (Reply == success) or (Reply == failure) -> - {[], Connection}; - false -> - {{channel_data, ChannelPid, Reply}, Connection} - end. +%%%---------------------------------------------------------------- +%%% +%%% Send-window handling +%%% update_send_window(Channel, _, undefined, #connection{channel_cache = Cache}) -> @@ -942,7 +868,7 @@ update_send_window(#channel{send_buf = SendBuffer} = Channel, DataType, Data, do_update_send_window(Channel0, Cache) -> {SendMsgs, Channel} = get_window(Channel0, []), - ssh_channel:cache_update(Cache, Channel), + ssh_client_channel:cache_update(Cache, Channel), {SendMsgs, Channel}. get_window(#channel{send_window_size = 0 @@ -984,17 +910,22 @@ handle_send_window({Type, Data}, _, PacketSize, WindowSize, Acc) -> <<Msg1:PacketSize/binary, Msg2/binary>> = Data, {WindowSize - PacketSize, [{Type, Msg1} | Acc], {Type, Msg2}}. +%%%---------------------------------------------------------------- +%%% +%%% Flow control +%%% + flow_control(Channel, Cache) -> flow_control([window_adjusted], Channel, Cache). flow_control([], Channel, Cache) -> - ssh_channel:cache_update(Cache, Channel), + ssh_client_channel:cache_update(Cache, Channel), []; flow_control([_|_], #channel{flow_control = From, send_buf = Buffer} = Channel, Cache) when From =/= undefined -> case queue:is_empty(Buffer) of true -> - ssh_channel:cache_update(Cache, Channel#channel{flow_control = undefined}), + ssh_client_channel:cache_update(Cache, Channel#channel{flow_control = undefined}), [{flow_control, Cache, Channel, From, ok}]; false -> [] @@ -1002,6 +933,11 @@ flow_control([_|_], #channel{flow_control = From, flow_control(_,_,_) -> []. +%%%---------------------------------------------------------------- +%%% +%%% Pseudo terminal stuff +%%% + pty_req(ConnectionHandler, Channel, Term, Width, Height, PixWidth, PixHeight, PtyOpts, TimeOut) -> ssh_connection_handler:request(ConnectionHandler, @@ -1027,8 +963,7 @@ pty_default_dimensions(Dimension, TermData) -> encode_pty_opts(Opts) -> Bin = list_to_binary(encode_pty_opts2(Opts)), - Len = size(Bin), - <<?UINT32(Len), Bin/binary>>. + <<?STRING(Bin)>>. encode_pty_opts2([]) -> [?TTY_OP_END]; @@ -1147,7 +1082,7 @@ decode_pty_opts(<<>>) -> []; decode_pty_opts(<<0, 0, 0, 0>>) -> []; -decode_pty_opts(<<?UINT32(Len), Modes:Len/binary>>) -> +decode_pty_opts(<<?DEC_BIN(Modes,_Len)>>) -> decode_pty_opts2(Modes); decode_pty_opts(Binary) -> decode_pty_opts2(Binary). @@ -1224,3 +1159,104 @@ backwards_compatible([{pixel_hight, Value} | Rest], Acc) -> backwards_compatible(Rest, [{height, Value} | Acc]); backwards_compatible([Value| Rest], Acc) -> backwards_compatible(Rest, [ Value | Acc]). + + +%%%---------------------------------------------------------------- +%%% +%%% Common part of handling channel messages meant for a cli (like "env", "exec" etc) +%%% Called at the finnish of handle_msg(#ssh_msg_channel_request,...) +%%% + +handle_cli_msg(C0, ChId, Reply0) -> + Cache = C0#connection.channel_cache, + Ch0 = ssh_client_channel:cache_lookup(Cache, ChId), + case Ch0#channel.user of + undefined -> + case (catch start_cli(C0, ChId)) of + {ok, Pid} -> + erlang:monitor(process, Pid), + Ch = Ch0#channel{user = Pid}, + ssh_client_channel:cache_update(Cache, Ch), + reply_msg(Ch, C0, Reply0); + _Other -> + Reply = {connection_reply, channel_failure_msg(Ch0#channel.remote_id)}, + {[Reply], C0} + end; + + _ -> + reply_msg(Ch0, C0, Reply0) + end. + +%%%---------------------------------------------------------------- +%%% +%%% Request response handling on return to the calling ssh_connection_handler +%%% state machine. +%%% + +channel_data_reply_msg(ChannelId, Connection, DataType, Data) -> + case ssh_client_channel:cache_lookup(Connection#connection.channel_cache, ChannelId) of + #channel{recv_window_size = Size} = Channel -> + WantedSize = Size - size(Data), + ssh_client_channel:cache_update(Connection#connection.channel_cache, + Channel#channel{recv_window_size = WantedSize}), + reply_msg(Channel, Connection, {data, ChannelId, DataType, Data}); + undefined -> + {[], Connection} + end. + + +reply_msg(ChId, C, Reply) when is_integer(ChId) -> + reply_msg(ssh_client_channel:cache_lookup(C#connection.channel_cache, ChId), C, Reply); + +reply_msg(Channel, Connection, {open, _} = Reply) -> + request_reply_or_data(Channel, Connection, Reply); +reply_msg(Channel, Connection, {open_error, _, _, _} = Reply) -> + request_reply_or_data(Channel, Connection, Reply); +reply_msg(Channel, Connection, success = Reply) -> + request_reply_or_data(Channel, Connection, Reply); +reply_msg(Channel, Connection, failure = Reply) -> + request_reply_or_data(Channel, Connection, Reply); +reply_msg(Channel, Connection, {closed, _} = Reply) -> + request_reply_or_data(Channel, Connection, Reply); +reply_msg(undefined, Connection, _Reply) -> + {[], Connection}; +reply_msg(#channel{user = ChannelPid}, Connection, Reply) -> + {[{channel_data, ChannelPid, Reply}], Connection}. + + +request_reply_or_data(#channel{local_id = ChannelId, user = ChannelPid}, + #connection{requests = Requests} = + Connection, Reply) -> + case lists:keysearch(ChannelId, 1, Requests) of + {value, {ChannelId, From}} -> + {[{channel_request_reply, From, Reply}], + Connection#connection{requests = + lists:keydelete(ChannelId, 1, Requests)}}; + false when (Reply == success) or (Reply == failure) -> + {[], Connection}; + false -> + {[{channel_data, ChannelPid, Reply}], Connection} + end. + + + +%%%---------------------------------------------------------------- +%%% l(ist)2b(inary) +%%% +l2b(L) when is_integer(hd(L)) -> + try list_to_binary(L) + of + B -> B + catch + _:_ -> + unicode:characters_to_binary(L) + end; +l2b([H|T]) -> + << (l2b(H))/binary, (l2b(T))/binary >>; +l2b(B) when is_binary(B) -> + B; +l2b([]) -> + <<>>. + + + diff --git a/lib/ssh/src/ssh_connection_handler.erl b/lib/ssh/src/ssh_connection_handler.erl index e11d3adee4..57641cf74c 100644 --- a/lib/ssh/src/ssh_connection_handler.erl +++ b/lib/ssh/src/ssh_connection_handler.erl @@ -56,10 +56,13 @@ connection_info/2, channel_info/3, adjust_window/3, close/2, - disconnect/1, disconnect/2, + disconnect/4, get_print_info/1 ]). +-type connection_ref() :: ssh:connection_ref(). +-type channel_id() :: ssh:channel_id(). + %%% Behaviour callbacks -export([init/1, callback_mode/0, handle_event/4, terminate/3, format_status/2, code_change/4]). @@ -68,16 +71,28 @@ -export([init_connection_handler/3, % proc_lib:spawn needs this init_ssh_record/3, % Export of this internal function % intended for low-level protocol test suites - renegotiate/1, renegotiate_data/1 % Export intended for test cases + renegotiate/1, renegotiate_data/1, alg/1 % Export intended for test cases ]). +-export([dbg_trace/3]). + + +-define(send_disconnect(Code, DetailedText, StateName, State), + send_disconnect(Code, DetailedText, ?MODULE, ?LINE, StateName, State)). + +-define(send_disconnect(Code, Reason, DetailedText, StateName, State), + send_disconnect(Code, Reason, DetailedText, ?MODULE, ?LINE, StateName, State)). + +-define(call_disconnectfun_and_log_cond(LogMsg, DetailedText, StateName, D), + call_disconnectfun_and_log_cond(LogMsg, DetailedText, ?MODULE, ?LINE, StateName, D)). + %%==================================================================== %% Start / stop %%==================================================================== %%-------------------------------------------------------------------- -spec start_link(role(), - inet:socket(), - ssh_options:options() + gen_tcp:socket(), + internal_options() ) -> {ok, pid()}. %% . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . start_link(Role, Socket, Options) -> @@ -106,8 +121,8 @@ stop(ConnectionHandler)-> %%-------------------------------------------------------------------- -spec start_connection(role(), - inet:socket(), - ssh_options:options(), + gen_tcp:socket(), + internal_options(), timeout() ) -> {ok, connection_ref()} | {error, term()}. %% . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . @@ -149,17 +164,16 @@ start_connection(server = Role, Socket, Options, Timeout) -> %%-------------------------------------------------------------------- %%% Some other module has decided to disconnect. --spec disconnect(#ssh_msg_disconnect{}) -> no_return(). --spec disconnect(#ssh_msg_disconnect{}, iodata()) -> no_return(). + +-spec disconnect(Code::integer(), Details::iodata(), + Module::atom(), Line::integer()) -> no_return(). %% . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . -disconnect(Msg = #ssh_msg_disconnect{}) -> - throw({keep_state_and_data, - [{next_event, internal, {disconnect, Msg, Msg#ssh_msg_disconnect.description}}]}). -disconnect(Msg = #ssh_msg_disconnect{}, ExtraInfo) -> - throw({keep_state_and_data, - [{next_event, internal, {disconnect, Msg, {Msg#ssh_msg_disconnect.description,ExtraInfo}}}]}). +% Preferable called with the macro ?DISCONNECT +disconnect(Code, DetailedText, Module, Line) -> + throw({keep_state_and_data, + [{next_event, internal, {send_disconnect, Code, DetailedText, Module, Line}}]}). %%-------------------------------------------------------------------- -spec open_channel(connection_ref(), @@ -320,6 +334,9 @@ renegotiate(ConnectionHandler) -> renegotiate_data(ConnectionHandler) -> cast(ConnectionHandler, data_size). +%%-------------------------------------------------------------------- +alg(ConnectionHandler) -> + call(ConnectionHandler, get_alg). %%==================================================================== %% Internal process state @@ -345,7 +362,7 @@ renegotiate_data(ConnectionHandler) -> | undefined, % ex: tcp_closed ssh_params :: #ssh{} | undefined, - socket :: inet:socket() + socket :: gen_tcp:socket() | undefined, decrypted_data_buffer = <<>> :: binary() | undefined, @@ -356,7 +373,6 @@ renegotiate_data(ConnectionHandler) -> | undefined, last_size_rekey = 0 :: non_neg_integer(), event_queue = [] :: list(), -% opts :: ssh_options:options(), inet_initial_recbuf_size :: pos_integer() | undefined }). @@ -366,8 +382,8 @@ renegotiate_data(ConnectionHandler) -> %%==================================================================== %%-------------------------------------------------------------------- -spec init_connection_handler(role(), - inet:socket(), - ssh_options:options() + gen_tcp:socket(), + internal_options() ) -> no_return(). %% . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . init_connection_handler(Role, Socket, Opts) -> @@ -398,7 +414,7 @@ init([Role,Socket,Opts]) -> case inet:peername(Socket) of {ok, PeerAddr} -> {Protocol, Callback, CloseTag} = ?GET_OPT(transport, Opts), - C = #connection{channel_cache = ssh_channel:cache_create(), + C = #connection{channel_cache = ssh_client_channel:cache_create(), channel_id_seed = 0, port_bindings = [], requests = [], @@ -442,7 +458,7 @@ init_ssh_record(Role, Socket, Opts) -> {ok,PeerAddr} = inet:peername(Socket), init_ssh_record(Role, Socket, PeerAddr, Opts). -init_ssh_record(Role, _Socket, PeerAddr, Opts) -> +init_ssh_record(Role, Socket, PeerAddr, Opts) -> AuthMethods = ?GET_OPT(auth_methods, Opts), S0 = #ssh{role = Role, key_cb = ?GET_OPT(key_cb, Opts), @@ -453,6 +469,10 @@ init_ssh_record(Role, _Socket, PeerAddr, Opts) -> }, {Vsn, Version} = ssh_transport:versions(Role, Opts), + LocalName = case inet:sockname(Socket) of + {ok,Local} -> Local; + _ -> undefined + end, case Role of client -> PeerName = case ?GET_INTERNAL_OPT(host, Opts) of @@ -471,7 +491,8 @@ init_ssh_record(Role, _Socket, PeerAddr, Opts) -> false -> ssh_no_io end, userauth_quiet_mode = ?GET_OPT(quiet_mode, Opts), - peer = {PeerName, PeerAddr} + peer = {PeerName, PeerAddr}, + local = LocalName }, S1#ssh{userauth_pubkeys = [K || K <- ?GET_OPT(pref_public_key_algs, Opts), is_usable_user_pubkey(K, S1) @@ -484,7 +505,8 @@ init_ssh_record(Role, _Socket, PeerAddr, Opts) -> io_cb = ?GET_INTERNAL_OPT(io_cb, Opts, ssh_io), userauth_methods = string:tokens(AuthMethods, ","), kb_tries_left = 3, - peer = {undefined, PeerAddr} + peer = {undefined, PeerAddr}, + local = LocalName } end. @@ -542,7 +564,7 @@ callback_mode() -> handle_event_function. -handle_event(_, _Event, {init_error,Error}, _) -> +handle_event(_, _Event, {init_error,Error}=StateName, D) -> case Error of enotconn -> %% Handles the abnormal sequence: @@ -550,6 +572,9 @@ handle_event(_, _Event, {init_error,Error}, _) -> %% <-SYNACK %% ACK-> %% RST-> + ?call_disconnectfun_and_log_cond("Protocol Error", + "TCP connenction to server was prematurely closed by the client", + StateName, D), {stop, {shutdown,"TCP connenction to server was prematurely closed by the client"}}; OtherError -> @@ -558,7 +583,7 @@ handle_event(_, _Event, {init_error,Error}, _) -> %%% ######## {hello, client|server} #### %% The very first event that is sent when the we are set as controlling process of Socket -handle_event(_, socket_control, {hello,_}, D) -> +handle_event(_, socket_control, {hello,_}=StateName, D) -> VsnMsg = ssh_transport:hello_version_msg(string_version(D#data.ssh_params)), send_bytes(VsnMsg, D), case inet:getopts(Socket=D#data.socket, [recbuf]) of @@ -573,10 +598,13 @@ handle_event(_, socket_control, {hello,_}, D) -> {keep_state, D#data{inet_initial_recbuf_size=Size}}; Other -> + ?call_disconnectfun_and_log_cond("Option return", + io_lib:format("Unexpected getopts return:~n ~p",[Other]), + StateName, D), {stop, {shutdown,{unexpected_getopts_return, Other}}} end; -handle_event(_, {info_line,_Line}, {hello,Role}, D) -> +handle_event(_, {info_line,_Line}, {hello,Role}=StateName, D) -> case Role of client -> %% The server may send info lines to the client before the version_exchange @@ -587,28 +615,33 @@ handle_event(_, {info_line,_Line}, {hello,Role}, D) -> %% But the client may NOT send them to the server. Openssh answers with cleartext, %% and so do we send_bytes("Protocol mismatch.", D), + ?call_disconnectfun_and_log_cond("Protocol mismatch.", + "Protocol mismatch in version exchange. Client sent info lines.", + StateName, D), {stop, {shutdown,"Protocol mismatch in version exchange. Client sent info lines."}} end; -handle_event(_, {version_exchange,Version}, {hello,Role}, D) -> +handle_event(_, {version_exchange,Version}, {hello,Role}, D0) -> {NumVsn, StrVsn} = ssh_transport:handle_hello_version(Version), - case handle_version(NumVsn, StrVsn, D#data.ssh_params) of + case handle_version(NumVsn, StrVsn, D0#data.ssh_params) of {ok, Ssh1} -> %% Since the hello part is finnished correctly, we set the %% socket to the packet handling mode (including recbuf size): - inet:setopts(D#data.socket, [{packet,0}, + inet:setopts(D0#data.socket, [{packet,0}, {mode,binary}, {active, once}, - {recbuf, D#data.inet_initial_recbuf_size}]), + {recbuf, D0#data.inet_initial_recbuf_size}]), {KeyInitMsg, SshPacket, Ssh} = ssh_transport:key_exchange_init_msg(Ssh1), - send_bytes(SshPacket, D), - {next_state, {kexinit,Role,init}, D#data{ssh_params = Ssh, + send_bytes(SshPacket, D0), + {next_state, {kexinit,Role,init}, D0#data{ssh_params = Ssh, key_exchange_init_msg = KeyInitMsg}}; not_supported -> - disconnect( - #ssh_msg_disconnect{code = ?SSH_DISCONNECT_PROTOCOL_VERSION_NOT_SUPPORTED, - description = ["Protocol version ",StrVsn," not supported"]}, - {next_state, {hello,Role}, D}) + {Shutdown, D} = + ?send_disconnect(?SSH_DISCONNECT_PROTOCOL_VERSION_NOT_SUPPORTED, + io_lib:format("Offending version is ~p",[string:chomp(Version)]), + {hello,Role}, + D0), + {stop, Shutdown, D} end; @@ -754,18 +787,20 @@ handle_event(internal, Msg, {ext_info,Role,_ReNegFlag}, D) when is_tuple(Msg) -> %%% ######## {service_request, client|server} #### -handle_event(_, Msg = #ssh_msg_service_request{name=ServiceName}, StateName = {service_request,server}, D) -> +handle_event(_, Msg = #ssh_msg_service_request{name=ServiceName}, StateName = {service_request,server}, D0) -> case ServiceName of "ssh-userauth" -> - Ssh0 = #ssh{session_id=SessionId} = D#data.ssh_params, + Ssh0 = #ssh{session_id=SessionId} = D0#data.ssh_params, {ok, {Reply, Ssh}} = ssh_auth:handle_userauth_request(Msg, SessionId, Ssh0), - send_bytes(Reply, D), - {next_state, {userauth,server}, D#data{ssh_params = Ssh}}; + send_bytes(Reply, D0), + {next_state, {userauth,server}, D0#data{ssh_params = Ssh}}; _ -> - disconnect(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_SERVICE_NOT_AVAILABLE, - description = "Unknown service"}, - StateName, D) + {Shutdown, D} = + ?send_disconnect(?SSH_DISCONNECT_SERVICE_NOT_AVAILABLE, + io_lib:format("Unknown service: ~p",[ServiceName]), + StateName, D0), + {stop, Shutdown, D} end; handle_event(_, #ssh_msg_service_accept{name = "ssh-userauth"}, {service_request,client}, @@ -781,15 +816,15 @@ handle_event(_, #ssh_msg_service_accept{name = "ssh-userauth"}, {service_request handle_event(_, Msg = #ssh_msg_userauth_request{service = ServiceName, method = Method}, StateName = {userauth,server}, - D = #data{ssh_params=Ssh0}) -> + D0 = #data{ssh_params=Ssh0}) -> case {ServiceName, Ssh0#ssh.service, Method} of {"ssh-connection", "ssh-connection", "none"} -> %% Probably the very first userauth_request but we deny unauthorized login {not_authorized, _, {Reply,Ssh}} = ssh_auth:handle_userauth_request(Msg, Ssh0#ssh.session_id, Ssh0), - send_bytes(Reply, D), - {keep_state, D#data{ssh_params = Ssh}}; + send_bytes(Reply, D0), + {keep_state, D0#data{ssh_params = Ssh}}; {"ssh-connection", "ssh-connection", Method} -> %% Userauth request with a method like "password" or so @@ -798,20 +833,20 @@ handle_event(_, %% Yepp! we support this method case ssh_auth:handle_userauth_request(Msg, Ssh0#ssh.session_id, Ssh0) of {authorized, User, {Reply, Ssh}} -> - send_bytes(Reply, D), - D#data.starter ! ssh_connected, - connected_fun(User, Method, D), + send_bytes(Reply, D0), + D0#data.starter ! ssh_connected, + connected_fun(User, Method, D0), {next_state, {connected,server}, - D#data{auth_user = User, + D0#data{auth_user = User, ssh_params = Ssh#ssh{authenticated = true}}}; {not_authorized, {User, Reason}, {Reply, Ssh}} when Method == "keyboard-interactive" -> - retry_fun(User, Reason, D), - send_bytes(Reply, D), - {next_state, {userauth_keyboard_interactive,server}, D#data{ssh_params = Ssh}}; + retry_fun(User, Reason, D0), + send_bytes(Reply, D0), + {next_state, {userauth_keyboard_interactive,server}, D0#data{ssh_params = Ssh}}; {not_authorized, {User, Reason}, {Reply, Ssh}} -> - retry_fun(User, Reason, D), - send_bytes(Reply, D), - {keep_state, D#data{ssh_params = Ssh}} + retry_fun(User, Reason, D0), + send_bytes(Reply, D0), + {keep_state, D0#data{ssh_params = Ssh}} end; false -> %% No we do not support this method (=/= none) @@ -825,9 +860,11 @@ handle_event(_, %% {ServiceName, Expected, Method} when Expected =/= ServiceName -> Do what? {ServiceName, _, _} when ServiceName =/= "ssh-connection" -> - disconnect(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_SERVICE_NOT_AVAILABLE, - description = "Unknown service"}, - StateName, D) + {Shutdown, D} = + ?send_disconnect(?SSH_DISCONNECT_SERVICE_NOT_AVAILABLE, + io_lib:format("Unknown service: ~p",[ServiceName]), + StateName, D0), + {stop, Shutdown, D} end; %%---- userauth success to client @@ -843,14 +880,14 @@ handle_event(_, #ssh_msg_userauth_success{}, {userauth,client}, D=#data{ssh_para %%---- userauth failure response to client handle_event(_, #ssh_msg_userauth_failure{}, {userauth,client}=StateName, - D = #data{ssh_params = #ssh{userauth_methods = []}}) -> - Msg = #ssh_msg_disconnect{code = ?SSH_DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE, - description = "Unable to connect using the available" - " authentication methods"}, - disconnect(Msg, StateName, D); - + #data{ssh_params = #ssh{userauth_methods = []}} = D0) -> + {Shutdown, D} = + ?send_disconnect(?SSH_DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE, + io_lib:format("User auth failed for: ~p",[D0#data.auth_user]), + StateName, D0), + {stop, Shutdown, D}; handle_event(_, #ssh_msg_userauth_failure{authentications = Methods}, StateName={userauth,client}, - D = #data{ssh_params = Ssh0}) -> + D0 = #data{ssh_params = Ssh0}) -> %% The prefered authentication method failed try next method Ssh1 = case Ssh0#ssh.userauth_methods of none -> @@ -861,15 +898,18 @@ handle_event(_, #ssh_msg_userauth_failure{authentications = Methods}, StateName= Ssh0 end, case ssh_auth:userauth_request_msg(Ssh1) of - {disconnect, DisconnectMsg, {Msg, Ssh}} -> - send_bytes(Msg, D), - disconnect(DisconnectMsg, StateName, D#data{ssh_params = Ssh}); + {send_disconnect, Code, Ssh} -> + {Shutdown, D} = + ?send_disconnect(Code, + io_lib:format("User auth failed for: ~p",[D0#data.auth_user]), + StateName, D0#data{ssh_params = Ssh}), + {stop, Shutdown, D}; {"keyboard-interactive", {Msg, Ssh}} -> - send_bytes(Msg, D), - {next_state, {userauth_keyboard_interactive,client}, D#data{ssh_params = Ssh}}; + send_bytes(Msg, D0), + {next_state, {userauth_keyboard_interactive,client}, D0#data{ssh_params = Ssh}}; {_Method, {Msg, Ssh}} -> - send_bytes(Msg, D), - {keep_state, D#data{ssh_params = Ssh}} + send_bytes(Msg, D0), + {keep_state, D0#data{ssh_params = Ssh}} end; %%---- banner to client @@ -960,10 +1000,10 @@ handle_event(_, {#ssh_msg_kexinit{},_}, {connected,Role}, D0) -> {next_state, {kexinit,Role,renegotiate}, D, [postpone]}; handle_event(_, #ssh_msg_disconnect{description=Desc} = Msg, StateName, D0) -> - {disconnect, _, {{replies,Replies}, _}} = + {disconnect, _, RepliesCon} = ssh_connection:handle_msg(Msg, D0#data.connection_state, role(StateName)), - {Actions,D} = send_replies(Replies, D0), - disconnect_fun(Desc, D), + {Actions,D} = send_replies(RepliesCon, D0), + disconnect_fun("Received disconnect: "++Desc, D), {stop_and_reply, {shutdown,Desc}, Actions, D}; handle_event(_, #ssh_msg_ignore{}, _, _) -> @@ -1030,6 +1070,10 @@ handle_event(cast, renegotiate, {connected,Role}, D) -> {next_state, {kexinit,Role,renegotiate}, D#data{ssh_params = Ssh, key_exchange_init_msg = KeyInitMsg}}; +handle_event({call,From}, get_alg, _, D) -> + #ssh{algorithms=Algs} = D#data.ssh_params, + {keep_state_and_data, [{reply,From,Algs}]}; + handle_event(cast, renegotiate, _, _) -> %% Already in key-exchange so safe to ignore timer:apply_after(?REKEY_TIMOUT, gen_statem, cast, [self(), renegotiate]), % FIXME: not here in original @@ -1065,13 +1109,13 @@ handle_event(cast, _, StateName, _) when not ?CONNECTED(StateName) -> {keep_state_and_data, [postpone]}; handle_event(cast, {adjust_window,ChannelId,Bytes}, StateName, D) when ?CONNECTED(StateName) -> - case ssh_channel:cache_lookup(cache(D), ChannelId) of + case ssh_client_channel:cache_lookup(cache(D), ChannelId) of #channel{recv_window_size = WinSize, recv_window_pending = Pending, recv_packet_size = PktSize} = Channel when (WinSize-Bytes) >= 2*PktSize -> %% The peer can send at least two more *full* packet, no hurry. - ssh_channel:cache_update(cache(D), + ssh_client_channel:cache_update(cache(D), Channel#channel{recv_window_pending = Pending + Bytes}), keep_state_and_data; @@ -1079,7 +1123,7 @@ handle_event(cast, {adjust_window,ChannelId,Bytes}, StateName, D) when ?CONNECTE recv_window_pending = Pending, remote_id = Id} = Channel -> %% Now we have to update the window - we can't receive so many more pkts - ssh_channel:cache_update(cache(D), + ssh_client_channel:cache_update(cache(D), Channel#channel{recv_window_size = WinSize + Bytes + Pending, recv_window_pending = 0}), @@ -1091,7 +1135,7 @@ handle_event(cast, {adjust_window,ChannelId,Bytes}, StateName, D) when ?CONNECTE end; handle_event(cast, {reply_request,success,ChannelId}, StateName, D) when ?CONNECTED(StateName) -> - case ssh_channel:cache_lookup(cache(D), ChannelId) of + case ssh_client_channel:cache_lookup(cache(D), ChannelId) of #channel{remote_id = RemoteId} -> Msg = ssh_connection:channel_success_msg(RemoteId), update_inet_buffers(D#data.socket), @@ -1134,7 +1178,7 @@ handle_event({call,From}, {connection_info, Options}, _, D) -> {keep_state_and_data, [{reply,From,Info}]}; handle_event({call,From}, {channel_info,ChannelId,Options}, _, D) -> - case ssh_channel:cache_lookup(cache(D), ChannelId) of + case ssh_client_channel:cache_lookup(cache(D), ChannelId) of #channel{} = Channel -> Info = fold_keys(Options, fun chann_info/2, Channel), {keep_state_and_data, [{reply,From,Info}]}; @@ -1144,14 +1188,14 @@ handle_event({call,From}, {channel_info,ChannelId,Options}, _, D) -> handle_event({call,From}, {info, all}, _, D) -> - Result = ssh_channel:cache_foldl(fun(Channel, Acc) -> + Result = ssh_client_channel:cache_foldl(fun(Channel, Acc) -> [Channel | Acc] end, [], cache(D)), {keep_state_and_data, [{reply, From, {ok,Result}}]}; handle_event({call,From}, {info, ChannelPid}, _, D) -> - Result = ssh_channel:cache_foldl( + Result = ssh_client_channel:cache_foldl( fun(Channel, Acc) when Channel#channel.user == ChannelPid -> [Channel | Acc]; (_, Acc) -> @@ -1159,15 +1203,9 @@ handle_event({call,From}, {info, ChannelPid}, _, D) -> end, [], cache(D)), {keep_state_and_data, [{reply, From, {ok,Result}}]}; -handle_event({call,From}, stop, StateName, D0) -> - {disconnect, _Reason, {{replies, Replies}, Connection}} = - ssh_connection:handle_msg(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_BY_APPLICATION, - description = "User closed down connection"}, - D0#data.connection_state, - role(StateName)), - {Repls,D} = send_replies(Replies, D0), - {stop_and_reply, normal, [{reply,From,ok}|Repls], D#data{connection_state=Connection}}; - +handle_event({call,From}, stop, _StateName, D0) -> + {Repls,D} = send_replies(ssh_connection:handle_stop(D0#data.connection_state), D0), + {stop_and_reply, normal, [{reply,From,ok}|Repls], D}; handle_event({call,_}, _, StateName, _) when not ?CONNECTED(StateName) -> {keep_state_and_data, [postpone]}; @@ -1196,15 +1234,14 @@ handle_event({call,From}, {request, ChannelId, Type, Data, Timeout}, StateName, handle_event({call,From}, {data, ChannelId, Type, Data, Timeout}, StateName, D0) when ?CONNECTED(StateName) -> - {{replies, Replies}, Connection} = - ssh_connection:channel_data(ChannelId, Type, Data, D0#data.connection_state, From), - {Repls,D} = send_replies(Replies, D0#data{connection_state = Connection}), + {Repls,D} = send_replies(ssh_connection:channel_data(ChannelId, Type, Data, D0#data.connection_state, From), + D0), start_channel_request_timer(ChannelId, From, Timeout), % FIXME: No message exchange so why? {keep_state, D, Repls}; handle_event({call,From}, {eof, ChannelId}, StateName, D0) when ?CONNECTED(StateName) -> - case ssh_channel:cache_lookup(cache(D0), ChannelId) of + case ssh_client_channel:cache_lookup(cache(D0), ChannelId) of #channel{remote_id = Id, sent_close = false} -> D = send_msg(ssh_connection:channel_eof_msg(Id), D0), {keep_state, D, [{reply,From,ok}]}; @@ -1222,7 +1259,7 @@ handle_event({call,From}, InitialWindowSize, MaxPacketSize, Data), D1), - ssh_channel:cache_update(cache(D2), + ssh_client_channel:cache_update(cache(D2), #channel{type = Type, sys = "none", user = ChannelPid, @@ -1237,7 +1274,7 @@ handle_event({call,From}, handle_event({call,From}, {send_window, ChannelId}, StateName, D) when ?CONNECTED(StateName) -> - Reply = case ssh_channel:cache_lookup(cache(D), ChannelId) of + Reply = case ssh_client_channel:cache_lookup(cache(D), ChannelId) of #channel{send_window_size = WinSize, send_packet_size = Packsize} -> {ok, {WinSize, Packsize}}; @@ -1248,7 +1285,7 @@ handle_event({call,From}, {send_window, ChannelId}, StateName, D) handle_event({call,From}, {recv_window, ChannelId}, StateName, D) when ?CONNECTED(StateName) -> - Reply = case ssh_channel:cache_lookup(cache(D), ChannelId) of + Reply = case ssh_client_channel:cache_lookup(cache(D), ChannelId) of #channel{recv_window_size = WinSize, recv_packet_size = Packsize} -> {ok, {WinSize, Packsize}}; @@ -1259,10 +1296,10 @@ handle_event({call,From}, {recv_window, ChannelId}, StateName, D) handle_event({call,From}, {close, ChannelId}, StateName, D0) when ?CONNECTED(StateName) -> - case ssh_channel:cache_lookup(cache(D0), ChannelId) of + case ssh_client_channel:cache_lookup(cache(D0), ChannelId) of #channel{remote_id = Id} = Channel -> D1 = send_msg(ssh_connection:channel_close_msg(Id), D0), - ssh_channel:cache_update(cache(D1), Channel#channel{sent_close = true}), + ssh_client_channel:cache_update(cache(D1), Channel#channel{sent_close = true}), {keep_state, cache_request_idle_timer_check(D1), [{reply,From,ok}]}; undefined -> {keep_state_and_data, [{reply,From,ok}]} @@ -1288,29 +1325,32 @@ handle_event(info, {Proto, Sock, NewData}, StateName, D0 = #data{socket = Sock, D0#data.ssh_params) of {packet_decrypted, DecryptedBytes, EncryptedDataRest, Ssh1} -> - D = D0#data{ssh_params = + D1 = D0#data{ssh_params = Ssh1#ssh{recv_sequence = ssh_transport:next_seqnum(Ssh1#ssh.recv_sequence)}, decrypted_data_buffer = <<>>, undecrypted_packet_length = undefined, encrypted_data_buffer = EncryptedDataRest}, try - ssh_message:decode(set_kex_overload_prefix(DecryptedBytes,D)) + ssh_message:decode(set_kex_overload_prefix(DecryptedBytes,D1)) of Msg = #ssh_msg_kexinit{} -> - {keep_state, D, [{next_event, internal, prepare_next_packet}, + {keep_state, D1, [{next_event, internal, prepare_next_packet}, {next_event, internal, {Msg,DecryptedBytes}} ]}; Msg -> - {keep_state, D, [{next_event, internal, prepare_next_packet}, + {keep_state, D1, [{next_event, internal, prepare_next_packet}, {next_event, internal, Msg} ]} catch - _C:_E -> - disconnect(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_PROTOCOL_ERROR, - description = "Bad packet"}, - StateName, D) + C:E -> + {Shutdown, D} = + ?send_disconnect(?SSH_DISCONNECT_PROTOCOL_ERROR, + io_lib:format("Bad packet: Decrypted, but can't decode~n~p:~p~n~p", + [C,E,erlang:get_stacktrace()]), + StateName, D1), + {stop, Shutdown, D} end; - + {get_more, DecryptedBytes, EncryptedDataRest, RemainingSshPacketLen, Ssh1} -> %% Here we know that there are not enough bytes in %% EncryptedDataRest to use. We must wait for more. @@ -1321,19 +1361,26 @@ handle_event(info, {Proto, Sock, NewData}, StateName, D0 = #data{socket = Sock, ssh_params = Ssh1}}; {bad_mac, Ssh1} -> - disconnect(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_PROTOCOL_ERROR, - description = "Bad packet"}, - StateName, D0#data{ssh_params=Ssh1}); - - {error, {exceeds_max_size,_PacketLen}} -> - disconnect(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_PROTOCOL_ERROR, - description = "Bad packet"}, - StateName, D0) + {Shutdown, D} = + ?send_disconnect(?SSH_DISCONNECT_PROTOCOL_ERROR, + "Bad packet: bad mac", + StateName, D0#data{ssh_params=Ssh1}), + {stop, Shutdown, D}; + + {error, {exceeds_max_size,PacketLen}} -> + {Shutdown, D} = + ?send_disconnect(?SSH_DISCONNECT_PROTOCOL_ERROR, + io_lib:format("Bad packet: Size (~p bytes) exceeds max size", + [PacketLen]), + StateName, D0), + {stop, Shutdown, D} catch - _C:_E -> - disconnect(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_PROTOCOL_ERROR, - description = "Bad packet"}, - StateName, D0) + C:E -> + {Shutdown, D} = + ?send_disconnect(?SSH_DISCONNECT_PROTOCOL_ERROR, + io_lib:format("Bad packet: Couldn't decrypt~n~p:~p~n~p",[C,E,erlang:get_stacktrace()]), + StateName, D0), + {stop, Shutdown, D} end; @@ -1349,15 +1396,13 @@ handle_event(internal, prepare_next_packet, _, D) -> inet:setopts(D#data.socket, [{active, once}]), keep_state_and_data; -handle_event(info, {CloseTag,Socket}, StateName, - D = #data{socket = Socket, - transport_close_tag = CloseTag}) -> - %% Simulate a disconnect from the peer - handle_event(info, - #ssh_msg_disconnect{code = ?SSH_DISCONNECT_BY_APPLICATION, - description = "Connection closed"}, - StateName, - D); +handle_event(info, {CloseTag,Socket}, _StateName, + D0 = #data{socket = Socket, + transport_close_tag = CloseTag, + connection_state = C0}) -> + {Repls, D} = send_replies(ssh_connection:handle_stop(C0), D0), + disconnect_fun("Received a transport close", D), + {stop_and_reply, {shutdown,"Connection closed"}, Repls, D}; handle_event(info, {timeout, {_, From} = Request}, _, #data{connection_state = #connection{requests = Requests} = C0} = D) -> @@ -1374,13 +1419,24 @@ handle_event(info, {timeout, {_, From} = Request}, _, %%% Handle that ssh channels user process goes down handle_event(info, {'DOWN', _Ref, process, ChannelPid, _Reason}, _, D0) -> - {{replies, Replies}, D1} = handle_channel_down(ChannelPid, D0), - {Repls, D} = send_replies(Replies, D1), - {keep_state, D, Repls}; + {keep_state, handle_channel_down(ChannelPid, D0)}; %%% So that terminate will be run when supervisor is shutdown -handle_event(info, {'EXIT', _Sup, Reason}, _, _) -> - {stop, {shutdown, Reason}}; +handle_event(info, {'EXIT', _Sup, Reason}, StateName, _) -> + Role = role(StateName), + if + Role == client -> + %% OTP-8111 tells this function clause fixes a problem in + %% clients, but there were no check for that role. + {stop, {shutdown, Reason}}; + + Reason == normal -> + %% An exit normal should not cause a server to crash. This has happend... + keep_state_and_data; + + true -> + {stop, {shutdown, Reason}} + end; handle_event(info, check_cache, _, D) -> {keep_state, cache_check_set_idle_timer(D)}; @@ -1425,62 +1481,78 @@ handle_event(info, UnexpectedMessage, StateName, D = #data{ssh_params = Ssh}) -> keep_state_and_data end; -handle_event(internal, {disconnect,Msg,_Reason}, StateName, D) -> - disconnect(Msg, StateName, D); +handle_event(internal, {send_disconnect,Code,DetailedText,Module,Line}, StateName, D0) -> + {Shutdown, D} = + send_disconnect(Code, DetailedText, Module, Line, StateName, D0), + {stop, Shutdown, D}; handle_event(_Type, _Msg, {ext_info,Role,_ReNegFlag}, D) -> %% If something else arrives, goto next state and handle the event in that one {next_state, {connected,Role}, D, [postpone]}; -handle_event(Type, Ev, StateName, D) -> - Descr = +handle_event(Type, Ev, StateName, D0) -> + Details = case catch atom_to_list(element(1,Ev)) of "ssh_msg_" ++_ when Type==internal -> -%% "Message in wrong state"; lists:flatten(io_lib:format("Message ~p in wrong state (~p)", [element(1,Ev), StateName])); _ -> - "Internal error" + io_lib:format("Unhandled event in state ~p:~n~p", [StateName,Ev]) end, - disconnect(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_PROTOCOL_ERROR, - description = Descr}, - StateName, D). + {Shutdown, D} = + ?send_disconnect(?SSH_DISCONNECT_PROTOCOL_ERROR, Details, StateName, D0), + {stop, Shutdown, D}. %%-------------------------------------------------------------------- -spec terminate(any(), state_name(), #data{} - ) -> finalize_termination_result() . + ) -> term(). %% . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . -terminate(normal, StateName, State) -> - finalize_termination(StateName, State); +terminate(normal, _StateName, D) -> + stop_subsystem(D), + close_transport(D); -terminate({shutdown,{init,Reason}}, StateName, State) -> - error_logger:info_report(io_lib:format("Erlang ssh in connection handler init: ~p~n",[Reason])), - finalize_termination(StateName, State); +terminate({shutdown,"Connection closed"}, _StateName, D) -> + %% Normal: terminated by a sent by peer + stop_subsystem(D), + close_transport(D); -terminate(shutdown, StateName, State0) -> - %% Terminated by supervisor - State = send_msg(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_BY_APPLICATION, - description = "Application shutdown"}, - State0), - finalize_termination(StateName, State); - -terminate({shutdown,_R}, StateName, State) -> - finalize_termination(StateName, State); +terminate({shutdown,{init,Reason}}, StateName, D) -> + %% Error in initiation. "This error should not occur". + log(error, D, io_lib:format("Shutdown in init (StateName=~p): ~p~n",[StateName,Reason])), + stop_subsystem(D), + close_transport(D); -terminate(kill, StateName, State) -> - finalize_termination(StateName, State); +terminate({shutdown,_R}, _StateName, D) -> + %% Internal termination, usually already reported via ?send_disconnect resulting in a log entry + stop_subsystem(D), + close_transport(D); -terminate(Reason, StateName, State0) -> - %% Others, e.g undef, {badmatch,_} - log_error(Reason), - State = send_msg(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_BY_APPLICATION, - description = "Internal error"}, - State0), - finalize_termination(StateName, State). +terminate(shutdown, _StateName, D0) -> + %% Terminated by supervisor + %% Use send_msg directly instead of ?send_disconnect to avoid filling the log + D = send_msg(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_BY_APPLICATION, + description = "Terminated (shutdown) by supervisor"}, + D0), + close_transport(D); + +terminate(kill, _StateName, D) -> + %% Got a kill signal + stop_subsystem(D), + close_transport(D); + +terminate(Reason, StateName, D0) -> + %% Others, e.g undef, {badmatch,_}, ... + log(error, D0, Reason), + {_ShutdownReason, D} = ?send_disconnect(?SSH_DISCONNECT_BY_APPLICATION, + "Internal error", + io_lib:format("Reason: ~p",[Reason]), + StateName, D0), + stop_subsystem(D), + close_transport(D). %%-------------------------------------------------------------------- @@ -1489,36 +1561,41 @@ terminate(Reason, StateName, State0) -> format_status(normal, [_, _StateName, D]) -> [{data, [{"State", D}]}]; format_status(terminate, [_, _StateName, D]) -> - DataPropList0 = fmt_stat_rec(record_info(fields, data), D, - [decrypted_data_buffer, - encrypted_data_buffer, - key_exchange_init_msg, - user_passwords, - opts, - inet_initial_recbuf_size]), - SshPropList = fmt_stat_rec(record_info(fields, ssh), D#data.ssh_params, - [c_keyinit, - s_keyinit, - send_mac_key, - send_mac_size, - recv_mac_key, - recv_mac_size, - encrypt_keys, - encrypt_ctx, - decrypt_keys, - decrypt_ctx, - compress_ctx, - decompress_ctx, - shared_secret, - exchanged_hash, - session_id, - keyex_key, - keyex_info, - available_host_keys]), - DataPropList = lists:keyreplace(ssh_params, 1, DataPropList0, - {ssh_params,SshPropList}), - [{data, [{"State", DataPropList}]}]. - + [{data, [{"State", state_data2proplist(D)}]}]. + + +state_data2proplist(D) -> + DataPropList0 = + fmt_stat_rec(record_info(fields, data), D, + [decrypted_data_buffer, + encrypted_data_buffer, + key_exchange_init_msg, + user_passwords, + opts, + inet_initial_recbuf_size]), + SshPropList = + fmt_stat_rec(record_info(fields, ssh), D#data.ssh_params, + [c_keyinit, + s_keyinit, + send_mac_key, + send_mac_size, + recv_mac_key, + recv_mac_size, + encrypt_keys, + encrypt_ctx, + decrypt_keys, + decrypt_ctx, + compress_ctx, + decompress_ctx, + shared_secret, + exchanged_hash, + session_id, + keyex_key, + keyex_info, + available_host_keys]), + lists:keyreplace(ssh_params, 1, DataPropList0, + {ssh_params,SshPropList}). + fmt_stat_rec(FieldNames, Rec, Exclude) -> Values = tl(tuple_to_list(Rec)), @@ -1555,21 +1632,25 @@ start_the_connection_child(UserPid, Role, Socket, Options0) -> %%-------------------------------------------------------------------- %% Stopping --type finalize_termination_result() :: ok . - -finalize_termination(_StateName, #data{transport_cb = Transport, - connection_state = Connection, - socket = Socket}) -> - case Connection of - #connection{system_supervisor = SysSup, - sub_system_supervisor = SubSysSup} when is_pid(SubSysSup) -> - ssh_system_sup:stop_subsystem(SysSup, SubSysSup); - _ -> - do_nothing - end, - (catch Transport:close(Socket)), + +stop_subsystem(#data{connection_state = + #connection{system_supervisor = SysSup, + sub_system_supervisor = SubSysSup}}) when is_pid(SubSysSup) -> + ssh_system_sup:stop_subsystem(SysSup, SubSysSup); +stop_subsystem(_) -> ok. + +close_transport(#data{transport_cb = Transport, + socket = Socket}) -> + try + Transport:close(Socket) + of + _ -> ok + catch + _:_ -> ok + end. + %%-------------------------------------------------------------------- %% "Invert" the Role peer_role(client) -> server; @@ -1668,7 +1749,20 @@ handle_connection_msg(Msg, StateName, D0 = #data{starter = User, Renegotiation = renegotiation(StateName), Role = role(StateName), try ssh_connection:handle_msg(Msg, Connection0, Role) of - {{replies, Replies}, Connection} -> + {disconnect, Reason0, RepliesConn} -> + {Repls, D} = send_replies(RepliesConn, D0), + case {Reason0,Role} of + {{_, Reason}, client} when ((StateName =/= {connected,client}) and (not Renegotiation)) -> + User ! {self(), not_connected, Reason}; + _ -> + ok + end, + {stop_and_reply, {shutdown,normal}, Repls, D}; + + {[], Connection} -> + {keep_state, D0#data{connection_state = Connection}}; + + {Replies, Connection} when is_list(Replies) -> {Repls, D} = case StateName of {connected,_} -> @@ -1677,30 +1771,15 @@ handle_connection_msg(Msg, StateName, D0 = #data{starter = User, {ConnReplies, NonConnReplies} = lists:splitwith(fun not_connected_filter/1, Replies), send_replies(NonConnReplies, D0#data{event_queue = Qev0 ++ ConnReplies}) end, - {keep_state, D, Repls}; - - {noreply, Connection} -> - {keep_state, D0#data{connection_state = Connection}}; - - {disconnect, Reason0, {{replies, Replies}, Connection}} -> - {Repls, D} = send_replies(Replies, D0#data{connection_state = Connection}), - case {Reason0,Role} of - {{_, Reason}, client} when ((StateName =/= {connected,client}) and (not Renegotiation)) -> - User ! {self(), not_connected, Reason}; - _ -> - ok - end, - {stop_and_reply, {shutdown,normal}, Repls, D#data{connection_state = Connection}} + {keep_state, D, Repls} catch - _:Error -> - {disconnect, _Reason, {{replies, Replies}, Connection}} = - ssh_connection:handle_msg( - #ssh_msg_disconnect{code = ?SSH_DISCONNECT_BY_APPLICATION, - description = "Internal error"}, - Connection0, Role), - {Repls, D} = send_replies(Replies, D0#data{connection_state = Connection}), - {stop_and_reply, {shutdown,Error}, Repls, D#data{connection_state = Connection}} + Class:Error -> + {Repls, D1} = send_replies(ssh_connection:handle_stop(Connection0), D0), + {Shutdown, D} = ?send_disconnect(?SSH_DISCONNECT_BY_APPLICATION, + io_lib:format("Internal error: ~p:~p",[Class,Error]), + StateName, D1), + {stop_and_reply, Shutdown, Repls, D} end. @@ -1780,7 +1859,7 @@ is_usable_user_pubkey(A, Ssh) -> %%%---------------------------------------------------------------- handle_request(ChannelPid, ChannelId, Type, Data, WantReply, From, D) -> - case ssh_channel:cache_lookup(cache(D), ChannelId) of + case ssh_client_channel:cache_lookup(cache(D), ChannelId) of #channel{remote_id = Id, sent_close = false} = Channel -> update_sys(cache(D), Channel, Type, ChannelPid), @@ -1795,7 +1874,7 @@ handle_request(ChannelPid, ChannelId, Type, Data, WantReply, From, D) -> end. handle_request(ChannelId, Type, Data, WantReply, From, D) -> - case ssh_channel:cache_lookup(cache(D), ChannelId) of + case ssh_client_channel:cache_lookup(cache(D), ChannelId) of #channel{remote_id = Id, sent_close = false} -> send_msg(ssh_connection:channel_request_msg(Id, Type, WantReply, Data), @@ -1810,19 +1889,20 @@ handle_request(ChannelId, Type, Data, WantReply, From, D) -> %%%---------------------------------------------------------------- handle_channel_down(ChannelPid, D) -> - ssh_channel:cache_foldl( - fun(Channel, Acc) when Channel#channel.user == ChannelPid -> - ssh_channel:cache_delete(cache(D), - Channel#channel.local_id), - Acc; - (_,Acc) -> - Acc - end, [], cache(D)), - {{replies, []}, cache_check_set_idle_timer(D)}. + Cache = cache(D), + ssh_client_channel:cache_foldl( + fun(#channel{user=U, + local_id=Id}, Acc) when U == ChannelPid -> + ssh_client_channel:cache_delete(Cache, Id), + Acc; + (_,Acc) -> + Acc + end, [], Cache), + cache_check_set_idle_timer(D). update_sys(Cache, Channel, Type, ChannelPid) -> - ssh_channel:cache_update(Cache, + ssh_client_channel:cache_update(Cache, Channel#channel{sys = Type, user = ChannelPid}). add_request(false, _ChannelId, _From, State) -> @@ -1840,11 +1920,49 @@ new_channel_id(#data{connection_state = #connection{channel_id_seed = Id} = Connection#connection{channel_id_seed = Id + 1}}}. %%%---------------------------------------------------------------- -%% %%% This server/client has decided to disconnect via the state machine: -disconnect(Msg=#ssh_msg_disconnect{description=Description}, _StateName, State0) -> - State = send_msg(Msg, State0), - disconnect_fun(Description, State), - {stop, {shutdown,Description}, State}. +%%% This server/client has decided to disconnect via the state machine: +%%% The unused arguments are for debugging. + +send_disconnect(Code, DetailedText, Module, Line, StateName, D) -> + send_disconnect(Code, default_text(Code), DetailedText, Module, Line, StateName, D). + +send_disconnect(Code, Reason, DetailedText, Module, Line, StateName, D0) -> + Msg = #ssh_msg_disconnect{code = Code, + description = Reason}, + D = send_msg(Msg, D0), + LogMsg = io_lib:format("Disconnects with code = ~p [RFC4253 11.1]: ~s",[Code,Reason]), + call_disconnectfun_and_log_cond(LogMsg, DetailedText, Module, Line, StateName, D), + {{shutdown,Reason}, D}. + +call_disconnectfun_and_log_cond(LogMsg, DetailedText, Module, Line, StateName, D) -> + case disconnect_fun(LogMsg, D) of + void -> + log(info, D, + io_lib:format("~s~n" + "State = ~p~n" + "Module = ~p, Line = ~p.~n" + "Details:~n ~s~n", + [LogMsg, StateName, Module, Line, DetailedText])); + _ -> + ok + end. + + +default_text(?SSH_DISCONNECT_HOST_NOT_ALLOWED_TO_CONNECT) -> "Host not allowed to connect"; +default_text(?SSH_DISCONNECT_PROTOCOL_ERROR) -> "Protocol error"; +default_text(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED) -> "Key exchange failed"; +default_text(?SSH_DISCONNECT_RESERVED) -> "Reserved"; +default_text(?SSH_DISCONNECT_MAC_ERROR) -> "Mac error"; +default_text(?SSH_DISCONNECT_COMPRESSION_ERROR) -> "Compression error"; +default_text(?SSH_DISCONNECT_SERVICE_NOT_AVAILABLE) -> "Service not available"; +default_text(?SSH_DISCONNECT_PROTOCOL_VERSION_NOT_SUPPORTED) -> "Protocol version not supported"; +default_text(?SSH_DISCONNECT_HOST_KEY_NOT_VERIFIABLE) -> "Host key not verifiable"; +default_text(?SSH_DISCONNECT_CONNECTION_LOST) -> "Connection lost"; +default_text(?SSH_DISCONNECT_BY_APPLICATION) -> "By application"; +default_text(?SSH_DISCONNECT_TOO_MANY_CONNECTIONS) -> "Too many connections"; +default_text(?SSH_DISCONNECT_AUTH_CANCELLED_BY_USER) -> "Auth cancelled by user"; +default_text(?SSH_DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE) -> "Unable to connect using the available authentication methods"; +default_text(?SSH_DISCONNECT_ILLEGAL_USER_NAME) -> "Illegal user name". %%%---------------------------------------------------------------- counterpart_versions(NumVsn, StrVsn, #ssh{role = server} = Ssh) -> @@ -1857,12 +1975,11 @@ conn_info(client_version, #data{ssh_params=S}) -> {S#ssh.c_vsn, S#ssh.c_version} conn_info(server_version, #data{ssh_params=S}) -> {S#ssh.s_vsn, S#ssh.s_version}; conn_info(peer, #data{ssh_params=S}) -> S#ssh.peer; conn_info(user, D) -> D#data.auth_user; -conn_info(sockname, D) -> {ok, SockName} = inet:sockname(D#data.socket), - SockName; +conn_info(sockname, #data{ssh_params=S}) -> S#ssh.local; %% dbg options ( = not documented): conn_info(socket, D) -> D#data.socket; conn_info(chan_ids, D) -> - ssh_channel:cache_foldl(fun(#channel{local_id=Id}, Acc) -> + ssh_client_channel:cache_foldl(fun(#channel{local_id=Id}, Acc) -> [Id | Acc] end, [], cache(D)). @@ -1889,23 +2006,54 @@ fold_keys(Keys, Fun, Extra) -> end, [], Keys). %%%---------------------------------------------------------------- -log_error(Reason) -> - Report = io_lib:format("Erlang ssh connection handler failed with reason:~n" - " ~p~n" - "Stacktrace:~n" - " ~p~n", - [Reason, erlang:get_stacktrace()]), - error_logger:error_report(Report). +log(Tag, D, Reason) -> + case atom_to_list(Tag) of % Dialyzer-technical reasons... + "error" -> do_log(error_msg, Reason, D); + "warning" -> do_log(warning_msg, Reason, D); + "info" -> do_log(info_msg, Reason, D) + end. + +do_log(F, Reason, #data{ssh_params = #ssh{role = Role} = S + }) -> + VSN = + case application:get_key(ssh,vsn) of + {ok,Vsn} -> Vsn; + undefined -> "" + end, + PeerVersion = + case Role of + server -> S#ssh.c_version; + client -> S#ssh.s_version + end, + CryptoInfo = + try + [{_,_,CI}] = crypto:info_lib(), + <<"(",CI/binary,")">> + catch + _:_ -> "" + end, + Other = + case Role of + server -> "Client"; + client -> "Server" + end, + error_logger:F("Erlang SSH ~p ~s ~s.~n" + "~s: ~p~n" + "~s~n", + [Role, VSN, CryptoInfo, + Other, PeerVersion, + Reason]). %%%---------------------------------------------------------------- not_connected_filter({connection_reply, _Data}) -> true; not_connected_filter(_) -> false. %%%---------------------------------------------------------------- + +send_replies({Repls,C = #connection{}}, D) when is_list(Repls) -> + send_replies(Repls, D#data{connection_state=C}); send_replies(Repls, State) -> - lists:foldl(fun get_repl/2, - {[],State}, - Repls). + lists:foldl(fun get_repl/2, {[],State}, Repls). get_repl({connection_reply,Msg}, {CallRepls,S}) -> if is_record(Msg, ssh_msg_channel_success) -> @@ -1922,19 +2070,21 @@ get_repl({channel_data,Pid,Data}, Acc) -> get_repl({channel_request_reply,From,Data}, {CallRepls,S}) -> {[{reply,From,Data}|CallRepls], S}; get_repl({flow_control,Cache,Channel,From,Msg}, {CallRepls,S}) -> - ssh_channel:cache_update(Cache, Channel#channel{flow_control = undefined}), + ssh_client_channel:cache_update(Cache, Channel#channel{flow_control = undefined}), {[{reply,From,Msg}|CallRepls], S}; get_repl({flow_control,From,Msg}, {CallRepls,S}) -> {[{reply,From,Msg}|CallRepls], S}; -get_repl(noreply, Acc) -> - Acc; +%% get_repl(noreply, Acc) -> +%% Acc; +%% get_repl([], Acc) -> +%% Acc; get_repl(X, Acc) -> exit({get_repl,X,Acc}). %%%---------------------------------------------------------------- -define(CALL_FUN(Key,D), catch (?GET_OPT(Key, (D#data.ssh_params)#ssh.opts)) ). -disconnect_fun({disconnect,Msg}, D) -> ?CALL_FUN(disconnectfun,D)(Msg); +%%disconnect_fun({disconnect,Msg}, D) -> ?CALL_FUN(disconnectfun,D)(Msg); disconnect_fun(Reason, D) -> ?CALL_FUN(disconnectfun,D)(Reason). unexpected_fun(UnexpectedMessage, #data{ssh_params = #ssh{peer = {_,Peer} }} = D) -> @@ -1996,7 +2146,7 @@ cache_init_idle_timer(D) -> cache_check_set_idle_timer(D = #data{idle_timer_ref = undefined, idle_timer_value = IdleTime}) -> %% No timer set - shall we set one? - case ssh_channel:cache_info(num_entries, cache(D)) of + case ssh_client_channel:cache_info(num_entries, cache(D)) of 0 when IdleTime == infinity -> %% No. Meaningless to set a timer that fires in an infinite time... D; @@ -2089,3 +2239,137 @@ update_inet_buffers(Socket) -> catch _:_ -> ok end. + +%%%################################################################ +%%%# +%%%# Tracing +%%%# + +dbg_trace(points, _, _) -> [terminate, disconnect, connections, connection_events]; + +dbg_trace(flags, connections, A) -> [c] ++ dbg_trace(flags, terminate, A); +dbg_trace(on, connections, A) -> dbg:tp(?MODULE, init_connection_handler, 3, x), + dbg_trace(on, terminate, A); +dbg_trace(off, connections, A) -> dbg:ctpg(?MODULE, init_connection_handler, 3), + dbg_trace(off, terminate, A); +dbg_trace(format, connections, {call, {?MODULE,init_connection_handler, [Role, Sock, Opts]}}) -> + DefaultOpts = ssh_options:handle_options(Role,[]), + ExcludedKeys = [internal_options, user_options], + NonDefaultOpts = + maps:filter(fun(K,V) -> + case lists:member(K,ExcludedKeys) of + true -> + false; + false -> + V =/= (catch maps:get(K,DefaultOpts)) + end + end, + Opts), + {ok, {IPp,Portp}} = inet:peername(Sock), + {ok, {IPs,Ports}} = inet:sockname(Sock), + [io_lib:format("Starting ~p connection:\n",[Role]), + io_lib:format("Socket = ~p, Peer = ~s:~p, Local = ~s:~p,~n" + "Non-default options:~n~p", + [Sock,inet:ntoa(IPp),Portp,inet:ntoa(IPs),Ports, + NonDefaultOpts]) + ]; +dbg_trace(format, connections, F) -> + dbg_trace(format, terminate, F); + +dbg_trace(flags, connection_events, _) -> [c]; +dbg_trace(on, connection_events, _) -> dbg:tp(?MODULE, handle_event, 4, x); +dbg_trace(off, connection_events, _) -> dbg:ctpg(?MODULE, handle_event, 4); +dbg_trace(format, connection_events, {call, {?MODULE,handle_event, [EventType, EventContent, State, _Data]}}) -> + ["Connection event\n", + io_lib:format("EventType: ~p~nEventContent: ~p~nState: ~p~n", [EventType, EventContent, State]) + ]; +dbg_trace(format, connection_events, {return_from, {?MODULE,handle_event,4}, Ret}) -> + ["Connection event result\n", + io_lib:format("~p~n", [event_handler_result(Ret)]) + ]; + +dbg_trace(flags, terminate, _) -> [c]; +dbg_trace(on, terminate, _) -> dbg:tp(?MODULE, terminate, 3, x); +dbg_trace(off, terminate, _) -> dbg:ctpg(?MODULE, terminate, 3); +dbg_trace(format, terminate, {call, {?MODULE,terminate, [Reason, StateName, D]}}) -> + ExtraInfo = + try + {conn_info(peer,D), + conn_info(user,D), + conn_info(sockname,D)} + of + {{_,{IPp,Portp}}, Usr, {IPs,Ports}} when is_tuple(IPp), is_tuple(IPs), + is_integer(Portp), is_integer(Ports) -> + io_lib:format("Peer=~s:~p, Local=~s:~p, User=~p", + [inet:ntoa(IPp),Portp,inet:ntoa(IPs),Ports,Usr]); + {Peer,Usr,Sockname} -> + io_lib:format("Peer=~p, Local=~p, User=~p",[Peer,Sockname,Usr]) + catch + _:_ -> + "" + end, + if + Reason == normal ; + Reason == shutdown ; + element(1,Reason) == shutdown + -> + ["Connection Terminating:\n", + io_lib:format("Reason: ~p, StateName: ~p~n~s", [Reason, StateName, ExtraInfo]) + ]; + + true -> + ["Connection Terminating:\n", + io_lib:format("Reason: ~p, StateName: ~p~n~s~nStateData = ~p", + [Reason, StateName, ExtraInfo, state_data2proplist(D)]) + ] + end; + +dbg_trace(flags, disconnect, _) -> [c]; +dbg_trace(on, disconnect, _) -> dbg:tpl(?MODULE, send_disconnect, 7, x); +dbg_trace(off, disconnect, _) -> dbg:ctpl(?MODULE, send_disconnect, 7); +dbg_trace(format, disconnect, {call,{?MODULE,send_disconnect, + [Code, Reason, DetailedText, Module, Line, StateName, _D]}}) -> + ["Disconnecting:\n", + io_lib:format(" Module = ~p, Line = ~p, StateName = ~p,~n" + " Code = ~p, Reason = ~p,~n" + " DetailedText =~n" + " ~p", + [Module, Line, StateName, Code, Reason, lists:flatten(DetailedText)]) + ]. + + +event_handler_result({next_state, NextState, _NewData}) -> + {next_state, NextState, "#data{}"}; +event_handler_result({next_state, NextState, _NewData, Actions}) -> + {next_state, NextState, "#data{}", Actions}; +event_handler_result(R) -> + state_callback_result(R). + +state_callback_result({keep_state, _NewData}) -> + {keep_state, "#data{}"}; +state_callback_result({keep_state, _NewData, Actions}) -> + {keep_state, "#data{}", Actions}; +state_callback_result(keep_state_and_data) -> + keep_state_and_data; +state_callback_result({keep_state_and_data, Actions}) -> + {keep_state_and_data, Actions}; +state_callback_result({repeat_state, _NewData}) -> + {repeat_state, "#data{}"}; +state_callback_result({repeat_state, _NewData, Actions}) -> + {repeat_state, "#data{}", Actions}; +state_callback_result(repeat_state_and_data) -> + repeat_state_and_data; +state_callback_result({repeat_state_and_data, Actions}) -> + {repeat_state_and_data, Actions}; +state_callback_result(stop) -> + stop; +state_callback_result({stop, Reason}) -> + {stop, Reason}; +state_callback_result({stop, Reason, _NewData}) -> + {stop, Reason, "#data{}"}; +state_callback_result({stop_and_reply, Reason, Replies}) -> + {stop_and_reply, Reason, Replies}; +state_callback_result({stop_and_reply, Reason, Replies, _NewData}) -> + {stop_and_reply, Reason, Replies, "#data{}"}; +state_callback_result(R) -> + R. diff --git a/lib/ssh/src/ssh_daemon_channel.erl b/lib/ssh/src/ssh_daemon_channel.erl index 6ca93eff44..fdb6c10971 100644 --- a/lib/ssh/src/ssh_daemon_channel.erl +++ b/lib/ssh/src/ssh_daemon_channel.erl @@ -25,7 +25,7 @@ -module(ssh_daemon_channel). -%% API to special server side channel that can be pluged into the erlang ssh daemeon +%% API to server side channel that can be pluged into the erlang ssh daemeon -callback init(Args :: term()) -> {ok, State :: term()} | {ok, State :: term(), timeout() | hibernate} | {stop, Reason :: term()} | ignore. @@ -36,34 +36,20 @@ term(). -callback handle_msg(Msg ::term(), State :: term()) -> - {ok, State::term()} | {stop, ChannelId::integer(), State::term()}. --callback handle_ssh_msg({ssh_cm, ConnectionRef::term(), SshMsg::term()}, + {ok, State::term()} | {stop, ChannelId::ssh:channel_id(), State::term()}. +-callback handle_ssh_msg({ssh_cm, ConnectionRef::ssh:connection_ref(), SshMsg::term()}, State::term()) -> {ok, State::term()} | - {stop, ChannelId::integer(), + {stop, ChannelId::ssh:channel_id(), State::term()}. -%%% API --export([start/4, start/5, start_link/4, start_link/5, enter_loop/1]). - -%% gen_server callbacks --export([init/1, terminate/2]). - -start(ConnectionManager, ChannelId, CallBack, CbInitArgs) -> - ssh_channel:start(ConnectionManager, ChannelId, CallBack, CbInitArgs, undefined). - -start(ConnectionManager, ChannelId, CallBack, CbInitArgs, Exec) -> - ssh_channel:start(ConnectionManager, ChannelId, CallBack, CbInitArgs, Exec). - -start_link(ConnectionManager, ChannelId, CallBack, CbInitArgs) -> - ssh_channel:start_link(ConnectionManager, ChannelId, CallBack, CbInitArgs, undefined). +%%% Internal API +-export([start_link/5, + get_print_info/1 + ]). start_link(ConnectionManager, ChannelId, CallBack, CbInitArgs, Exec) -> - ssh_channel:start_link(ConnectionManager, ChannelId, CallBack, CbInitArgs, Exec). + ssh_server_channel:start_link(ConnectionManager, ChannelId, CallBack, CbInitArgs, Exec). -enter_loop(State) -> - ssh_channel:enter_loop(State). -init(Args) -> - ssh_channel:init(Args). -terminate(Reason, State) -> - ssh_channel:terminate(Reason, State). +get_print_info(Pid) -> + ssh_server_channel:get_print_info(Pid). diff --git a/lib/ssh/src/ssh_dbg.erl b/lib/ssh/src/ssh_dbg.erl index eb2c2848f3..2ee4237e05 100644 --- a/lib/ssh/src/ssh_dbg.erl +++ b/lib/ssh/src/ssh_dbg.erl @@ -20,339 +20,110 @@ %% +%%% Purpose: +%%% This module implements support for using the Erlang trace in a simple way for ssh +%%% debugging. +%%% +%%% Begin the session with ssh_dbg:start(). This will do a dbg:start() if needed and +%%% then dbg:p/2 to set some flags. +%%% +%%% Next select trace points to activate: for example plain text printouts of messages +%%% sent or received. This is switched on and off with ssh_dbg:on(TracePoint(s)) and +%%% ssh_dbg:off(TracePoint(s)). For example: +%%% +%%% ssh_dbg:on(messages) -- switch on printing plain text messages +%%% ssh_dbg:on([alg,terminate]) -- switch on printing info about algorithm negotiation +%%% ssh_dbg:on() -- switch on all ssh debugging +%%% +%%% To switch, use the off/0 or off/1 function in the same way, for example: +%%% +%%% ssh_dbg:off(alg) -- switch off algorithm negotiation tracing, but keep all other +%%% ssh_dbg:off() -- switch off all ssh debugging +%%% +%%% Present the trace result with some other method than the default io:format/2: +%%% ssh_dbg:start(fun(Format,Args) -> +%%% my_special( io_lib:format(Format,Args) ) +%%% end) +%%% + -module(ssh_dbg). --export([messages/0, messages/1, messages/2, messages/3, - auth/0, auth/1, auth/2, auth/3, - algs/0, algs/1, algs/2, algs/3, - hostkey/0, hostkey/1, hostkey/2, hostkey/3, - stop/0 +-export([start/0, start/1, + stop/0, + start_server/0, + start_tracer/0, start_tracer/1, + on/1, on/0, + off/1, off/0, + go_on/0 ]). -export([shrink_bin/1, - wr_record/3]). + reduce_state/1, + wr_record/3]). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2]). -include("ssh.hrl"). -include("ssh_transport.hrl"). -include("ssh_connect.hrl"). -include("ssh_auth.hrl"). -%%%================================================================ -messages() -> start(msg). -messages(F) -> start(msg,F). -messages(F,X) -> start(msg,F,X). -messages(F,M,I) -> start(msg,F,M,I). - -auth() -> start(auth). -auth(F) -> start(auth,F). -auth(F,X) -> start(auth,F,X). -auth(F,M,I) -> start(auth,F,M,I). - -algs() -> start(algs). -algs(F) -> start(algs,F). -algs(F,X) -> start(algs,F,X). -algs(F,M,I) -> start(algs,F,M,I). - -hostkey() -> start(hostkey). -hostkey(F) -> start(hostkey,F). -hostkey(F,X) -> start(hostkey,F,X). -hostkey(F,M,I) -> start(hostkey,F,M,I). - -stop() -> dbg:stop(). - -%%%---------------------------------------------------------------- -start(Type) -> start(Type, fun io:format/2). +-behaviour(gen_server). +-define(SERVER, ?MODULE). -start(Type, F) when is_function(F,2) -> start(Type, fmt_fun(F)); -start(Type, F) when is_function(F,3) -> start(Type, F, id_fun()). - -start(Type, WriteFun, MangleArgFun) when is_function(WriteFun, 3), - is_function(MangleArgFun, 1) -> - start(Type, WriteFun, MangleArgFun, []); -start(Type, WriteFun, InitValue) -> - start(Type, WriteFun, id_fun(), InitValue). +%%%================================================================ -start(Type, WriteFun, MangleArgFun, InitValue) when is_function(WriteFun, 3), - is_function(MangleArgFun, 1) -> - cond_start(Type, WriteFun, MangleArgFun, InitValue), - dbg_ssh(Type). +-define(ALL_DBG_TYPES, get_all_dbg_types()). -%%%---------------------------------------------------------------- -fmt_fun(F) -> fun(Fmt,Args,Data) -> F(Fmt,Args), Data end. +start() -> start(fun io:format/2). -id_fun() -> fun(X) -> X end. +start(IoFmtFun) when is_function(IoFmtFun,2) ; is_function(IoFmtFun,3) -> + start_server(), + catch dbg:start(), + start_tracer(IoFmtFun), + dbg:p(all, get_all_trace_flags()), + ?ALL_DBG_TYPES. -%%%---------------------------------------------------------------- -dbg_ssh(What) -> - case [E || E <- lists:flatten(dbg_ssh0(What)), - element(1,E) =/= ok] of - [] -> ok; - Other -> Other - end. - - -dbg_ssh0(auth) -> - [dbg:tp(ssh_transport,hello_version_msg,1, x), - dbg:tp(ssh_transport,handle_hello_version,1, x), - dbg:tp(ssh_message,encode,1, x), - dbg:tpl(ssh_transport,select_algorithm,4, x), - dbg:tpl(ssh_connection_handler,ext_info,2, x), - lists:map(fun(F) -> dbg:tp(ssh_auth, F, x) end, - [publickey_msg, password_msg, keyboard_interactive_msg]) - ]; - -dbg_ssh0(algs) -> - [dbg:tpl(ssh_transport,select_algorithm,4, x), - dbg:tpl(ssh_connection_handler,ext_info,2, x) - ]; - -dbg_ssh0(hostkey) -> - [dbg:tpl(ssh_transport, verify_host_key, 4, x), - dbg:tp(ssh_transport, verify, 4, x), - dbg:tpl(ssh_transport, known_host_key, 3, x), -%% dbg:tpl(ssh_transport, accepted_host, 4, x), - dbg:tpl(ssh_transport, add_host_key, 4, x), - dbg:tpl(ssh_transport, is_host_key, 5, x) - ]; - -dbg_ssh0(msg) -> - [dbg_ssh0(hostkey), - dbg_ssh0(auth), - dbg:tp(ssh_message,encode,1, x), - dbg:tp(ssh_message,decode,1, x), - dbg:tpl(ssh_transport,select_algorithm,4, x), - dbg:tp(ssh_transport,hello_version_msg,1, x), - dbg:tp(ssh_transport,handle_hello_version,1, x), - dbg:tpl(ssh_connection_handler,ext_info,2, x) - ]. - - -%%%================================================================ -cond_start(Type, WriteFun, MangleArgFun, Init) -> +stop() -> try - dbg:start(), - setup_tracer(Type, WriteFun, MangleArgFun, Init), - dbg:p(new,[c,timestamp]) + dbg:stop_clear(), + gen_server:stop(?SERVER) catch _:_ -> ok end. +start_server() -> + gen_server:start({local,?SERVER}, ?MODULE, [], []). + -msg_formater(msg, {trace_ts,Pid,call,{ssh_message,encode,[Msg]},TS}, D) -> - fmt("~n~s SEND ~p ~s~n", [ts(TS),Pid,wr_record(shrink_bin(Msg))], D); -msg_formater(msg, {trace_ts,_Pid,return_from,{ssh_message,encode,1},_Res,_TS}, D) -> - D; - -msg_formater(msg, {trace_ts,_Pid,call,{ssh_message,decode,_},_TS}, D) -> - D; -msg_formater(msg, {trace_ts,Pid,return_from,{ssh_message,decode,1},Msg,TS}, D) -> - Extra = - case Msg of - #ssh_msg_userauth_info_request{data = D0} -> - try ssh_message:decode_keyboard_interactive_prompts(D0, []) - of - Acc -> - io_lib:format(" -- decoded data:~n", []) ++ - element(1, - lists:mapfoldl( - fun({Prompt,Echo}, N) -> - {io_lib:format(" prompt[~p]: \"~s\" (echo=~p)~n",[N,Prompt,Echo]), N+1} - end, 1, Acc)) - catch - _:_ -> - "" - end; - _ -> - "" +start_tracer() -> start_tracer(fun io:format/2). + +start_tracer(WriteFun) when is_function(WriteFun,2) -> + start_tracer(fun(F,A,S) -> WriteFun(F,A), S end); +start_tracer(WriteFun) when is_function(WriteFun,3) -> + start_tracer(WriteFun, undefined). + + +start_tracer(WriteFun, InitAcc) when is_function(WriteFun, 3) -> + Handler = + fun(Arg, Acc0) -> + try_all_types_in_all_modules(gen_server:call(?SERVER, get_on), + Arg, WriteFun, + Acc0) end, - fmt("~n~s ~p RECV ~s~s~n", [ts(TS),Pid,wr_record(shrink_bin(Msg)),Extra], D); - -msg_formater(_auth, {trace_ts,Pid,return_from,{ssh_message,decode,1},#ssh_msg_userauth_failure{authentications=As},TS}, D) -> - fmt("~n~s ~p Client login FAILURE. Try ~s~n", [ts(TS),Pid,As], D); - -msg_formater(_auth, {trace_ts,Pid,return_from,{ssh_message,decode,1},#ssh_msg_userauth_success{},TS}, D) -> - fmt("~n~s ~p Client login SUCCESS~n", [ts(TS),Pid], D); - - -msg_formater(_, {trace_ts,_Pid,call,{ssh_transport,select_algorithm,_},_TS}, D) -> - D; -msg_formater(_, {trace_ts,Pid,return_from,{ssh_transport,select_algorithm,_},{ok,Alg},TS}, D) -> - fmt("~n~s ~p ALGORITHMS~n~s~n", [ts(TS),Pid, wr_record(Alg)], D); - -msg_formater(_, {trace_ts,_Pid,call,{ssh_transport,hello_version_msg,_},_TS}, D) -> - D; -msg_formater(_, {trace_ts,Pid,return_from,{ssh_transport,hello_version_msg,1},Hello,TS}, D) -> - fmt("~n~s ~p TCP SEND HELLO~n ~p~n", [ts(TS),Pid,lists:flatten(Hello)], D); - -msg_formater(_, {trace_ts,Pid,call,{ssh_transport,handle_hello_version,[Hello]},TS}, D) -> - fmt("~n~s ~p RECV HELLO~n ~p~n", [ts(TS),Pid,lists:flatten(Hello)], D); -msg_formater(_, {trace_ts,_Pid,return_from,{ssh_transport,handle_hello_version,1},_,_TS}, D) -> - D; - -msg_formater(_, {trace_ts,Pid,call,{ssh_connection_handler,ext_info,[{"server-sig-algs",SigAlgs},State]},TS}, D) -> - try lists:keyfind(ssh, 1, tuple_to_list(State)) of - false -> - D; - #ssh{userauth_pubkeys = PKs} -> - fmt("~n~s ~p Client got suggestion to use user public key sig-algs~n ~p~n and can use~n ~p~n", - [ts(TS),Pid,string:tokens(SigAlgs,","),PKs], D) - catch - _:_ -> - D - end; - -msg_formater(_, {trace_ts,Pid,return_from,{ssh_connection_handler,ext_info,2},State,TS}, D) -> - try lists:keyfind(ssh, 1, tuple_to_list(State)) of - false -> - D; - #ssh{userauth_pubkeys = PKs} -> - fmt("~n~s ~p Client will try user public key sig-algs~n ~p~n", [ts(TS),Pid,PKs], D) - catch - _:_ -> - D - end; - -msg_formater(_, {trace_ts,Pid,call, {ssh_transport,verify_host_key,[_Ssh,_PK,_Dgst,{AlgStr,_Sign}]},TS}, D) -> - fmt("~n~s ~p Client got a ~s hostkey. Will try to verify it~n", [ts(TS),Pid,AlgStr], D); -msg_formater(_, {trace_ts,Pid,return_from, {ssh_transport,verify_host_key,4}, Result, TS}, D) -> - case Result of - ok -> fmt("~n~s ~p Hostkey verified.~n", [ts(TS),Pid], D); - {error,E} -> - fmt("~n~s ~p ***** Hostkey NOT verified: ~p ******!~n", [ts(TS),Pid,E], D); - _ -> fmt("~n~s ~p ***** Hostkey is NOT verified: ~p ******!~n", [ts(TS),Pid,Result], D) - end; - -msg_formater(_, {trace_ts,Pid,return_from, {ssh_transport,verify,4}, Result, TS}, D) -> - case Result of - true -> D; - _ -> fmt("~n~s ~p Couldn't verify the signature!~n", [ts(TS),Pid], D) - end; - -msg_formater(_, {trace_ts,_Pid,call, {ssh_transport,is_host_key,_}, _TS}, D) -> D; -msg_formater(_, {trace_ts,Pid,return_from, {ssh_transport,is_host_key,5}, {CbMod,Result}, TS}, D) -> - case Result of - true -> fmt("~n~s ~p Hostkey found by ~p.~n", [ts(TS),Pid,CbMod], D); - _ -> fmt("~n~s ~p Hostkey NOT found by ~p.~n", [ts(TS),Pid,CbMod], D) - end; - -msg_formater(_, {trace_ts,_Pid,call, {ssh_transport,add_host_key,_}, _TS}, D) -> D; -msg_formater(_, {trace_ts,Pid,return_from, {ssh_transport,add_host_key,4}, {CbMod,Result}, TS}, D) -> - case Result of - ok -> fmt("~n~s ~p New hostkey added by ~p.~n", [ts(TS),Pid,CbMod], D); - _ -> D - end; - -msg_formater(_, {trace_ts,_Pid,call,{ssh_transport,known_host_key,_},_TS}, D) -> D; -msg_formater(_, {trace_ts,Pid,return_from, {ssh_transport,known_host_key,3}, Result, TS}, D) -> - case Result of - ok -> D; - {error,E} -> fmt("~n~s ~p Hostkey addition failed: ~p~n", [ts(TS),Pid,E], D); - _ -> fmt("~n~s ~p Hostkey addition: ~p~n", [ts(TS),Pid,Result], D) - end; - -msg_formater(_, {trace_ts,Pid,call,{ssh_auth,publickey_msg,[[SigAlg,#ssh{user=User}]]},TS}, D) -> - fmt("~n~s ~p Client will try to login user ~p with method: public key algorithm ~p~n", [ts(TS),Pid,User,SigAlg], D); -msg_formater(_, {trace_ts,Pid,return_from,{ssh_auth,publickey_msg,1},{not_ok,#ssh{user=User}},TS}, D) -> - fmt("~s ~p User ~p can't use that kind of public key~n", [ts(TS),Pid,User], D); -msg_formater(_, {trace_ts,_Pid,return_from,{ssh_auth,publickey_msg,1},_,_TS}, D) -> D; - -msg_formater(_, {trace_ts,Pid,call,{ssh_auth,password_msg,[[#ssh{user=User}]]},TS}, D) -> - fmt("~n~s ~p Client will try to login user ~p with method: password~n", [ts(TS),Pid,User], D); -msg_formater(_, {trace_ts,Pid,return_from,{ssh_auth,password_msg,1},{not_ok,#ssh{user=User}},TS}, D) -> - fmt("~s ~p User ~p can't use method password as login method~n", [ts(TS),Pid,User], D); -msg_formater(_, {trace_ts,_Pid,return_from,{ssh_auth,password_msg,1},_Result,_TS}, D) -> D; - -msg_formater(_, {trace_ts,Pid,call,{ssh_auth,keyboard_interactive_msg,[[#ssh{user=User}]]},TS}, D) -> - fmt("~n~s ~p Client will try to login user ~p with method: keyboard-interactive~n", [ts(TS),Pid,User], D); -msg_formater(_, {trace_ts,Pid,return_from,{ssh_auth,keyboard_interactive_msg,1},{not_ok,#ssh{user=User}},TS}, D) -> - fmt("~s ~p User ~p can't use method keyboard-interactive as login method~n", [ts(TS),Pid,User], D); -msg_formater(_, {trace_ts,_Pid,return_from,{ssh_auth,keyboard_interactive_msg,1},_Result,_TS}, D) -> D; - -msg_formater(msg, {trace_ts,Pid,send,{tcp,Sock,Bytes},Pid,TS}, D) -> - fmt("~n~s ~p TCP SEND on ~p~n ~p~n", [ts(TS),Pid,Sock, shrink_bin(Bytes)], D); - -msg_formater(msg, {trace_ts,Pid,send,{tcp,Sock,Bytes},Dest,TS}, D) -> - fmt("~n~s ~p TCP SEND from ~p TO ~p~n ~p~n", [ts(TS),Pid,Sock,Dest, shrink_bin(Bytes)], D); - -msg_formater(msg, {trace_ts,Pid,send,ErlangMsg,Dest,TS}, D) -> - fmt("~n~s ~p ERL MSG SEND TO ~p~n ~p~n", [ts(TS),Pid,Dest, shrink_bin(ErlangMsg)], D); - - -msg_formater(msg, {trace_ts,Pid,'receive',{tcp,Sock,Bytes},TS}, D) -> - fmt("~n~s ~p TCP RECEIVE on ~p~n ~p~n", [ts(TS),Pid,Sock,shrink_bin(Bytes)], D); - -msg_formater(msg, {trace_ts,Pid,'receive',ErlangMsg,TS}, D) -> - fmt("~n~s ~p ERL MSG RECEIVE~n ~p~n", [ts(TS),Pid,shrink_bin(ErlangMsg)], D); - - -msg_formater(_, _M, D) -> - fmt("~nDBG other ~n~p~n", [shrink_bin(_M)], D), - D. + dbg:tracer(process, {Handler,InitAcc}). %%%---------------------------------------------------------------- --record(data, {writer, - initialized, - acc}). - -fmt(Fmt, Args, D=#data{initialized=false}) -> - fmt(Fmt, Args, - D#data{acc = (D#data.writer)("~s~n", [initial_info()], D#data.acc), - initialized = true} - ); -fmt(Fmt, Args, D=#data{writer=Write, acc=Acc}) -> - D#data{acc = Write(Fmt,Args,Acc)}. - -ts({_,_,Usec}=Now) -> - {_Date,{HH,MM,SS}} = calendar:now_to_local_time(Now), - io_lib:format("~.2.0w:~.2.0w:~.2.0w.~.6.0w",[HH,MM,SS,Usec]); -ts(_) -> - "-". +on() -> on(?ALL_DBG_TYPES). +on(Type) -> switch(on, Type). -setup_tracer(Type, WriteFun, MangleArgFun, Init) -> - Handler = fun(Arg, D) -> - msg_formater(Type, MangleArgFun(Arg), D) - end, - InitialData = #data{writer = WriteFun, - initialized = false, - acc = Init}, - {ok,_} = dbg:tracer(process, {Handler, InitialData}), - ok. - - -initial_info() -> - Lines = - [ts(erlang:timestamp()), - "", - "SSH:"] - ++ as_list_of_lines(case application:get_key(ssh,vsn) of - {ok,Vsn} -> Vsn; - _ -> "(ssh not started)" - end) - ++ ["", - "Cryptolib:"] - ++ as_list_of_lines(crypto:info_lib()) - ++ ["", - "Crypto app:"] - ++ as_list_of_lines(crypto:supports()), - W = max_len(Lines), - append_lines([line_of($*, W+4)] - ++ prepend_lines("* ", Lines) - ++ [line_of($-, W+4)], - io_lib:nl() - ). - + +off() -> off(?ALL_DBG_TYPES). % A bit overkill... +off(Type) -> switch(off, Type). -as_list_of_lines(Term) -> - prepend_lines(" ", - string:tokens(lists:flatten(io_lib:format("~p",[Term])), - io_lib:nl() % Get line endings in current OS - ) - ). - -line_of(Char,W) -> lists:duplicate(W,Char). -max_len(L) -> lists:max([length(S) || S<-L]). -append_lines(L, X) -> [S++X || S<-L]. -prepend_lines(X, L) -> [X++S || S<-L]. +go_on() -> + IsOn = gen_server:call(?SERVER, get_on), + on(IsOn). %%%---------------------------------------------------------------- shrink_bin(B) when is_binary(B), size(B)>256 -> {'*** SHRINKED BIN', @@ -365,69 +136,198 @@ shrink_bin(L) when is_list(L) -> lists:map(fun shrink_bin/1, L); shrink_bin(T) when is_tuple(T) -> list_to_tuple(shrink_bin(tuple_to_list(T))); shrink_bin(X) -> X. +%%%---------------------------------------------------------------- +%% Replace last element (the state) with "#<state-name>{}" +reduce_state(T) -> + try + erlang:setelement(size(T), + T, + lists:concat(['#',element(1,element(size(T),T)),'{}']) + ) + catch + _:_ -> + T + end. + +%%%================================================================ +-record(data, { + types_on = [] + }). + +%%%---------------------------------------------------------------- +init(_) -> + {ok, #data{}}. + +%%%---------------------------------------------------------------- +handle_call({switch,on,Types}, _From, D) -> + NowOn = lists:usort(Types ++ D#data.types_on), + call_modules(on, Types, NowOn), + {reply, {ok,NowOn}, D#data{types_on = NowOn}}; + +handle_call({switch,off,Types}, _From, D) -> + StillOn = D#data.types_on -- Types, + call_modules(off, Types, StillOn), + call_modules(on, StillOn, StillOn), + {reply, {ok,StillOn}, D#data{types_on = StillOn}}; + +handle_call(get_on, _From, D) -> + {reply, D#data.types_on, D}; + +handle_call(C, _From, D) -> + io:format('*** Unknown call: ~p~n',[C]), + {reply, {error,{unknown_call,C}}, D}. + + +handle_cast(C, D) -> + io:format('*** Unknown cast: ~p~n',[C]), + {noreply, D}. + +handle_info(C, D) -> + io:format('*** Unknown info: ~p~n',[C]), + {noreply, D}. + + +%%%================================================================ + +%%%---------------------------------------------------------------- +ssh_modules_with_trace() -> + {ok,AllSshModules} = application:get_key(ssh, modules), + [M || M <- AllSshModules, + lists:member({dbg_trace,3}, M:module_info(exports))]. + %%%---------------------------------------------------------------- --define(wr_record(N,BlackList), wr_record(R=#N{}) -> wr_record(R, record_info(fields,N), BlackList)). - --define(wr_record(N), ?wr_record(N, [])). - - -?wr_record(alg); - -?wr_record(ssh_msg_disconnect); -?wr_record(ssh_msg_ignore); -?wr_record(ssh_msg_unimplemented); -?wr_record(ssh_msg_debug); -?wr_record(ssh_msg_service_request); -?wr_record(ssh_msg_service_accept); -?wr_record(ssh_msg_kexinit); -?wr_record(ssh_msg_kexdh_init); -?wr_record(ssh_msg_kexdh_reply); -?wr_record(ssh_msg_newkeys); -?wr_record(ssh_msg_ext_info); -?wr_record(ssh_msg_kex_dh_gex_request); -?wr_record(ssh_msg_kex_dh_gex_request_old); -?wr_record(ssh_msg_kex_dh_gex_group); -?wr_record(ssh_msg_kex_dh_gex_init); -?wr_record(ssh_msg_kex_dh_gex_reply); -?wr_record(ssh_msg_kex_ecdh_init); -?wr_record(ssh_msg_kex_ecdh_reply); - -?wr_record(ssh_msg_userauth_request); -?wr_record(ssh_msg_userauth_failure); -?wr_record(ssh_msg_userauth_success); -?wr_record(ssh_msg_userauth_banner); -?wr_record(ssh_msg_userauth_passwd_changereq); -?wr_record(ssh_msg_userauth_pk_ok); -?wr_record(ssh_msg_userauth_info_request); -?wr_record(ssh_msg_userauth_info_response); - -?wr_record(ssh_msg_global_request); -?wr_record(ssh_msg_request_success); -?wr_record(ssh_msg_request_failure); -?wr_record(ssh_msg_channel_open); -?wr_record(ssh_msg_channel_open_confirmation); -?wr_record(ssh_msg_channel_open_failure); -?wr_record(ssh_msg_channel_window_adjust); -?wr_record(ssh_msg_channel_data); -?wr_record(ssh_msg_channel_extended_data); -?wr_record(ssh_msg_channel_eof); -?wr_record(ssh_msg_channel_close); -?wr_record(ssh_msg_channel_request); -?wr_record(ssh_msg_channel_success); -?wr_record(ssh_msg_channel_failure); - -wr_record(R) -> io_lib:format('~p~n',[R]). +get_all_trace_flags() -> + get_all_trace_flags(ssh_modules_with_trace()). +get_all_trace_flags(Modules) -> + lists:usort( + lists:flatten( + lists:foldl( + fun(Type, Acc) -> + call_modules(flags, Type, undefined, Acc, Modules) + end, [timestamp], ?ALL_DBG_TYPES))). +%%%---------------------------------------------------------------- +get_all_dbg_types() -> + lists:usort( + lists:flatten( + call_modules(points, undefined) )). + +%%%---------------------------------------------------------------- +call_modules(Cmnd, Type) -> + call_modules(Cmnd, Type, undefined). + +call_modules(Cmnd, Type, Arg) -> + call_modules(Cmnd, Type, Arg, []). + +call_modules(Cmnd, Type, Arg, Acc0) -> + call_modules(Cmnd, Type, Arg, Acc0, ssh_modules_with_trace()). + +call_modules(Cmnd, Types, Arg, Acc0, Modules) when is_list(Types) -> + lists:foldl( + fun(Type, Acc) -> + call_modules(Cmnd, Type, Arg, Acc, Modules) + end, Acc0, Types); + +call_modules(Cmnd, Type, Arg, Acc0, Modules) -> + lists:foldl( + fun(Mod, Acc) -> + try Mod:dbg_trace(Cmnd, Type, Arg) + of + Result -> [Result|Acc] + catch + _:_ -> Acc + end + end, Acc0, Modules). + +%%%---------------------------------------------------------------- +switch(X, Type) when is_atom(Type) -> + switch(X, [Type]); + +switch(X, Types) when is_list(Types) -> + case whereis(?SERVER) of + undefined -> + start(); + _ -> + ok + end, + case lists:usort(Types) -- ?ALL_DBG_TYPES of + [] -> + gen_server:call(?SERVER, {switch,X,Types}); + L -> + {error, {unknown, L}} + end. + +%%%---------------------------------------------------------------- +%%% Format of trace messages are described in reference manual for erlang:trace/4 +%%% {call,MFA} +%%% {return_from,{M,F,N},Result} +%%% {send,Msg,To} +%%% {'receive',Msg} + +trace_pid({trace,Pid,_}) -> Pid; +trace_pid({trace,Pid,_,_}) -> Pid; +trace_pid({trace,Pid,_,_,_}) -> Pid; +trace_pid({trace,Pid,_,_,_,_}) -> Pid; +trace_pid({trace,Pid,_,_,_,_,_}) -> Pid; +trace_pid({trace_ts,Pid,_,_TS}) -> Pid; +trace_pid({trace_ts,Pid,_,_,_TS}) -> Pid; +trace_pid({trace_ts,Pid,_,_,_,_TS}) -> Pid; +trace_pid({trace_ts,Pid,_,_,_,_,_TS}) -> Pid; +trace_pid({trace_ts,Pid,_,_,_,_,_,_TS}) -> Pid. + +trace_ts({trace_ts,_Pid,_,TS}) -> ts(TS); +trace_ts({trace_ts,_Pid,_,_,TS}) -> ts(TS); +trace_ts({trace_ts,_Pid,_,_,_,TS}) -> ts(TS); +trace_ts({trace_ts,_Pid,_,_,_,_,TS}) -> ts(TS); +trace_ts({trace_ts,_Pid,_,_,_,_,_,TS}) -> ts(TS); +trace_ts(_) -> "-". + +trace_info({trace,_Pid,A}) -> A; +trace_info({trace,_Pid,A,B}) -> {A,B}; +trace_info({trace,_Pid,A,B,C}) -> {A,B,C}; +trace_info({trace,_Pid,A,B,C,D}) -> {A,B,C,D}; +trace_info({trace,_Pid,A,B,C,D,E}) -> {A,B,C,D,E}; +trace_info({trace_ts,_Pid,A,_TS}) -> A; +trace_info({trace_ts,_Pid,A,B,_TS}) -> {A,B}; +trace_info({trace_ts,_Pid,A,B,C,_TS}) -> {A,B,C}; +trace_info({trace_ts,_Pid,A,B,C,D,_TS}) -> {A,B,C,D}; +trace_info({trace_ts,_Pid,A,B,C,D,E,_TS}) -> {A,B,C,D,E}. + + +try_all_types_in_all_modules(TypesOn, Arg, WriteFun, Acc0) -> + SshModules = ssh_modules_with_trace(), + TS = trace_ts(Arg), + PID = trace_pid(Arg), + INFO = trace_info(Arg), + lists:foldl( + fun(Type, Acc1) -> + lists:foldl( + fun(SshMod,Acc) -> + try WriteFun("~n~s ~p ~s~n", + [lists:flatten(TS), PID, lists:flatten(SshMod:dbg_trace(format,Type,INFO))], + Acc) + catch + _:_ -> Acc + end + end, Acc1, SshModules) + end, Acc0, TypesOn). + +%%%---------------------------------------------------------------- wr_record(T, Fs, BL) when is_tuple(T) -> wr_record(tuple_to_list(T), Fs, BL); -wr_record([Name|Values], Fields, BlackL) -> +wr_record([_Name|Values], Fields, BlackL) -> W = case Fields of [] -> 0; _ -> lists:max([length(atom_to_list(F)) || F<-Fields]) end, - [io_lib:format("~p:~n",[string:to_upper(atom_to_list(Name))]) - | [io_lib:format(" ~*p: ~p~n",[W,Tag,Value]) || {Tag,Value} <- lists:zip(Fields,Values), - not lists:member(Tag,BlackL) - ] + [io_lib:format(" ~*p: ~p~n",[W,Tag,Value]) || {Tag,Value} <- lists:zip(Fields,Values), + not lists:member(Tag,BlackL) ]. + +%%%---------------------------------------------------------------- +ts({_,_,Usec}=Now) when is_integer(Usec) -> + {_Date,{HH,MM,SS}} = calendar:now_to_local_time(Now), + io_lib:format("~.2.0w:~.2.0w:~.2.0w.~.6.0w",[HH,MM,SS,Usec]); +ts(_) -> + "-". diff --git a/lib/ssh/src/ssh_file.erl b/lib/ssh/src/ssh_file.erl index 33792da38f..9cab2fe0bd 100644 --- a/lib/ssh/src/ssh_file.erl +++ b/lib/ssh/src/ssh_file.erl @@ -45,27 +45,6 @@ %%% API -%%% client --spec add_host_key(string(), - public_key:public_key(), - proplists:proplist()) -> ok | {error,term()}. - --spec is_host_key(public_key:public_key(), - string(), - ssh_client_key_api:algorithm(), - proplists:proplist()) -> boolean(). - --spec user_key(ssh_client_key_api:algorithm(), - proplists:proplist()) -> {ok, public_key:private_key()} | {error,term()}. - -%%% server --spec host_key(ssh_server_key_api:algorithm(), - proplists:proplist()) -> {ok, public_key:private_key()} | {error,term()}. - --spec is_auth_key(public_key:public_key(), - string(), proplists:proplist()) -> boolean(). - - %% Used by server host_key(Algorithm, Opts) -> File = file_name(system, file_base_name(Algorithm), Opts), diff --git a/lib/ssh/src/ssh_info.erl b/lib/ssh/src/ssh_info.erl index d464def6fa..ee244f1432 100644 --- a/lib/ssh/src/ssh_info.erl +++ b/lib/ssh/src/ssh_info.erl @@ -140,15 +140,15 @@ print_system_sup({{ssh_acceptor_sup,_LocalHost,_LocalPort,_Profile}, Pid, superv -print_channels({{server,ssh_channel_sup,_,_},Pid,supervisor,[ssh_channel_sup]}) when is_pid(Pid) -> +print_channels({{server,ssh_server_channel_sup,_,_},Pid,supervisor,[ssh_server_channel_sup]}) when is_pid(Pid) -> Children = supervisor:which_children(Pid), - ChannelPids = [P || {R,P,worker,[ssh_channel]} <- Children, + ChannelPids = [P || {R,P,worker,[ssh_server_channel]} <- Children, is_pid(P), is_reference(R)], case ChannelPids of [] -> io_lib:format(?INDENT?INDENT"No channels~n",[]); [Ch1Pid|_] -> - {{ConnManager,_}, _Str} = ssh_channel:get_print_info(Ch1Pid), + {{ConnManager,_}, _Str} = ssh_server_channel:get_print_info(Ch1Pid), {{_,Remote},_} = ssh_connection_handler:get_print_info(ConnManager), [io_lib:format(?INDENT?INDENT"Remote: ~s ConnectionRef = ~p~n",[fmt_host_port(Remote),ConnManager]), lists:map(fun print_ch/1, ChannelPids) @@ -159,7 +159,7 @@ print_channels({{server,ssh_connection_sup,_,_},Pid,supervisor,[ssh_connection_s print_ch(Pid) -> try - {{ConnManager,ChannelID}, Str} = ssh_channel:get_print_info(Pid), + {{ConnManager,ChannelID}, Str} = ssh_server_channel:get_print_info(Pid), {_LocalRemote,StrM} = ssh_connection_handler:get_print_info(ConnManager), io_lib:format(?INDENT?INDENT?INDENT"ch ~p ~p: ~s ~s~n",[ChannelID, Pid, StrM, Str]) catch diff --git a/lib/ssh/src/ssh_message.erl b/lib/ssh/src/ssh_message.erl index eb06f05a4a..a2251eab97 100644 --- a/lib/ssh/src/ssh_message.erl +++ b/lib/ssh/src/ssh_message.erl @@ -32,6 +32,8 @@ -export([encode/1, decode/1, decode_keyboard_interactive_prompts/2]). +-export([dbg_trace/3]). + -define('2bin'(X), (if is_binary(X) -> X; is_list(X) -> list_to_binary(X); X==undefined -> <<>> @@ -611,3 +613,86 @@ encode_signature({#'ECPoint'{}, {namedCurve,OID}}, _SigAlg, Signature) -> CurveName = public_key:oid2ssh_curvename(OID), <<?Ebinary(<<"ecdsa-sha2-",CurveName/binary>>), ?Ebinary(Signature)>>. +%%%################################################################ +%%%# +%%%# Tracing +%%%# + +dbg_trace(points, _, _) -> [ssh_messages, raw_messages]; + +dbg_trace(flags, ssh_messages, _) -> [c]; +dbg_trace(on, ssh_messages, _) -> dbg:tp(?MODULE,encode,1,x), + dbg:tp(?MODULE,decode,1,x); +dbg_trace(off, ssh_messages, _) -> dbg:ctpg(?MODULE,encode,1), + dbg:ctpg(?MODULE,decode,1); + +dbg_trace(flags, raw_messages, A) -> dbg_trace(flags, ssh_messages, A); +dbg_trace(on, raw_messages, A) -> dbg_trace(on, ssh_messages, A); +dbg_trace(off, raw_messages, A) -> dbg_trace(off, ssh_messages, A); + +dbg_trace(format, ssh_messages, {call,{?MODULE,encode,[Msg]}}) -> + Name = string:to_upper(atom_to_list(element(1,Msg))), + ["Going to send ",Name,":\n", + wr_record(ssh_dbg:shrink_bin(Msg)) + ]; +dbg_trace(format, ssh_messages, {return_from,{?MODULE,decode,1},Msg}) -> + Name = string:to_upper(atom_to_list(element(1,Msg))), + ["Received ",Name,":\n", + wr_record(ssh_dbg:shrink_bin(Msg)) + ]; + +dbg_trace(format, raw_messages, {call,{?MODULE,decode,[BytesPT]}}) -> + ["Received plain text bytes (shown after decryption):\n", + io_lib:format("~p",[BytesPT]) + ]; +dbg_trace(format, raw_messages, {return_from,{?MODULE,encode,1},BytesPT}) -> + ["Going to send plain text bytes (shown before encryption):\n", + io_lib:format("~p",[BytesPT]) + ]. + + +?wr_record(ssh_msg_disconnect); +?wr_record(ssh_msg_ignore); +?wr_record(ssh_msg_unimplemented); +?wr_record(ssh_msg_debug); +?wr_record(ssh_msg_service_request); +?wr_record(ssh_msg_service_accept); +?wr_record(ssh_msg_kexinit); +?wr_record(ssh_msg_kexdh_init); +?wr_record(ssh_msg_kexdh_reply); +?wr_record(ssh_msg_newkeys); +?wr_record(ssh_msg_ext_info); +?wr_record(ssh_msg_kex_dh_gex_request); +?wr_record(ssh_msg_kex_dh_gex_request_old); +?wr_record(ssh_msg_kex_dh_gex_group); +?wr_record(ssh_msg_kex_dh_gex_init); +?wr_record(ssh_msg_kex_dh_gex_reply); +?wr_record(ssh_msg_kex_ecdh_init); +?wr_record(ssh_msg_kex_ecdh_reply); + +?wr_record(ssh_msg_userauth_request); +?wr_record(ssh_msg_userauth_failure); +?wr_record(ssh_msg_userauth_success); +?wr_record(ssh_msg_userauth_banner); +?wr_record(ssh_msg_userauth_passwd_changereq); +?wr_record(ssh_msg_userauth_pk_ok); +?wr_record(ssh_msg_userauth_info_request); +?wr_record(ssh_msg_userauth_info_response); + +?wr_record(ssh_msg_global_request); +?wr_record(ssh_msg_request_success); +?wr_record(ssh_msg_request_failure); +?wr_record(ssh_msg_channel_open); +?wr_record(ssh_msg_channel_open_confirmation); +?wr_record(ssh_msg_channel_open_failure); +?wr_record(ssh_msg_channel_window_adjust); +?wr_record(ssh_msg_channel_data); +?wr_record(ssh_msg_channel_extended_data); +?wr_record(ssh_msg_channel_eof); +?wr_record(ssh_msg_channel_close); +?wr_record(ssh_msg_channel_request); +?wr_record(ssh_msg_channel_success); +?wr_record(ssh_msg_channel_failure); + +wr_record(R) -> io_lib:format('~p~n',[R]). + diff --git a/lib/ssh/src/ssh_no_io.erl b/lib/ssh/src/ssh_no_io.erl index 1da257ed99..25be0023e9 100644 --- a/lib/ssh/src/ssh_no_io.erl +++ b/lib/ssh/src/ssh_no_io.erl @@ -31,35 +31,24 @@ -spec yes_no(any(), any()) -> no_return(). yes_no(_, _) -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{code = ?SSH_DISCONNECT_SERVICE_NOT_AVAILABLE, - description = "User interaction is not allowed"}, - {no_io_allowed, yes_no}). + ?DISCONNECT(?SSH_DISCONNECT_SERVICE_NOT_AVAILABLE, + "User interaction is not allowed"). -spec read_password(any(), any()) -> no_return(). read_password(_, _) -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{code = ?SSH_DISCONNECT_SERVICE_NOT_AVAILABLE, - description = "User interaction is not allowed"}, - {no_io_allowed, read_password}). - + ?DISCONNECT(?SSH_DISCONNECT_SERVICE_NOT_AVAILABLE, + "User interaction is not allowed"). -spec read_line(any(), any()) -> no_return(). read_line(_, _) -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{code = ?SSH_DISCONNECT_SERVICE_NOT_AVAILABLE, - description = "User interaction is not allowed"}, - {no_io_allowed, read_line}). - + ?DISCONNECT(?SSH_DISCONNECT_SERVICE_NOT_AVAILABLE, + "User interaction is not allowed"). -spec format(any(), any()) -> no_return(). format(_, _) -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{code = ?SSH_DISCONNECT_SERVICE_NOT_AVAILABLE, - description = "User interaction is not allowed"}, - {no_io_allowed, format}). - + ?DISCONNECT(?SSH_DISCONNECT_SERVICE_NOT_AVAILABLE, + "User interaction is not allowed"). diff --git a/lib/ssh/src/ssh_options.erl b/lib/ssh/src/ssh_options.erl index 1e10f72956..4dd9082250 100644 --- a/lib/ssh/src/ssh_options.erl +++ b/lib/ssh/src/ssh_options.erl @@ -32,7 +32,7 @@ handle_options/2 ]). --export_type([options/0 +-export_type([private_options/0 ]). %%%================================================================ @@ -47,16 +47,23 @@ default => any() }. +-type option_key() :: atom(). + -type option_declarations() :: #{ {option_key(),def} := option_declaration() }. -type error() :: {error,{eoptions,any()}} . +-type private_options() :: #{socket_options := socket_options(), + internal_options := internal_options(), + option_key() => any() + }. + %%%================================================================ %%% %%% Get an option %%% --spec get_value(option_class(), option_key(), options(), +-spec get_value(option_class(), option_key(), private_options(), atom(), non_neg_integer()) -> any() | no_return(). get_value(Class, Key, Opts, _CallerMod, _CallerLine) when is_map(Opts) -> @@ -69,7 +76,7 @@ get_value(Class, Key, Opts, _CallerMod, _CallerLine) -> error({bad_options,Class, Key, Opts, _CallerMod, _CallerLine}). --spec get_value(option_class(), option_key(), options(), fun(() -> any()), +-spec get_value(option_class(), option_key(), private_options(), fun(() -> any()), atom(), non_neg_integer()) -> any() | no_return(). get_value(socket_options, Key, Opts, DefFun, _CallerMod, _CallerLine) when is_map(Opts) -> @@ -91,8 +98,8 @@ get_value(Class, Key, Opts, _DefFun, _CallerMod, _CallerLine) -> %%% Put an option %%% --spec put_value(option_class(), option_in(), options(), - atom(), non_neg_integer()) -> options(). +-spec put_value(option_class(), option_in(), private_options(), + atom(), non_neg_integer()) -> private_options(). put_value(user_options, KeyVal, Opts, _CallerMod, _CallerLine) when is_map(Opts) -> put_user_value(KeyVal, Opts); @@ -131,8 +138,8 @@ put_socket_value(A, SockOpts) when is_atom(A) -> %%% Delete an option %%% --spec delete_key(option_class(), option_key(), options(), - atom(), non_neg_integer()) -> options(). +-spec delete_key(option_class(), option_key(), private_options(), + atom(), non_neg_integer()) -> private_options(). delete_key(internal_options, Key, Opts, _CallerMod, _CallerLine) when is_map(Opts) -> InternalOpts = maps:get(internal_options,Opts), @@ -144,9 +151,7 @@ delete_key(internal_options, Key, Opts, _CallerMod, _CallerLine) when is_map(Opt %%% Initialize the options %%% --spec handle_options(role(), proplists:proplist()) -> options() | error() . - --spec handle_options(role(), proplists:proplist(), options()) -> options() | error() . +-spec handle_options(role(), client_options()|daemon_options()) -> private_options() | error() . handle_options(Role, PropList0) -> handle_options(Role, PropList0, #{socket_options => [], @@ -155,7 +160,7 @@ handle_options(Role, PropList0) -> }). handle_options(Role, PropList0, Opts0) when is_map(Opts0), - is_list(PropList0) -> + is_list(PropList0) -> PropList1 = proplists:unfold(PropList0), try OptionDefinitions = default(Role), @@ -275,10 +280,12 @@ default(server) -> class => user_options }, - {exec, def} => % FIXME: need some archeology.... + {exec, def} => #{default => undefined, - chk => fun({M,F,_}) -> is_atom(M) andalso is_atom(F); - (V) -> is_function(V) + chk => fun({direct, V}) -> check_function1(V) orelse check_function2(V) orelse check_function3(V); + %% Compatibility (undocumented): + ({M,F,A}) -> is_atom(M) andalso is_atom(F) andalso is_list(A); + (V) -> check_function1(V) orelse check_function2(V) orelse check_function3(V) end, class => user_options }, diff --git a/lib/ssh/src/ssh_server_channel.erl b/lib/ssh/src/ssh_server_channel.erl new file mode 100644 index 0000000000..f1c9a85639 --- /dev/null +++ b/lib/ssh/src/ssh_server_channel.erl @@ -0,0 +1,55 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2013-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +%% Description: a gen_server implementing a simple +%% terminal (using the group module) for a CLI +%% over SSH + +-module(ssh_server_channel). + +%% API to server side channel that can be pluged into the erlang ssh daemeon +-callback init(Args :: term()) -> + {ok, State :: term()} | {ok, State :: term(), timeout() | hibernate} | + {stop, Reason :: term()} | ignore. + +-callback terminate(Reason :: (normal | shutdown | {shutdown, term()} | + term()), + State :: term()) -> + term(). + +-callback handle_msg(Msg ::term(), State :: term()) -> + {ok, State::term()} | {stop, ChannelId::ssh:channel_id(), State::term()}. +-callback handle_ssh_msg({ssh_cm, ConnectionRef::ssh:connection_ref(), SshMsg::term()}, + State::term()) -> {ok, State::term()} | + {stop, ChannelId::ssh:channel_id(), + State::term()}. + +%%% Internal API +-export([start_link/5, + get_print_info/1 + ]). + +start_link(ConnectionManager, ChannelId, CallBack, CbInitArgs, Exec) -> + ssh_client_channel:start_link(ConnectionManager, ChannelId, CallBack, CbInitArgs, Exec). + + +get_print_info(Pid) -> + ssh_client_channel:get_print_info(Pid). diff --git a/lib/ssh/src/ssh_channel_sup.erl b/lib/ssh/src/ssh_server_channel_sup.erl index 8444533fd1..70799db714 100644 --- a/lib/ssh/src/ssh_channel_sup.erl +++ b/lib/ssh/src/ssh_server_channel_sup.erl @@ -22,7 +22,7 @@ %%---------------------------------------------------------------------- %% Purpose: Ssh channel supervisor. %%---------------------------------------------------------------------- --module(ssh_channel_sup). +-module(ssh_server_channel_sup). -behaviour(supervisor). @@ -40,18 +40,16 @@ start_link(Args) -> start_child(Sup, Callback, Id, Args, Exec) -> ChildSpec = #{id => make_ref(), - start => {ssh_channel, start_link, [self(), Id, Callback, Args, Exec]}, + start => {ssh_server_channel, start_link, [self(), Id, Callback, Args, Exec]}, restart => temporary, type => worker, - modules => [ssh_channel] + modules => [ssh_server_channel] }, supervisor:start_child(Sup, ChildSpec). %%%========================================================================= %%% Supervisor callback %%%========================================================================= --spec init( [term()] ) -> {ok,{supervisor:sup_flags(),[supervisor:child_spec()]}} | ignore . - init(_Args) -> RestartStrategy = one_for_one, MaxR = 10, diff --git a/lib/ssh/src/ssh_server_key_api.erl b/lib/ssh/src/ssh_server_key_api.erl index 3f1b886fa7..a285bf9475 100644 --- a/lib/ssh/src/ssh_server_key_api.erl +++ b/lib/ssh/src/ssh_server_key_api.erl @@ -23,16 +23,18 @@ -include_lib("public_key/include/public_key.hrl"). -include("ssh.hrl"). --export_type([algorithm/0]). +-export_type([daemon_key_cb_options/0]). --type algorithm() :: ssh_client_key_api:algorithm(). +-type daemon_key_cb_options() :: [{key_cb_private,term()} | ssh:daemon_option()]. --callback host_key(Algorithm :: algorithm(), - DaemonOptions :: proplists:proplist()) -> +-callback host_key(Algorithm :: ssh:pubkey_alg(), + DaemonOptions :: daemon_key_cb_options() + ) -> {ok, PrivateKey :: public_key:private_key()} | {error, term()}. -callback is_auth_key(PublicKey :: public_key:public_key(), User :: string(), - DaemonOptions :: proplists:proplist()) -> + DaemonOptions :: daemon_key_cb_options() + ) -> boolean(). diff --git a/lib/ssh/src/ssh_sftp.erl b/lib/ssh/src/ssh_sftp.erl index 9e1229dc85..5984713ec9 100644 --- a/lib/ssh/src/ssh_sftp.erl +++ b/lib/ssh/src/ssh_sftp.erl @@ -24,7 +24,7 @@ -module(ssh_sftp). --behaviour(ssh_channel). +-behaviour(ssh_client_channel). -include_lib("kernel/include/file.hrl"). -include("ssh.hrl"). @@ -47,11 +47,13 @@ recv_window/1, list_dir/2, read_file/2, write_file/3, recv_window/2, list_dir/3, read_file/3, write_file/4]). -%% ssh_channel callbacks +%% ssh_client_channel callbacks -export([init/1, handle_call/3, handle_cast/2, code_change/3, handle_msg/2, handle_ssh_msg/2, terminate/2]). %% TODO: Should be placed elsewhere ssh_sftpd should not call functions in ssh_sftp! -export([info_to_attr/1, attr_to_info/1]). +-export([dbg_trace/3]). + -record(state, { xf, @@ -121,7 +123,7 @@ start_channel(Cm, UserOptions) when is_pid(Cm) -> {_SshOpts, ChanOpts, SftpOpts} = handle_options(UserOptions), case ssh_xfer:attach(Cm, [], ChanOpts) of {ok, ChannelId, Cm} -> - case ssh_channel:start(Cm, ChannelId, + case ssh_client_channel:start(Cm, ChannelId, ?MODULE, [Cm, ChannelId, SftpOpts]) of {ok, Pid} -> case wait_for_version_negotiation(Pid, Timeout) of @@ -149,7 +151,7 @@ start_channel(Host, Port, UserOptions) -> proplists:get_value(timeout, SftpOpts, infinity)), case ssh_xfer:connect(Host, Port, SshOpts, ChanOpts, Timeout) of {ok, ChannelId, Cm} -> - case ssh_channel:start(Cm, ChannelId, ?MODULE, [Cm,ChannelId,SftpOpts]) of + case ssh_client_channel:start(Cm, ChannelId, ?MODULE, [Cm,ChannelId,SftpOpts]) of {ok, Pid} -> case wait_for_version_negotiation(Pid, Timeout) of ok -> @@ -823,7 +825,7 @@ handle_msg({ssh_channel_up, _, _}, #state{opts = Options, xf = Xf} = State) -> %% Version negotiation timed out handle_msg({timeout, undefined, From}, #state{xf = #ssh_xfer{channel = ChannelId}} = State) -> - ssh_channel:reply(From, {error, timeout}), + ssh_client_channel:reply(From, {error, timeout}), {stop, ChannelId, State}; handle_msg({timeout, Id, From}, #state{req_list = ReqList0} = State) -> @@ -832,7 +834,7 @@ handle_msg({timeout, Id, From}, #state{req_list = ReqList0} = State) -> {ok, State}; _ -> ReqList = lists:keydelete(Id, 1, ReqList0), - ssh_channel:reply(From, {error, timeout}), + ssh_client_channel:reply(From, {error, timeout}), {ok, State#state{req_list = ReqList}} end; @@ -880,7 +882,7 @@ handle_options([Opt|Rest], Sftp, Chan, Ssh) -> handle_options(Rest, Sftp, Chan, [Opt|Ssh]). call(Pid, Msg, TimeOut) -> - ssh_channel:call(Pid, {{timeout, TimeOut}, Msg}, infinity). + ssh_client_channel:call(Pid, {{timeout, TimeOut}, Msg}, infinity). handle_reply(State, <<?UINT32(Len),Reply:Len/binary,Rest/binary>>) -> do_handle_reply(State, Reply, Rest); @@ -899,7 +901,7 @@ do_handle_reply(#state{xf = Xf} = State, true -> ok end, - ssh_channel:reply(From, ok) + ssh_client_channel:reply(From, ok) end, State#state{xf = Xf#ssh_xfer{vsn = Version, ext = Ext}, rep_buf = Rest}; @@ -947,7 +949,7 @@ async_reply(ReqID, Reply, _From={To,_}, State) -> State. sync_reply(Reply, From, State) -> - catch (ssh_channel:reply(From, Reply)), + catch (ssh_client_channel:reply(From, Reply)), State. open2(OrigReqID,FileName,Handle,Mode,Async,From,State) -> @@ -1460,3 +1462,21 @@ format_channel_start_error({shutdown, Reason}) -> Reason; format_channel_start_error(Reason) -> Reason. + +%%%################################################################ +%%%# +%%%# Tracing +%%%# + +dbg_trace(points, _, _) -> [terminate]; + +dbg_trace(flags, terminate, _) -> [c]; +dbg_trace(on, terminate, _) -> dbg:tp(?MODULE, terminate, 2, x); +dbg_trace(off, terminate, _) -> dbg:ctpg(?MODULE, terminate, 2); +dbg_trace(format, terminate, {call, {?MODULE,terminate, [Reason, State]}}) -> + ["Sftp Terminating:\n", + io_lib:format("Reason: ~p,~nState:~n~s", [Reason, wr_record(State)]) + ]. + +?wr_record(state). + diff --git a/lib/ssh/src/ssh_sftpd.erl b/lib/ssh/src/ssh_sftpd.erl index 427edf01ab..cb2eab1fec 100644 --- a/lib/ssh/src/ssh_sftpd.erl +++ b/lib/ssh/src/ssh_sftpd.erl @@ -24,7 +24,7 @@ -module(ssh_sftpd). --behaviour(ssh_daemon_channel). +-behaviour(ssh_server_channel). -include_lib("kernel/include/file.hrl"). @@ -38,6 +38,8 @@ -export([init/1, handle_ssh_msg/2, handle_msg/2, terminate/2]). +-export([dbg_trace/3]). + -record(state, { xf, % [{channel,ssh_xfer states}...] cwd, % current dir (on first connect) @@ -56,21 +58,7 @@ %%==================================================================== %% API %%==================================================================== --spec init(Args :: term()) -> - {ok, State :: term()} | {ok, State :: term(), timeout() | hibernate} | - {stop, Reason :: term()} | ignore. - --spec terminate(Reason :: (normal | shutdown | {shutdown, term()} | - term()), - State :: term()) -> - term(). - --spec handle_msg(Msg ::term(), State :: term()) -> - {ok, State::term()} | {stop, ChannelId::integer(), State::term()}. --spec handle_ssh_msg({ssh_cm, ConnectionRef::term(), SshMsg::term()}, - State::term()) -> {ok, State::term()} | - {stop, ChannelId::integer(), - State::term()}. +-spec subsystem_spec(list()) -> subsystem_spec(). subsystem_spec(Options) -> {"sftp", {?MODULE, Options}}. @@ -360,10 +348,12 @@ handle_op(?SSH_FXP_REMOVE, ReqId, <<?UINT32(PLen), BPath:PLen/binary>>, case IsDir of %% This version 6 we still have ver 5 true when Vsn > 5 -> ssh_xfer:xf_send_status(State0#state.xf, ReqId, - ?SSH_FX_FILE_IS_A_DIRECTORY, "File is a directory"); + ?SSH_FX_FILE_IS_A_DIRECTORY, "File is a directory"), + State0; true -> ssh_xfer:xf_send_status(State0#state.xf, ReqId, - ?SSH_FX_FAILURE, "File is a directory"); + ?SSH_FX_FAILURE, "File is a directory"), + State0; false -> {Status, FS1} = FileMod:delete(Path, FS0), State1 = State0#state{file_state = FS1}, @@ -947,3 +937,20 @@ maybe_increase_recv_window(ConnectionManager, ChannelId, Options) -> Increment =< 0 -> do_nothing end. + +%%%################################################################ +%%%# +%%%# Tracing +%%%# + +dbg_trace(points, _, _) -> [terminate]; + +dbg_trace(flags, terminate, _) -> [c]; +dbg_trace(on, terminate, _) -> dbg:tp(?MODULE, terminate, 2, x); +dbg_trace(off, terminate, _) -> dbg:ctpg(?MODULE, terminate, 2); +dbg_trace(format, terminate, {call, {?MODULE,terminate, [Reason, State]}}) -> + ["SftpD Terminating:\n", + io_lib:format("Reason: ~p,~nState:~n~s", [Reason, wr_record(State)]) + ]. + +?wr_record(state). diff --git a/lib/ssh/src/ssh_shell.erl b/lib/ssh/src/ssh_shell.erl index 17224b6ef4..084daa6821 100644 --- a/lib/ssh/src/ssh_shell.erl +++ b/lib/ssh/src/ssh_shell.erl @@ -22,18 +22,21 @@ -module(ssh_shell). +-include("ssh.hrl"). -include("ssh_connect.hrl"). %%% As this is an user interactive client it behaves like a daemon %%% channel inspite of it being a client. --behaviour(ssh_daemon_channel). +-behaviour(ssh_server_channel). -%% ssh_channel callbacks +%% ssh_server_channel callbacks -export([init/1, handle_msg/2, handle_ssh_msg/2, terminate/2]). %% Spawn export -export([input_loop/2]). +-export([dbg_trace/3]). + -record(state, { io, %% Io process @@ -43,23 +46,8 @@ ). %%==================================================================== -%% ssh_channel callbacks +%% ssh_server_channel callbacks %%==================================================================== --spec init(Args :: term()) -> - {ok, State :: term()} | {ok, State :: term(), timeout() | hibernate} | - {stop, Reason :: term()} | ignore. - --spec terminate(Reason :: (normal | shutdown | {shutdown, term()} | - term()), - State :: term()) -> - term(). - --spec handle_msg(Msg ::term(), State :: term()) -> - {ok, State::term()} | {stop, ChannelId::integer(), State::term()}. --spec handle_ssh_msg({ssh_cm, ConnectionRef::term(), SshMsg::term()}, - State::term()) -> {ok, State::term()} | - {stop, ChannelId::integer(), - State::term()}. %%-------------------------------------------------------------------- %% Function: init(Args) -> {ok, State} @@ -194,3 +182,20 @@ get_ancestors() -> A when is_list(A) -> A; _ -> [] end. + +%%%################################################################ +%%%# +%%%# Tracing +%%%# + +dbg_trace(points, _, _) -> [terminate]; + +dbg_trace(flags, terminate, _) -> [c]; +dbg_trace(on, terminate, _) -> dbg:tp(?MODULE, terminate, 2, x); +dbg_trace(off, terminate, _) -> dbg:ctpg(?MODULE, terminate, 2); +dbg_trace(format, terminate, {call, {?MODULE,terminate, [Reason, State]}}) -> + ["Shell Terminating:\n", + io_lib:format("Reason: ~p,~nState:~n~s", [Reason, wr_record(State)]) + ]. + +?wr_record(state). diff --git a/lib/ssh/src/ssh_subsystem_sup.erl b/lib/ssh/src/ssh_subsystem_sup.erl index 77da240a66..f80be7f20b 100644 --- a/lib/ssh/src/ssh_subsystem_sup.erl +++ b/lib/ssh/src/ssh_subsystem_sup.erl @@ -48,7 +48,7 @@ connection_supervisor(SupPid) -> channel_supervisor(SupPid) -> Children = supervisor:which_children(SupPid), - ssh_channel_sup(Children). + ssh_server_channel_sup(Children). %%%========================================================================= %%% Supervisor callback @@ -78,8 +78,8 @@ ssh_connection_child_spec(Role, Address, Port, _Profile, Options) -> }. ssh_channel_child_spec(Role, Address, Port, _Profile, Options) -> - #{id => id(Role, ssh_channel_sup, Address, Port), - start => {ssh_channel_sup, start_link, [Options]}, + #{id => id(Role, ssh_server_channel_sup, Address, Port), + start => {ssh_server_channel_sup, start_link, [Options]}, restart => temporary, type => supervisor }. @@ -92,10 +92,10 @@ ssh_connection_sup([{_, Child, _, [ssh_connection_sup]} | _]) -> ssh_connection_sup([_ | Rest]) -> ssh_connection_sup(Rest). -ssh_channel_sup([{_, Child, _, [ssh_channel_sup]} | _]) -> +ssh_server_channel_sup([{_, Child, _, [ssh_server_channel_sup]} | _]) -> Child; -ssh_channel_sup([_ | Rest]) -> - ssh_channel_sup(Rest). +ssh_server_channel_sup([_ | Rest]) -> + ssh_server_channel_sup(Rest). diff --git a/lib/ssh/src/ssh_system_sup.erl b/lib/ssh/src/ssh_system_sup.erl index 17f990c5d8..469f9560e9 100644 --- a/lib/ssh/src/ssh_system_sup.erl +++ b/lib/ssh/src/ssh_system_sup.erl @@ -88,11 +88,11 @@ stop_listener(Address, Port, Profile) -> stop_system(SysSup) -> - spawn(fun() -> sshd_sup:stop_child(SysSup) end), + catch sshd_sup:stop_child(SysSup), ok. stop_system(Address, Port, Profile) -> - spawn(fun() -> sshd_sup:stop_child(Address, Port, Profile) end), + catch sshd_sup:stop_child(Address, Port, Profile), ok. diff --git a/lib/ssh/src/ssh_transport.erl b/lib/ssh/src/ssh_transport.erl index 975053d301..f5bba9f824 100644 --- a/lib/ssh/src/ssh_transport.erl +++ b/lib/ssh/src/ssh_transport.erl @@ -53,6 +53,8 @@ valid_key_sha_alg/2, sha/1, sign/3, verify/5]). +-export([dbg_trace/3]). + %%% For test suites -export([pack/3, adjust_algs_for_peer_version/2]). -export([decompress/2, decrypt_blocks/3, is_valid_mac/3 ]). % FIXME: remove @@ -319,10 +321,11 @@ handle_kexinit_msg(#ssh_msg_kexinit{} = CounterPart, #ssh_msg_kexinit{} = Own, key_exchange_first_msg(Algos#alg.kex, Ssh#ssh{algorithms = Algos}) catch - _:_ -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{code = ?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, - description = "Selection of key exchange algorithm failed"}) + Class:Error -> + ?DISCONNECT(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, + io_lib:format("Kexinit failed in client: ~p:~p", + [Class,Error]) + ) end; handle_kexinit_msg(#ssh_msg_kexinit{} = CounterPart, #ssh_msg_kexinit{} = Own, @@ -335,10 +338,11 @@ handle_kexinit_msg(#ssh_msg_kexinit{} = CounterPart, #ssh_msg_kexinit{} = Own, Algos -> {ok, Ssh#ssh{algorithms = Algos}} catch - _:_ -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{code = ?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, - description = "Selection of key exchange algorithm failed"}) + Class:Error -> + ?DISCONNECT(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, + io_lib:format("Kexinit failed in server: ~p:~p", + [Class,Error]) + ) end. @@ -439,12 +443,10 @@ handle_kexdh_init(#ssh_msg_kexdh_init{e = E}, session_id = sid(Ssh1, H)}}; true -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{ - code = ?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, - description = "Key exchange failed, 'e' out of bounds"}, - {error,bad_e_from_peer} - ) + ?DISCONNECT(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, + io_lib:format("Kexdh init failed, received 'e' out of bounds~n E=~p~n P=~p", + [E,P]) + ) end. handle_kexdh_reply(#ssh_msg_kexdh_reply{public_host_key = PeerPubHostKey, @@ -464,20 +466,16 @@ handle_kexdh_reply(#ssh_msg_kexdh_reply{public_host_key = PeerPubHostKey, exchanged_hash = H, session_id = sid(Ssh, H)})}; Error -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{ - code = ?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, - description = "Key exchange failed"}, - Error) + ?DISCONNECT(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, + io_lib:format("Kexdh init failed. Verify host key: ~p",[Error]) + ) end; true -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{ - code = ?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, - description = "Key exchange failed, 'f' out of bounds"}, - bad_f_from_peer - ) + ?DISCONNECT(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, + io_lib:format("Kexdh init failed, received 'f' out of bounds~n F=~p~n P=~p", + [F,P]) + ) end. @@ -501,11 +499,9 @@ handle_kex_dh_gex_request(#ssh_msg_kex_dh_gex_request{min = Min0, keyex_info = {Min0, Max0, NBits} }}; {error,_} -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{ - code = ?SSH_DISCONNECT_PROTOCOL_ERROR, - description = "No possible diffie-hellman-group-exchange group found" - }) + ?DISCONNECT(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, + io_lib:format("No possible diffie-hellman-group-exchange group found",[]) + ) end; handle_kex_dh_gex_request(#ssh_msg_kex_dh_gex_request_old{n = NBits}, @@ -535,20 +531,14 @@ handle_kex_dh_gex_request(#ssh_msg_kex_dh_gex_request_old{n = NBits}, keyex_info = {-1, -1, NBits} % flag for kex_hash calc }}; {error,_} -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{ - code = ?SSH_DISCONNECT_PROTOCOL_ERROR, - description = "No possible diffie-hellman-group-exchange group found" - }) + ?DISCONNECT(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, + io_lib:format("No possible diffie-hellman-group-exchange group found",[]) + ) end; handle_kex_dh_gex_request(_, _) -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{ - code = ?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, - description = "Key exchange failed, bad values in ssh_msg_kex_dh_gex_request"}, - bad_ssh_msg_kex_dh_gex_request). - + ?DISCONNECT(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, + "Key exchange failed, bad values in ssh_msg_kex_dh_gex_request"). adjust_gex_min_max(Min0, Max0, Opts) -> {Min1, Max1} = ?GET_OPT(dh_gex_limits, Opts), @@ -558,11 +548,8 @@ adjust_gex_min_max(Min0, Max0, Opts) -> Min2 =< Max2 -> {Min2, Max2}; Max2 < Min2 -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{ - code = ?SSH_DISCONNECT_PROTOCOL_ERROR, - description = "No possible diffie-hellman-group-exchange group possible" - }) + ?DISCONNECT(?SSH_DISCONNECT_PROTOCOL_ERROR, + "No possible diffie-hellman-group-exchange group possible") end. @@ -600,18 +587,15 @@ handle_kex_dh_gex_init(#ssh_msg_kex_dh_gex_init{e = E}, session_id = sid(Ssh, H) }}; true -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{ - code = ?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, - description = "Key exchange failed, 'K' out of bounds"}, - bad_K) + ?DISCONNECT(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, + "Kexdh init failed, received 'k' out of bounds" + ) end; true -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{ - code = ?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, - description = "Key exchange failed, 'e' out of bounds"}, - bad_e_from_peer) + ?DISCONNECT(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, + io_lib:format("Kexdh gex init failed, received 'e' out of bounds~n E=~p~n P=~p", + [E,P]) + ) end. handle_kex_dh_gex_reply(#ssh_msg_kex_dh_gex_reply{public_host_key = PeerPubHostKey, @@ -634,28 +618,22 @@ handle_kex_dh_gex_reply(#ssh_msg_kex_dh_gex_reply{public_host_key = PeerPubHostK {ok, SshPacket, install_alg(snd, Ssh#ssh{shared_secret = ssh_bits:mpint(K), exchanged_hash = H, session_id = sid(Ssh, H)})}; - _Error -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{ - code = ?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, - description = "Key exchange failed" - }) + Error -> + ?DISCONNECT(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, + io_lib:format("Kexdh gex reply failed. Verify host key: ~p",[Error]) + ) end; true -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{ - code = ?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, - description = "Key exchange failed, 'K' out of bounds"}, - bad_K) + ?DISCONNECT(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, + "Kexdh gex init failed, 'K' out of bounds" + ) end; true -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{ - code = ?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, - description = "Key exchange failed, 'f' out of bounds"}, - bad_f_from_peer - ) + ?DISCONNECT(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, + io_lib:format("Kexdh gex init failed, received 'f' out of bounds~n F=~p~n P=~p", + [F,P]) + ) end. %%%---------------------------------------------------------------- @@ -686,12 +664,11 @@ handle_kex_ecdh_init(#ssh_msg_kex_ecdh_init{q_c = PeerPublic}, exchanged_hash = H, session_id = sid(Ssh1, H)}} catch - _:_ -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{ - code = ?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, - description = "Peer ECDH public key is invalid"}, - invalid_peer_public_key) + Class:Error -> + ?DISCONNECT(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, + io_lib:format("ECDH compute key failed in server: ~p:~p", + [Class,Error]) + ) end. handle_kex_ecdh_reply(#ssh_msg_kex_ecdh_reply{public_host_key = PeerPubHostKey, @@ -713,19 +690,16 @@ handle_kex_ecdh_reply(#ssh_msg_kex_ecdh_reply{public_host_key = PeerPubHostKey, exchanged_hash = H, session_id = sid(Ssh, H)})}; Error -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{ - code = ?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, - description = "Key exchange failed"}, - Error) + ?DISCONNECT(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, + io_lib:format("ECDH reply failed. Verify host key: ~p",[Error]) + ) end catch - _:_ -> - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{ - code = ?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, - description = "Peer ECDH public key is invalid"}, - invalid_peer_public_key) + Class:Error -> + ?DISCONNECT(?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, + io_lib:format("Peer ECDH public key seem invalid: ~p:~p", + [Class,Error]) + ) end. @@ -735,11 +709,11 @@ handle_new_keys(#ssh_msg_newkeys{}, Ssh0) -> #ssh{} = Ssh -> {ok, Ssh} catch - _C:_Error -> %% TODO: Throw earlier .... - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{code = ?SSH_DISCONNECT_PROTOCOL_ERROR, - description = "Install alg failed" - }) + Class:Error -> %% TODO: Throw earlier ... + ?DISCONNECT(?SSH_DISCONNECT_PROTOCOL_ERROR, + io_lib:format("Install alg failed: ~p:~p", + [Class,Error]) + ) end. @@ -1057,9 +1031,7 @@ install_alg(Dir, SSH) -> alg_setup(snd, SSH) -> ALG = SSH#ssh.algorithms, - SSH#ssh{kex = ALG#alg.kex, - hkey = ALG#alg.hkey, - encrypt = ALG#alg.encrypt, + SSH#ssh{encrypt = ALG#alg.encrypt, send_mac = ALG#alg.send_mac, send_mac_size = mac_digest_size(ALG#alg.send_mac), compress = ALG#alg.compress, @@ -1071,9 +1043,7 @@ alg_setup(snd, SSH) -> alg_setup(rcv, SSH) -> ALG = SSH#ssh.algorithms, - SSH#ssh{kex = ALG#alg.kex, - hkey = ALG#alg.hkey, - decrypt = ALG#alg.decrypt, + SSH#ssh{decrypt = ALG#alg.decrypt, recv_mac = ALG#alg.recv_mac, recv_mac_size = mac_digest_size(ALG#alg.recv_mac), decompress = ALG#alg.decompress, @@ -1115,10 +1085,9 @@ select_all(CL, SL) when length(CL) + length(SL) < ?MAX_NUM_ALGORITHMS -> %% algorithms used by client and server (client pref) lists:map(fun(ALG) -> list_to_atom(ALG) end, (CL -- A)); select_all(CL, SL) -> - Err = lists:concat(["Received too many algorithms (",length(CL),"+",length(SL)," >= ",?MAX_NUM_ALGORITHMS,")."]), - ssh_connection_handler:disconnect( - #ssh_msg_disconnect{code = ?SSH_DISCONNECT_PROTOCOL_ERROR, - description = Err}). + Error = lists:concat(["Received too many algorithms (",length(CL),"+",length(SL)," >= ",?MAX_NUM_ALGORITHMS,")."]), + ?DISCONNECT(?SSH_DISCONNECT_PROTOCOL_ERROR, + Error). select([], []) -> @@ -1810,7 +1779,7 @@ mac('hmac-sha2-512', Key, SeqNum, Data) -> hash(_SSH, _Char, 0) -> <<>>; hash(SSH, Char, N) -> - HashAlg = sha(SSH#ssh.kex), + HashAlg = sha(SSH#ssh.algorithms#alg.kex), K = SSH#ssh.shared_secret, H = SSH#ssh.exchanged_hash, K1 = crypto:hash(HashAlg, [K, H, Char, SSH#ssh.session_id]), @@ -2041,3 +2010,40 @@ trim_tail(Str) -> lists:takewhile(fun(C) -> C=/=$\r andalso C=/=$\n end, Str). + +%%%################################################################ +%%%# +%%%# Tracing +%%%# + +dbg_trace(points, _, _) -> [alg, ssh_messages, raw_messages, hello]; + +dbg_trace(flags, hello, _) -> [c]; +dbg_trace(on, hello, _) -> dbg:tp(?MODULE,hello_version_msg,1,x), + dbg:tp(?MODULE,handle_hello_version,1,x); +dbg_trace(off, hello, _) -> dbg:ctpg(?MODULE,hello_version_msg,1), + dbg:ctpg(?MODULE,handle_hello_version,1); + +dbg_trace(C, raw_messages, A) -> dbg_trace(C, hello, A); +dbg_trace(C, ssh_messages, A) -> dbg_trace(C, hello, A); + +dbg_trace(flags, alg, _) -> [c]; +dbg_trace(on, alg, _) -> dbg:tpl(?MODULE,select_algorithm,4,x); +dbg_trace(off, alg, _) -> dbg:ctpl(?MODULE,select_algorithm,4); + + +dbg_trace(format, hello, {return_from,{?MODULE,hello_version_msg,1},Hello}) -> + ["Going to send hello message:\n", + Hello + ]; +dbg_trace(format, hello, {call,{?MODULE,handle_hello_version,[Hello]}}) -> + ["Received hello message:\n", + Hello + ]; + +dbg_trace(format, alg, {return_from,{?MODULE,select_algorithm,4},{ok,Alg}}) -> + ["Negotiated algorithms:\n", + wr_record(Alg) + ]. + +?wr_record(alg). diff --git a/lib/ssh/src/ssh_transport.hrl b/lib/ssh/src/ssh_transport.hrl index 87c3719514..7d5a4c153e 100644 --- a/lib/ssh/src/ssh_transport.hrl +++ b/lib/ssh/src/ssh_transport.hrl @@ -220,6 +220,9 @@ %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +-define(DISCONNECT(Code, DetailedText), + ssh_connection_handler:disconnect(Code, DetailedText, ?MODULE, ?LINE)). + -define(SSH_DISCONNECT_HOST_NOT_ALLOWED_TO_CONNECT, 1). -define(SSH_DISCONNECT_PROTOCOL_ERROR, 2). -define(SSH_DISCONNECT_KEY_EXCHANGE_FAILED, 3). diff --git a/lib/ssh/src/sshc_sup.erl b/lib/ssh/src/sshc_sup.erl index fd4d8a3c07..f4b39dbbdc 100644 --- a/lib/ssh/src/sshc_sup.erl +++ b/lib/ssh/src/sshc_sup.erl @@ -27,7 +27,7 @@ -behaviour(supervisor). --export([start_link/0, start_child/1, stop_child/1]). +-export([start_link/0, start_child/1]). %% Supervisor callback -export([init/1]). @@ -43,13 +43,6 @@ start_link() -> start_child(Args) -> supervisor:start_child(?MODULE, Args). -stop_child(Client) -> - spawn(fun() -> - ClientSup = whereis(?SSHC_SUP), - supervisor:terminate_child(ClientSup, Client) - end), - ok. - %%%========================================================================= %%% Supervisor callback %%%========================================================================= diff --git a/lib/ssh/test/Makefile b/lib/ssh/test/Makefile index 21359a0386..0a99d31a63 100644 --- a/lib/ssh/test/Makefile +++ b/lib/ssh/test/Makefile @@ -34,11 +34,11 @@ VSN=$(GS_VSN) MODULES= \ ssh_algorithms_SUITE \ ssh_options_SUITE \ - ssh_renegotiate_SUITE \ ssh_basic_SUITE \ ssh_bench_SUITE \ ssh_compat_SUITE \ ssh_connection_SUITE \ + ssh_dbg_SUITE \ ssh_engine_SUITE \ ssh_protocol_SUITE \ ssh_property_test_SUITE \ diff --git a/lib/ssh/test/property_test/ssh_eqc_subsys.erl b/lib/ssh/test/property_test/ssh_eqc_subsys.erl index 30b254b9c0..e7de3ea068 100644 --- a/lib/ssh/test/property_test/ssh_eqc_subsys.erl +++ b/lib/ssh/test/property_test/ssh_eqc_subsys.erl @@ -21,7 +21,7 @@ -module(ssh_eqc_subsys). --behaviour(ssh_daemon_channel). +-behaviour(ssh_server_channel). -export([init/1, handle_msg/2, handle_ssh_msg/2, terminate/2]). diff --git a/lib/ssh/test/ssh_algorithms_SUITE.erl b/lib/ssh/test/ssh_algorithms_SUITE.erl index de6e448ebd..0ce4bd8699 100644 --- a/lib/ssh/test/ssh_algorithms_SUITE.erl +++ b/lib/ssh/test/ssh_algorithms_SUITE.erl @@ -35,7 +35,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}, - {timetrap,{seconds,round(1.5*?TIMEOUT/1000)}}]. + {timetrap,{seconds,60}}]. all() -> %% [{group,kex},{group,cipher}... etc @@ -257,15 +257,14 @@ try_exec_simple_group(Group, Config) -> of _ -> ct:fail("Exec though no group available") catch - error:{badmatch,{error,"No possible diffie-hellman-group-exchange group found"}} -> ok; - error:{badmatch,{error,"Connection closed"}} -> ok + error:{badmatch,{error,"Key exchange failed"}} -> ok end. %%-------------------------------------------------------------------- %% Testing all default groups simple_exec_groups() -> - [{timetrap,{seconds,120}}]. + [{timetrap,{seconds,180}}]. simple_exec_groups(Config) -> Sizes = interpolate( public_key:dh_gex_group_sizes() ), diff --git a/lib/ssh/test/ssh_basic_SUITE.erl b/lib/ssh/test/ssh_basic_SUITE.erl index 365f25fabb..1fa94bef11 100644 --- a/lib/ssh/test/ssh_basic_SUITE.erl +++ b/lib/ssh/test/ssh_basic_SUITE.erl @@ -28,60 +28,12 @@ -include("ssh_test_lib.hrl"). %% Note: This directive should only be used in test suites. -%%-compile(export_all). - -%%% Test cases --export([ - app_test/1, - appup_test/1, - cli/1, - close/1, - daemon_already_started/1, - daemon_opt_fd/1, - multi_daemon_opt_fd/1, - double_close/1, - exec/1, - exec_compressed/1, - exec_key_differs1/1, - exec_key_differs2/1, - exec_key_differs3/1, - exec_key_differs_fail/1, - fail_daemon_start/1, - idle_time_client/1, - idle_time_server/1, - inet6_option/1, - inet_option/1, - internal_error/1, - known_hosts/1, - login_bad_pwd_no_retry1/1, - login_bad_pwd_no_retry2/1, - login_bad_pwd_no_retry3/1, - login_bad_pwd_no_retry4/1, - login_bad_pwd_no_retry5/1, - misc_ssh_options/1, - openssh_zlib_basic_test/1, - packet_size/1, - pass_phrase/1, - peername_sockname/1, - send/1, - shell/1, - shell_no_unicode/1, - shell_unicode_string/1, - ssh_info_print/1, - key_callback/1, - key_callback_options/1, - shell_exit_status/1 - ]). - -%%% Common test callbacks --export([suite/0, all/0, groups/0, - init_per_suite/1, end_per_suite/1, - init_per_group/2, end_per_group/2, - init_per_testcase/2, end_per_testcase/2 - ]). +-compile(export_all). -define(NEWLINE, <<"\r\n">>). +-define(REKEY_DATA_TMO, 65000). + %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- @@ -91,76 +43,97 @@ suite() -> {timetrap,{seconds,40}}]. all() -> - [app_test, - appup_test, - {group, dsa_key}, - {group, rsa_key}, - {group, ecdsa_sha2_nistp256_key}, - {group, ecdsa_sha2_nistp384_key}, - {group, ecdsa_sha2_nistp521_key}, - {group, dsa_pass_key}, - {group, rsa_pass_key}, - {group, ecdsa_sha2_nistp256_pass_key}, - {group, ecdsa_sha2_nistp384_pass_key}, - {group, ecdsa_sha2_nistp521_pass_key}, - {group, host_user_key_differs}, - {group, key_cb}, - {group, internal_error}, - {group, rsa_host_key_is_actualy_ecdsa}, - daemon_already_started, - double_close, - daemon_opt_fd, - multi_daemon_opt_fd, - packet_size, - ssh_info_print, - {group, login_bad_pwd_no_retry}, - shell_exit_status - ]. + [{group, all_tests}]. + groups() -> - [{dsa_key, [], basic_tests()}, - {rsa_key, [], basic_tests()}, - {ecdsa_sha2_nistp256_key, [], basic_tests()}, - {ecdsa_sha2_nistp384_key, [], basic_tests()}, - {ecdsa_sha2_nistp521_key, [], basic_tests()}, + [{all_tests, [parallel], [{group, ssh_renegotiate_SUITE}, + {group, ssh_basic_SUITE} + ]}, + {ssh_basic_SUITE, [], [app_test, + appup_test, + {group, dsa_key}, + {group, rsa_key}, + {group, ecdsa_sha2_nistp256_key}, + {group, ecdsa_sha2_nistp384_key}, + {group, ecdsa_sha2_nistp521_key}, + {group, dsa_pass_key}, + {group, rsa_pass_key}, + {group, ecdsa_sha2_nistp256_pass_key}, + {group, ecdsa_sha2_nistp384_pass_key}, + {group, ecdsa_sha2_nistp521_pass_key}, + {group, host_user_key_differs}, + {group, key_cb}, + {group, internal_error}, + {group, rsa_host_key_is_actualy_ecdsa}, + daemon_already_started, + double_close, + daemon_opt_fd, + multi_daemon_opt_fd, + packet_size, + ssh_info_print, + {group, login_bad_pwd_no_retry}, + shell_exit_status + ]}, + + {ssh_renegotiate_SUITE, [parallel], [rekey, + rekey_limit, + renegotiate1, + renegotiate2]}, + + {dsa_key, [], [{group, basic}]}, + {rsa_key, [], [{group, basic}]}, + {ecdsa_sha2_nistp256_key, [], [{group, basic}]}, + {ecdsa_sha2_nistp384_key, [], [{group, basic}]}, + {ecdsa_sha2_nistp521_key, [], [{group, basic}]}, {rsa_host_key_is_actualy_ecdsa, [], [fail_daemon_start]}, - {host_user_key_differs, [], [exec_key_differs1, - exec_key_differs2, - exec_key_differs3, - exec_key_differs_fail]}, + {host_user_key_differs, [parallel], [exec_key_differs1, + exec_key_differs2, + exec_key_differs3, + exec_key_differs_fail]}, {dsa_pass_key, [], [pass_phrase]}, {rsa_pass_key, [], [pass_phrase]}, {ecdsa_sha2_nistp256_pass_key, [], [pass_phrase]}, {ecdsa_sha2_nistp384_pass_key, [], [pass_phrase]}, {ecdsa_sha2_nistp521_pass_key, [], [pass_phrase]}, - {key_cb, [], [key_callback, key_callback_options]}, + {key_cb, [parallel], [key_callback, key_callback_options]}, {internal_error, [], [internal_error]}, - {login_bad_pwd_no_retry, [], [login_bad_pwd_no_retry1, - login_bad_pwd_no_retry2, - login_bad_pwd_no_retry3, - login_bad_pwd_no_retry4, - login_bad_pwd_no_retry5 - ]} + {login_bad_pwd_no_retry, [parallel], [login_bad_pwd_no_retry1, + login_bad_pwd_no_retry2, + login_bad_pwd_no_retry3, + login_bad_pwd_no_retry4, + login_bad_pwd_no_retry5 + ]}, + + {basic, [], [{group,p_basic}, + shell, shell_no_unicode, shell_unicode_string, + close, + known_hosts + ]}, + {p_basic, [parallel], [send, peername_sockname, + exec, exec_compressed, + cli, + idle_time_client, idle_time_server, openssh_zlib_basic_test, + misc_ssh_options, inet_option, inet6_option]} ]. -basic_tests() -> - [send, close, peername_sockname, - exec, exec_compressed, - shell, shell_no_unicode, shell_unicode_string, - cli, known_hosts, - idle_time_client, idle_time_server, openssh_zlib_basic_test, - misc_ssh_options, inet_option, inet6_option]. + %%-------------------------------------------------------------------- init_per_suite(Config) -> - ?CHECK_CRYPTO(Config). + ?CHECK_CRYPTO(begin + ssh:start(), + Config + end). end_per_suite(_Config) -> ssh:stop(). %%-------------------------------------------------------------------- +init_per_group(ssh_renegotiate_SUITE, Config) -> + [{preferred_algorithms, ssh:default_algorithms()} | Config]; init_per_group(dsa_key, Config) -> case lists:member('ssh-dss', ssh_transport:default_algorithms(public_key)) of @@ -414,7 +387,6 @@ init_per_testcase(TC, Config) when TC==shell_no_unicode ; PrivDir = proplists:get_value(priv_dir, Config), UserDir = proplists:get_value(priv_dir, Config), SysDir = proplists:get_value(data_dir, Config), - ssh:start(), Sftpd = {_Pid, _Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, {user_dir, PrivDir}, @@ -437,7 +409,6 @@ init_per_testcase(inet6_option, Config) -> {skip,"No ipv6 interface address"} end; init_per_testcase(_TestCase, Config) -> - ssh:start(), Config. end_per_testcase(TestCase, Config) when TestCase == server_password_option; @@ -458,7 +429,6 @@ end_per_testcase(_TestCase, Config) -> end_per_testcase(Config). end_per_testcase(_Config) -> - ssh:stop(), ok. %%-------------------------------------------------------------------- @@ -480,8 +450,8 @@ misc_ssh_options(Config) when is_list(Config) -> SystemDir = filename:join(proplists:get_value(priv_dir, Config), system), UserDir = proplists:get_value(priv_dir, Config), - CMiscOpt0 = [{connect_timeout, 1000}, {user_dir, UserDir}], - CMiscOpt1 = [{connect_timeout, infinity}, {user_dir, UserDir}], + CMiscOpt0 = [{connect_timeout, 1000}, {user_dir, UserDir}, {silently_accept_hosts, true}], + CMiscOpt1 = [{connect_timeout, infinity}, {user_dir, UserDir}, {silently_accept_hosts, true}], SMiscOpt0 = [{user_dir, UserDir}, {system_dir, SystemDir}], SMiscOpt1 = [{user_dir, UserDir}, {system_dir, SystemDir}], @@ -1124,11 +1094,14 @@ packet_size(Config) -> ct:log("Try max_packet_size=~p",[MaxPacketSize]), {ok,Ch} = ssh_connection:session_channel(Conn, 1000, MaxPacketSize, 60000), ok = ssh_connection:shell(Conn, Ch), - rec(Server, Conn, Ch, MaxPacketSize) + rec(Server, Conn, Ch, MaxPacketSize), + ssh_connection:close(Conn, Ch) end, [0, 1, 10, 25]), ssh:close(Conn), - ssh:stop_daemon(Server). + ssh:stop_daemon(Server), + ok. + rec(Server, Conn, Ch, MaxSz) -> receive @@ -1141,7 +1114,9 @@ rec(Server, Conn, Ch, MaxSz) -> ssh:stop_daemon(Server), ct:fail("Does not obey max_packet_size=~p",[MaxSz]) after - 2000 -> ok + 2000 -> + ct:log("~p: ok!",[MaxSz]), + ok end. %%-------------------------------------------------------------------- @@ -1350,6 +1325,156 @@ shell_exit_status(Config) when is_list(Config) -> ssh:stop_daemon(Pid). +%%% Idle timeout test +rekey() -> [{timetrap,{seconds,90}}]. + +rekey(Config) -> + {Pid, Host, Port} = + ssh_test_lib:std_daemon(Config, + [{rekey_limit, 0}]), + ConnectionRef = + ssh_test_lib:std_connect(Config, Host, Port, + [{rekey_limit, 0}]), + Kex1 = ssh_test_lib:get_kex_init(ConnectionRef), + receive + after ?REKEY_DATA_TMO -> + %%By this time rekeying would have been done + Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), + false = (Kex2 == Kex1), + ssh:close(ConnectionRef), + ssh:stop_daemon(Pid) + end. + +%%-------------------------------------------------------------------- + +%%% Test rekeying by data volume + +rekey_limit() -> [{timetrap,{seconds,400}}]. + +rekey_limit(Config) -> + UserDir = proplists:get_value(priv_dir, Config), + DataFile = filename:join(UserDir, "rekey.data"), + + Algs = proplists:get_value(preferred_algorithms, Config), + {Pid, Host, Port} = ssh_test_lib:std_daemon(Config,[{max_random_length_padding,0}, + {preferred_algorithms,Algs}]), + + ConnectionRef = ssh_test_lib:std_connect(Config, Host, Port, [{rekey_limit, 6000}, + {max_random_length_padding,0}]), + {ok, SftpPid} = ssh_sftp:start_channel(ConnectionRef), + + Kex1 = ssh_test_lib:get_kex_init(ConnectionRef), + + timer:sleep(?REKEY_DATA_TMO), + Kex1 = ssh_test_lib:get_kex_init(ConnectionRef), + + Data = lists:duplicate(159000,1), + ok = ssh_sftp:write_file(SftpPid, DataFile, Data), + + timer:sleep(?REKEY_DATA_TMO), + Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), + + false = (Kex2 == Kex1), + + timer:sleep(?REKEY_DATA_TMO), + Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), + + ok = ssh_sftp:write_file(SftpPid, DataFile, "hi\n"), + + timer:sleep(?REKEY_DATA_TMO), + Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), + + false = (Kex2 == Kex1), + + timer:sleep(?REKEY_DATA_TMO), + Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), + + ssh_sftp:stop_channel(SftpPid), + ssh:close(ConnectionRef), + ssh:stop_daemon(Pid). + +%%-------------------------------------------------------------------- + +%%% Test rekeying with simulataneous send request + +renegotiate1(Config) -> + UserDir = proplists:get_value(priv_dir, Config), + DataFile = filename:join(UserDir, "renegotiate1.data"), + + Algs = proplists:get_value(preferred_algorithms, Config), + {Pid, Host, DPort} = ssh_test_lib:std_daemon(Config,[{max_random_length_padding,0}, + {preferred_algorithms,Algs}]), + + RPort = ssh_test_lib:inet_port(), + {ok,RelayPid} = ssh_relay:start_link({0,0,0,0}, RPort, Host, DPort), + + + ConnectionRef = ssh_test_lib:std_connect(Config, Host, RPort, [{max_random_length_padding,0}]), + {ok, SftpPid} = ssh_sftp:start_channel(ConnectionRef), + + Kex1 = ssh_test_lib:get_kex_init(ConnectionRef), + + {ok, Handle} = ssh_sftp:open(SftpPid, DataFile, [write]), + + ok = ssh_sftp:write(SftpPid, Handle, "hi\n"), + + ssh_relay:hold(RelayPid, rx, 20, 1000), + ssh_connection_handler:renegotiate(ConnectionRef), + spawn(fun() -> ok=ssh_sftp:write(SftpPid, Handle, "another hi\n") end), + + timer:sleep(2000), + + Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), + + false = (Kex2 == Kex1), + + ssh_relay:stop(RelayPid), + ssh_sftp:stop_channel(SftpPid), + ssh:close(ConnectionRef), + ssh:stop_daemon(Pid). + +%%-------------------------------------------------------------------- + +%%% Test rekeying with inflight messages from peer + +renegotiate2(Config) -> + UserDir = proplists:get_value(priv_dir, Config), + DataFile = filename:join(UserDir, "renegotiate2.data"), + + Algs = proplists:get_value(preferred_algorithms, Config), + {Pid, Host, DPort} = ssh_test_lib:std_daemon(Config,[{max_random_length_padding,0}, + {preferred_algorithms,Algs}]), + + RPort = ssh_test_lib:inet_port(), + {ok,RelayPid} = ssh_relay:start_link({0,0,0,0}, RPort, Host, DPort), + + ConnectionRef = ssh_test_lib:std_connect(Config, Host, RPort, [{max_random_length_padding,0}]), + {ok, SftpPid} = ssh_sftp:start_channel(ConnectionRef), + + Kex1 = ssh_test_lib:get_kex_init(ConnectionRef), + + {ok, Handle} = ssh_sftp:open(SftpPid, DataFile, [write]), + + ok = ssh_sftp:write(SftpPid, Handle, "hi\n"), + + ssh_relay:hold(RelayPid, rx, 20, infinity), + spawn(fun() -> ok=ssh_sftp:write(SftpPid, Handle, "another hi\n") end), + %% need a small pause here to ensure ssh_sftp:write is executed + ct:sleep(10), + ssh_connection_handler:renegotiate(ConnectionRef), + ssh_relay:release(RelayPid, rx), + + timer:sleep(2000), + + Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), + + false = (Kex2 == Kex1), + + ssh_relay:stop(RelayPid), + ssh_sftp:stop_channel(SftpPid), + ssh:close(ConnectionRef), + ssh:stop_daemon(Pid). + %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- diff --git a/lib/ssh/test/ssh_bench_dev_null.erl b/lib/ssh/test/ssh_bench_dev_null.erl index 5166247714..f9da80b6d7 100644 --- a/lib/ssh/test/ssh_bench_dev_null.erl +++ b/lib/ssh/test/ssh_bench_dev_null.erl @@ -22,7 +22,7 @@ %%% Description: Example ssh server -module(ssh_bench_dev_null). --behaviour(ssh_daemon_channel). +-behaviour(ssh_server_channel). -record(state, { cm, diff --git a/lib/ssh/test/ssh_compat_SUITE.erl b/lib/ssh/test/ssh_compat_SUITE.erl index f7eda1dc08..6c0e010bf5 100644 --- a/lib/ssh/test/ssh_compat_SUITE.erl +++ b/lib/ssh/test/ssh_compat_SUITE.erl @@ -41,8 +41,7 @@ %%-------------------------------------------------------------------- suite() -> - [%%{ct_hooks,[ts_install_cth]}, - {timetrap,{seconds,40}}]. + [{timetrap,{seconds,60}}]. all() -> %% [check_docker_present] ++ diff --git a/lib/ssh/test/ssh_connection_SUITE.erl b/lib/ssh/test/ssh_connection_SUITE.erl index 9587c0c251..257f2f70d7 100644 --- a/lib/ssh/test/ssh_connection_SUITE.erl +++ b/lib/ssh/test/ssh_connection_SUITE.erl @@ -50,6 +50,13 @@ all() -> start_shell, start_shell_exec, start_shell_exec_fun, + start_shell_exec_fun2, + start_shell_exec_fun3, + start_shell_exec_direct_fun, + start_shell_exec_direct_fun2, + start_shell_exec_direct_fun3, + start_shell_exec_direct_fun1_error, + start_shell_exec_direct_fun1_error_type, start_shell_sock_exec_fun, start_shell_sock_daemon_exec, connect_sock_not_tcp, @@ -522,7 +529,7 @@ start_shell_exec(Config) when is_list(Config) -> {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, {user_dir, UserDir}, {password, "morot"}, - {exec, {?MODULE,ssh_exec,[]}} ]), + {exec, {?MODULE,ssh_exec_echo,[]}} ]), ConnectionRef = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, {user, "foo"}, @@ -535,7 +542,7 @@ start_shell_exec(Config) when is_list(Config) -> success = ssh_connection:exec(ConnectionRef, ChannelId0, "testing", infinity), receive - {ssh_cm, ConnectionRef, {data, _ChannelId, 0, <<"testing\r\n">>}} -> + {ssh_cm, ConnectionRef, {data, _ChannelId, 0, <<"echo testing\r\n">>}} -> ok after 5000 -> ct:fail("Exec Timeout") @@ -618,10 +625,49 @@ exec_erlang_term_non_default_shell(Config) when is_list(Config) -> TestResult. %%-------------------------------------------------------------------- -start_shell_exec_fun() -> - [{doc, "start shell to exec command"}]. +start_shell_exec_fun(Config) -> + do_start_shell_exec_fun(fun ssh_exec_echo/1, + "testing", <<"echo testing\r\n">>, 0, + Config). + +start_shell_exec_fun2(Config) -> + do_start_shell_exec_fun(fun ssh_exec_echo/2, + "testing", <<"echo foo testing\r\n">>, 0, + Config). + +start_shell_exec_fun3(Config) -> + do_start_shell_exec_fun(fun ssh_exec_echo/3, + "testing", <<"echo foo testing\r\n">>, 0, + Config). + +start_shell_exec_direct_fun(Config) -> + do_start_shell_exec_fun({direct, fun ssh_exec_direct_echo/1}, + "testing", <<"echo testing\n">>, 0, + Config). + +start_shell_exec_direct_fun2(Config) -> + do_start_shell_exec_fun({direct, fun ssh_exec_direct_echo/2}, + "testing", <<"echo foo testing">>, 0, + Config). + +start_shell_exec_direct_fun3(Config) -> + do_start_shell_exec_fun({direct, fun ssh_exec_direct_echo/3}, + "testing", <<"echo foo testing">>, 0, + Config). + +start_shell_exec_direct_fun1_error(Config) -> + do_start_shell_exec_fun({direct, fun ssh_exec_direct_echo_error_return/1}, + "testing", <<"Error in \"testing\": {bad}\n">>, 1, + Config). + +start_shell_exec_direct_fun1_error_type(Config) -> + do_start_shell_exec_fun({direct, fun ssh_exec_direct_echo_error_return_type/1}, + "testing", <<"Error in \"testing\": Bad exec-plugin return: very_bad\n">>, 1, + Config). + + -start_shell_exec_fun(Config) when is_list(Config) -> +do_start_shell_exec_fun(Fun, Command, Expect, ExpectType, Config) -> PrivDir = proplists:get_value(priv_dir, Config), UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth file:make_dir(UserDir), @@ -629,7 +675,7 @@ start_shell_exec_fun(Config) when is_list(Config) -> {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, {user_dir, UserDir}, {password, "morot"}, - {exec, fun ssh_exec/1}]), + {exec, Fun}]), ConnectionRef = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, {user, "foo"}, @@ -639,14 +685,19 @@ start_shell_exec_fun(Config) when is_list(Config) -> {ok, ChannelId0} = ssh_connection:session_channel(ConnectionRef, infinity), - success = ssh_connection:exec(ConnectionRef, ChannelId0, - "testing", infinity), + success = ssh_connection:exec(ConnectionRef, ChannelId0, Command, infinity), receive - {ssh_cm, ConnectionRef, {data, _ChannelId, 0, <<"testing\r\n">>}} -> + {ssh_cm, ConnectionRef, {data, _ChannelId, ExpectType, Expect}} -> ok after 5000 -> - ct:fail("Exec Timeout") + receive + Other -> + ct:pal("Received other:~n~p",[Other]), + ct:fail("Unexpected response") + after 0 -> + ct:fail("Exec Timeout") + end end, ssh:close(ConnectionRef), @@ -664,7 +715,7 @@ start_shell_sock_exec_fun(Config) when is_list(Config) -> {Pid, HostD, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, {user_dir, UserDir}, {password, "morot"}, - {exec, fun ssh_exec/1}]), + {exec, fun ssh_exec_echo/1}]), Host = ssh_test_lib:ntoa(ssh_test_lib:mangle_connect_address(HostD)), {ok, Sock} = ssh_test_lib:gen_tcp_connect(Host, Port, [{active,false}]), @@ -680,7 +731,7 @@ start_shell_sock_exec_fun(Config) when is_list(Config) -> "testing", infinity), receive - {ssh_cm, ConnectionRef, {data, _ChannelId, 0, <<"testing\r\n">>}} -> + {ssh_cm, ConnectionRef, {data, _ChannelId, 0, <<"echo testing\r\n">>}} -> ok after 5000 -> ct:fail("Exec Timeout") @@ -704,7 +755,7 @@ start_shell_sock_daemon_exec(Config) -> {ok, _Pid} = ssh:daemon(Ss, [{system_dir, SysDir}, {user_dir, UserDir}, {password, "morot"}, - {exec, fun ssh_exec/1}]) + {exec, fun ssh_exec_echo/1}]) end), {ok,Sc} = gen_tcp:accept(Sl), {ok,ConnectionRef} = ssh:connect(Sc, [{silently_accept_hosts, true}, @@ -719,7 +770,7 @@ start_shell_sock_daemon_exec(Config) -> "testing", infinity), receive - {ssh_cm, ConnectionRef, {data, _ChannelId, 0, <<"testing\r\n">>}} -> + {ssh_cm, ConnectionRef, {data, _ChannelId, 0, <<"echo testing\r\n">>}} -> ok after 5000 -> ct:fail("Exec Timeout") @@ -830,7 +881,7 @@ stop_listener(Config) when is_list(Config) -> {Pid0, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, {user_dir, UserDir}, {password, "morot"}, - {exec, fun ssh_exec/1}]), + {exec, fun ssh_exec_echo/1}]), ConnectionRef0 = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, {user, "foo"}, @@ -850,7 +901,7 @@ stop_listener(Config) when is_list(Config) -> success = ssh_connection:exec(ConnectionRef0, ChannelId0, "testing", infinity), receive - {ssh_cm, ConnectionRef0, {data, ChannelId0, 0, <<"testing\r\n">>}} -> + {ssh_cm, ConnectionRef0, {data, ChannelId0, 0, <<"echo testing\r\n">>}} -> ok after 5000 -> ct:fail("Exec Timeout") @@ -859,7 +910,7 @@ stop_listener(Config) when is_list(Config) -> case ssh_test_lib:daemon(Port, [{system_dir, SysDir}, {user_dir, UserDir}, {password, "potatis"}, - {exec, fun ssh_exec/1}]) of + {exec, fun ssh_exec_echo/1}]) of {Pid1, Host, Port} -> ConnectionRef1 = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, {user, "foo"}, @@ -1070,7 +1121,22 @@ start_our_shell(_User, _Peer) -> %% Don't actually loop, just exit end). -ssh_exec(Cmd) -> + +ssh_exec_echo(Cmd) -> spawn(fun() -> - io:format(Cmd ++ "\n") + io:format("echo "++Cmd ++ "\n") end). + +ssh_exec_echo(Cmd, User) -> + spawn(fun() -> + io:format(io_lib:format("echo ~s ~s\n",[User,Cmd])) + end). +ssh_exec_echo(Cmd, User, _PeerAddr) -> + ssh_exec_echo(Cmd,User). + +ssh_exec_direct_echo(Cmd) -> {ok, io_lib:format("echo ~s~n",[Cmd])}. +ssh_exec_direct_echo(Cmd, User) -> {ok, io_lib:format("echo ~s ~s",[User,Cmd])}. +ssh_exec_direct_echo(Cmd, User, _PeerAddr) -> ssh_exec_direct_echo(Cmd,User). + +ssh_exec_direct_echo_error_return(_Cmd) -> {error, {bad}}. +ssh_exec_direct_echo_error_return_type(_Cmd) -> very_bad. diff --git a/lib/ssh/test/ssh_dbg_SUITE.erl b/lib/ssh/test/ssh_dbg_SUITE.erl new file mode 100644 index 0000000000..5439817d10 --- /dev/null +++ b/lib/ssh/test/ssh_dbg_SUITE.erl @@ -0,0 +1,409 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% + +-module(ssh_dbg_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("ssh/src/ssh.hrl"). +-include("ssh_test_lib.hrl"). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +%%-------------------------------------------------------------------- +%% Common Test interface functions ----------------------------------- +%%-------------------------------------------------------------------- + +suite() -> + [{ct_hooks,[ts_install_cth]}, + {timetrap,{seconds,60}}]. + +all() -> + [basic, + dbg_alg_terminate, + dbg_ssh_messages, + dbg_connections, + dbg_channels + ]. + +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + ?CHECK_CRYPTO(begin + ssh:start(), + Config + end). + +end_per_suite(_Config) -> + ssh:stop(). + +%%-------------------------------------------------------------------- +init_per_testcase(_TC, Config) -> + Config. + +end_per_testcase(_TC, Config) -> + ssh_dbg:stop(), + Config. + +%%-------------------------------------------------------------------- +-define(USR, "foo"). +-define(PWD, "bar"). + +-define(DBG_RECEIVE(ExpectPfx, Ref, C, Pid), + receive + {Ref, [_, C, ExpectPfx++_]} -> + ok + + after 5000 -> + ssh_dbg:stop(), + ssh:stop_daemon(Pid), + ct:fail("No '~s' debug message",[ExpectPfx]) + end + ). +%%-------------------------------------------------------------------- +%% Test Cases -------------------------------------------------------- +%%-------------------------------------------------------------------- + +basic(_Config) -> + L0 = ssh_dbg:start(), + true = is_pid(whereis(ssh_dbg)), + true = is_list(L0), + + {ok,L0} = ssh_dbg:on(), + {ok,L0} = ssh_dbg:on(), + + L1 = [hd(L0)], + {ok,L1} = ssh_dbg:off(tl(L0)), + + {ok,L1} = ssh_dbg:go_on(), + + {ok,[]} = ssh_dbg:off(), + {ok,[]} = ssh_dbg:off(), + + ok = ssh_dbg:stop(), + undefined = whereis(ssh_dbg). + + +%%-------------------------------------------------------------------- +dbg_alg_terminate(Config) -> + SystemDir = proplists:get_value(data_dir, Config), + UserDir = proplists:get_value(priv_dir, Config), + + Ref = ssh_dbg_start(), + {ok,[alg,connections,terminate]} = ssh_dbg:on([alg,terminate,connections]), + {ok,[alg,terminate]} = ssh_dbg:off(connections), % just testing that terminate is not canceled + + Parent = self(), + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, + {user_dir, UserDir}, + {user_passwords, [{?USR,?PWD}]}, + {connectfun, fun(_,_,_) -> + Parent ! {daemon_c,Ref,self()} + end}, + {failfun, fun ssh_test_lib:failfun/2}]), + C = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user_dir, UserDir}, + {user,?USR}, + {password,?PWD}, + {user_interaction, false}]), + + %% Daemon connection ref (D): + D = receive + {daemon_c,Ref,D0} -> D0 + end, + ct:log("~p:~p~nC = ~p, D=~p",[?MODULE,?LINE, C, D]), + + ?DBG_RECEIVE("Negotiated algorithms:", Ref, C, Pid), + ?DBG_RECEIVE("Negotiated algorithms:", Ref, D, Pid), + + ssh:close(C), + ?DBG_RECEIVE("Connection Terminating:", Ref, C, Pid), + ?DBG_RECEIVE("Connection Terminating:", Ref, D, Pid), + + stop_and_fail_if_unhandled_dbg_msgs(Ref, [C,D], Pid). + +%%-------------------------------------------------------------------- +dbg_connections(Config) -> + SystemDir = proplists:get_value(data_dir, Config), + UserDir = proplists:get_value(priv_dir, Config), + + Ref = ssh_dbg_start(), + {ok,[connections,terminate]} = ssh_dbg:on([connections, terminate]), + {ok,[connections]} = ssh_dbg:off(terminate), % Just testing that terminate doesn't cancel connections + + Parent = self(), + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, + {user_dir, UserDir}, + {user_passwords, [{?USR,?PWD}]}, + {connectfun, fun(_,_,_) -> + Parent ! {daemon_c,Ref,self()} + end}, + {failfun, fun ssh_test_lib:failfun/2}]), + + ?DBG_RECEIVE("Starting LISTENER on ", Ref, _, Pid), + + C = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user_dir, UserDir}, + {user,?USR}, + {password,?PWD}, + {user_interaction, false}]), + + %% Daemon connection ref (D): + D = receive + {daemon_c,Ref,D0} -> D0 + end, + ct:log("~p:~p~nC = ~p, D=~p",[?MODULE,?LINE, C, D]), + + ?DBG_RECEIVE("Starting server connection:", Ref, D, Pid), + ?DBG_RECEIVE("Starting client connection:", Ref, C, Pid), + + ssh:close(C), + ?DBG_RECEIVE("Connection Terminating:", Ref, C, Pid), + ?DBG_RECEIVE("Connection Terminating:", Ref, D, Pid), + + stop_and_fail_if_unhandled_dbg_msgs(Ref, [C,D], Pid). + +%%-------------------------------------------------------------------- +dbg_ssh_messages(Config) -> + SystemDir = proplists:get_value(data_dir, Config), + UserDir = proplists:get_value(priv_dir, Config), + + Parent = self(), + Ref = make_ref(), + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, + {user_dir, UserDir}, + {user_passwords, [{?USR,?PWD}]}, + {connectfun, fun(_,_,_) -> + Parent ! {daemon_c,Ref,self()} + end}, + {failfun, fun ssh_test_lib:failfun/2}]), + + ssh_dbg_start(Ref), + {ok,[ssh_messages]} = ssh_dbg:on([ssh_messages]), + + C = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user_dir, UserDir}, + {user,?USR}, + {password,?PWD}, + {user_interaction, false}]), + + %% Daemon connection ref (D): + D = receive + {daemon_c,Ref,D0} -> D0 + end, + ct:log("~p:~p~nC = ~p, D=~p",[?MODULE,?LINE, C, D]), + + ?DBG_RECEIVE("Going to send hello message:", Ref, C, Pid), + ?DBG_RECEIVE("Received hello message:", Ref, D, Pid), + + ?DBG_RECEIVE("Going to send hello message:", Ref, D, Pid), + ?DBG_RECEIVE("Received hello message:", Ref, C, Pid), + + ?DBG_RECEIVE("Going to send SSH_MSG_KEXINIT:", Ref, C, Pid), + ?DBG_RECEIVE("Received SSH_MSG_KEXINIT:", Ref, D, Pid), + + ?DBG_RECEIVE("Going to send SSH_MSG_KEXINIT:", Ref, D, Pid), + ?DBG_RECEIVE("Received SSH_MSG_KEXINIT:", Ref, C, Pid), + + case atom_to_list( (ssh_connection_handler:alg(C))#alg.kex ) of + "ecdh-"++_ -> + ?DBG_RECEIVE("Going to send SSH_MSG_KEX_ECDH_INIT:", Ref, C, Pid), + ?DBG_RECEIVE("Received SSH_MSG_KEX_ECDH_INIT:", Ref, D, Pid), + ?DBG_RECEIVE("Going to send SSH_MSG_KEX_ECDH_REPLY:", Ref, D, Pid), + ?DBG_RECEIVE("Received SSH_MSG_KEX_ECDH_REPLY:", Ref, C, Pid); + + "diffie-hellman-group-exchange-"++_ -> + ?DBG_RECEIVE("Going to send SSH_MSG_KEX_DH_GEX_REQUEST:", Ref, C, Pid), + ?DBG_RECEIVE("Received SSH_MSG_KEX_DH_GEX_REQUEST:", Ref, D, Pid), + ?DBG_RECEIVE("Going to send SSH_MSG_KEX_DH_GEX_GROUP:", Ref, D, Pid), + ?DBG_RECEIVE("Received SSH_MSG_KEX_DH_GEX_GROUP:", Ref, C, Pid), + ?DBG_RECEIVE("Going to send SSH_MSG_KEX_DH_GEX_INIT:", Ref, C, Pid), + ?DBG_RECEIVE("Received SSH_MSG_KEX_DH_GEX_INIT:", Ref, D, Pid), + ?DBG_RECEIVE("Going to send SSH_MSG_KEX_DH_GEX_REPLY:", Ref, D, Pid), + ?DBG_RECEIVE("Received SSH_MSG_KEX_DH_GEX_REPLY:", Ref, C, Pid); + + "diffie-hellman-group"++_ -> + ?DBG_RECEIVE("Going to send SSH_MSG_KEXDH_INIT:", Ref, C, Pid), + ?DBG_RECEIVE("Received SSH_MSG_KEXDH_INIT:", Ref, D, Pid), + ?DBG_RECEIVE("Going to send SSH_MSG_KEXDH_REPLY:", Ref, D, Pid), + ?DBG_RECEIVE("Received SSH_MSG_KEXDH_REPLY:", Ref, C, Pid) + end, + + + ?DBG_RECEIVE("Going to send SSH_MSG_NEWKEYS:", Ref, C, Pid), + ?DBG_RECEIVE("Received SSH_MSG_NEWKEYS:", Ref, D, Pid), + + ?DBG_RECEIVE("Going to send SSH_MSG_NEWKEYS:", Ref, D, Pid), + ?DBG_RECEIVE("Received SSH_MSG_NEWKEYS:", Ref, C, Pid), + + ?DBG_RECEIVE("Going to send SSH_MSG_SERVICE_REQUEST:", Ref, C, Pid), + ?DBG_RECEIVE("Received SSH_MSG_SERVICE_REQUEST:", Ref, D, Pid), + + ?DBG_RECEIVE("Going to send SSH_MSG_SERVICE_ACCEPT:", Ref, D, Pid), + ?DBG_RECEIVE("Received SSH_MSG_SERVICE_ACCEPT:", Ref, C, Pid), + + ?DBG_RECEIVE("Going to send SSH_MSG_USERAUTH_REQUEST:", Ref, C, Pid), + ?DBG_RECEIVE("Received SSH_MSG_USERAUTH_REQUEST:", Ref, D, Pid), + + ?DBG_RECEIVE("Going to send SSH_MSG_USERAUTH_FAILURE:", Ref, D, Pid), + ?DBG_RECEIVE("Received SSH_MSG_USERAUTH_FAILURE:", Ref, C, Pid), + + ?DBG_RECEIVE("Going to send SSH_MSG_USERAUTH_REQUEST:", Ref, C, Pid), + ?DBG_RECEIVE("Received SSH_MSG_USERAUTH_REQUEST:", Ref, D, Pid), + + ?DBG_RECEIVE("Going to send SSH_MSG_USERAUTH_SUCCESS:", Ref, D, Pid), + ?DBG_RECEIVE("Received SSH_MSG_USERAUTH_SUCCESS:", Ref, C, Pid), + + + UnexpectedMsgs = + dbg_SKIP(Ref, + [S_R ++ P ++ ":" || P <- ["SSH_MSG_USERAUTH_REQUEST", + "SSH_MSG_USERAUTH_INFO_REQUEST", + "SSH_MSG_USERAUTH_INFO_RESPONSE", + "SSH_MSG_USERAUTH_FAILURE", + "SSH_MSG_EXT_INFO" + ], + S_R <- ["Going to send ", + "Received " + ] + ]), + + ssh:close(C), + stop_and_fail_if_unhandled_dbg_msgs(UnexpectedMsgs, Ref, [C,D], Pid). + +%%-------------------------------------------------------------------- +dbg_channels(Config) -> + SystemDir = proplists:get_value(data_dir, Config), + UserDir = proplists:get_value(priv_dir, Config), + + Ref = ssh_dbg_start(), + {ok,[channels,connections]} = ssh_dbg:on([connections, channels]), + + Parent = self(), + TimeoutShell = + fun() -> + io:format("TimeoutShell started!~n",[]), + timer:sleep(1000), + Parent ! {daemon_channel,Ref,self()}, + ct:log("~p TIMEOUT!",[self()]) + end, + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, + {user_dir, UserDir}, + {user_passwords, [{?USR,?PWD}]}, + {connectfun, fun(_,_,_) -> + Parent ! {daemon_c,Ref,self()} + end}, + {shell, fun(_User) -> + spawn(TimeoutShell) + end + }, + {failfun, fun ssh_test_lib:failfun/2}]), + + ?DBG_RECEIVE("Starting LISTENER on ", Ref, _, Pid), + + C = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user_dir, UserDir}, + {user,?USR}, + {password,?PWD}, + {user_interaction, false}]), + {ok, Ch0} = ssh_connection:session_channel(C, infinity), + ok = ssh_connection:shell(C, Ch0), + + %% Daemon connection ref (D): + D = receive {daemon_c,Ref,D0} -> D0 end, + + %% Daemon channel (Dch): + Dch = receive {daemon_channel,Ref,Dch0} -> Dch0 end, + ct:log("~p:~p~nC = ~p, D=~p, Dch=~p~n~s",[?MODULE,?LINE, C, D, Dch, ssh_info:string()]), + + ?DBG_RECEIVE("Starting server connection:", Ref, D, Pid), + ?DBG_RECEIVE("Starting client connection:", Ref, C, Pid), + ?DBG_RECEIVE("Server Channel Starting:", Ref, _, Pid), + ?DBG_RECEIVE("Server Channel Terminating:", Ref, _, Pid), + + stop_and_fail_if_unhandled_dbg_msgs(Ref, [C,D], Pid). + +%%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- + +ssh_dbg_start() -> + ssh_dbg_start(make_ref()). + +ssh_dbg_start(Ref) -> + Parent = self(), + [_|_] = ssh_dbg:start(fun(_F,A) -> + Parent ! {Ref,A} + end), + Ref. + +%%-------------------------------------------------------------------- +queued_msgs(Ref, Conns) -> + queued_msgs(Ref, Conns, []). + +queued_msgs(Ref, Conns, Acc) -> + receive + {Ref, [_, C, _]=Msg} -> + case is_list(Conns) andalso lists:member(C, Conns) of + true -> + queued_msgs(Ref, [Msg|Acc]); + false -> + queued_msgs(Ref, Conns, Acc) + end + after 0 -> + lists:reverse(Acc) + end. + +%%-------------------------------------------------------------------- +stop_and_fail_if_unhandled_dbg_msgs(Ref, Conns, DaemonPid) -> + stop_and_fail_if_unhandled_dbg_msgs(queued_msgs(Ref,Conns), Ref, Conns, DaemonPid). + +stop_and_fail_if_unhandled_dbg_msgs(Msgs, _Ref, _Conns, DaemonPid) -> + ssh:stop_daemon(DaemonPid), + case Msgs of + [] -> + ok; + _ -> + ct:log("Unexpected messages:~n~p",[Msgs]), + ct:fail("Unexpected messages") + end. + +%%-------------------------------------------------------------------- +dbg_SKIP(Ref, Prefixes) -> + dbg_SKIP(Ref, Prefixes, []). + +dbg_SKIP(Ref, Prefixes, UnexpectedAcc) -> + receive + {Ref, [_, _C, Msg]=M} -> + case lists:any( + fun(Pfx) -> + lists:prefix(Pfx, Msg) + end, Prefixes) of + true -> + ct:log("Skip:~n~p", [M]), + dbg_SKIP(Ref, Prefixes, UnexpectedAcc); + false -> + dbg_SKIP(Ref, Prefixes, [Msg|UnexpectedAcc]) + end + after 0 -> + lists:reverse(UnexpectedAcc) + end. + diff --git a/lib/ssh/test/ssh_renegotiate_SUITE_data/ssh_host_dsa_key b/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_dsa_key index 51ab6fbd88..51ab6fbd88 100644 --- a/lib/ssh/test/ssh_renegotiate_SUITE_data/ssh_host_dsa_key +++ b/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_dsa_key diff --git a/lib/ssh/test/ssh_renegotiate_SUITE_data/ssh_host_dsa_key.pub b/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_dsa_key.pub index 4dbb1305b0..4dbb1305b0 100644 --- a/lib/ssh/test/ssh_renegotiate_SUITE_data/ssh_host_dsa_key.pub +++ b/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_dsa_key.pub diff --git a/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_ecdsa_key256 b/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_ecdsa_key256 new file mode 100644 index 0000000000..2979ea88ed --- /dev/null +++ b/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_ecdsa_key256 @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIMe4MDoit0t8RzSVPwkCBemQ9fhXL+xnTSAWISw8HNCioAoGCCqGSM49 +AwEHoUQDQgAEo2q7U3P6r0W5WGOLtM78UQtofM9UalEhiZeDdiyylsR/RR17Op0s +VPGSADLmzzgcucLEKy17j2S+oz42VUJy5A== +-----END EC PRIVATE KEY----- diff --git a/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_ecdsa_key256.pub b/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_ecdsa_key256.pub new file mode 100644 index 0000000000..85dc419345 --- /dev/null +++ b/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_ecdsa_key256.pub @@ -0,0 +1 @@ +ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKNqu1Nz+q9FuVhji7TO/FELaHzPVGpRIYmXg3YsspbEf0UdezqdLFTxkgAy5s84HLnCxCste49kvqM+NlVCcuQ= uabhnil@elxadlj3q32 diff --git a/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_ecdsa_key384 b/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_ecdsa_key384 new file mode 100644 index 0000000000..fb1a862ded --- /dev/null +++ b/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_ecdsa_key384 @@ -0,0 +1,6 @@ +-----BEGIN EC PRIVATE KEY----- +MIGkAgEBBDArxbDfh3p1okrD9wQw6jJ4d4DdlBPD5GqXE8bIeRJiK41Sh40LgvPw +mkqEDSXK++CgBwYFK4EEACKhZANiAAScl43Ih2lWTDKrSox5ve5uiTXil4smsup3 +CfS1XPjKxgBAmlfBim8izbdrT0BFdQzz2joduNMtpt61wO4rGs6jm0UP7Kim9PC7 +Hneb/99fIYopdMH5NMnk60zGO1uZ2vc= +-----END EC PRIVATE KEY----- diff --git a/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_ecdsa_key384.pub b/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_ecdsa_key384.pub new file mode 100644 index 0000000000..428d5fb7d7 --- /dev/null +++ b/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_ecdsa_key384.pub @@ -0,0 +1 @@ +ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBJyXjciHaVZMMqtKjHm97m6JNeKXiyay6ncJ9LVc+MrGAECaV8GKbyLNt2tPQEV1DPPaOh240y2m3rXA7isazqObRQ/sqKb08Lsed5v/318hiil0wfk0yeTrTMY7W5na9w== uabhnil@elxadlj3q32 diff --git a/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_ecdsa_key521 b/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_ecdsa_key521 new file mode 100644 index 0000000000..3e51ec2ecd --- /dev/null +++ b/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_ecdsa_key521 @@ -0,0 +1,7 @@ +-----BEGIN EC PRIVATE KEY----- +MIHcAgEBBEIB8O1BFkl2HQjQLRLonEZ97da/h39DMa9/0/hvPZWAI8gUPEQcHxRx +U7b09p3Zh+EBbMFq8+1ae9ds+ZTxE4WFSvKgBwYFK4EEACOhgYkDgYYABAAlWVjq +Bzg7Wt4gE6UNb1lRE2cnlmH2L/A5uo6qZRx5lPnSKOxEhxSb/Oay1+9d6KRdrh6/ +vlhd9SHDBhLcAPDvWgBnJIEj92Q3pXX4JtoitL0yl+SvvU+vUh966mzHShHzj8p5 +ccOgPkPNoA70yrpGzkIhPezpZOQdCaOXj/jFqNCTDg== +-----END EC PRIVATE KEY----- diff --git a/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_ecdsa_key521.pub b/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_ecdsa_key521.pub new file mode 100644 index 0000000000..017a29f4da --- /dev/null +++ b/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_ecdsa_key521.pub @@ -0,0 +1 @@ +ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAAlWVjqBzg7Wt4gE6UNb1lRE2cnlmH2L/A5uo6qZRx5lPnSKOxEhxSb/Oay1+9d6KRdrh6/vlhd9SHDBhLcAPDvWgBnJIEj92Q3pXX4JtoitL0yl+SvvU+vUh966mzHShHzj8p5ccOgPkPNoA70yrpGzkIhPezpZOQdCaOXj/jFqNCTDg== uabhnil@elxadlj3q32 diff --git a/lib/ssh/test/ssh_renegotiate_SUITE_data/ssh_host_rsa_key b/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_rsa_key index 79968bdd7d..79968bdd7d 100644 --- a/lib/ssh/test/ssh_renegotiate_SUITE_data/ssh_host_rsa_key +++ b/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_rsa_key diff --git a/lib/ssh/test/ssh_renegotiate_SUITE_data/ssh_host_rsa_key.pub b/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_rsa_key.pub index 75d2025c71..75d2025c71 100644 --- a/lib/ssh/test/ssh_renegotiate_SUITE_data/ssh_host_rsa_key.pub +++ b/lib/ssh/test/ssh_dbg_SUITE_data/ssh_host_rsa_key.pub diff --git a/lib/ssh/test/ssh_echo_server.erl b/lib/ssh/test/ssh_echo_server.erl index 5387d21efd..d03fe9543e 100644 --- a/lib/ssh/test/ssh_echo_server.erl +++ b/lib/ssh/test/ssh_echo_server.erl @@ -22,7 +22,7 @@ %%% Description: Example ssh server -module(ssh_echo_server). --behaviour(ssh_daemon_channel). +-behaviour(ssh_server_channel). -record(state, { n, id, diff --git a/lib/ssh/test/ssh_options_SUITE.erl b/lib/ssh/test/ssh_options_SUITE.erl index 12a85c40aa..86a8ac5aa8 100644 --- a/lib/ssh/test/ssh_options_SUITE.erl +++ b/lib/ssh/test/ssh_options_SUITE.erl @@ -1227,7 +1227,7 @@ max_sessions(Config, ParallelLogin, Connect0) when is_function(Connect0,2) -> [_|_] = Connections, %% Now try one more than alowed: - ct:log("Info Report might come here...",[]), + ct:pal("Info Report expected here (if not disabled) ...",[]), try Connect(Host,Port) of _ConnectionRef1 -> @@ -1235,8 +1235,7 @@ max_sessions(Config, ParallelLogin, Connect0) when is_function(Connect0,2) -> {fail,"Too many connections accepted"} catch error:{badmatch,{error,"Connection closed"}} -> - %% Step 2 ok: could not set up max_sessions+1 connections - %% This is expected + ct:log("Step 2 ok: could not set up too many connections. Good.",[]), %% Now stop one connection and try to open one more ok = ssh:close(hd(Connections)), try_to_connect(Connect, Host, Port, Pid) @@ -1249,16 +1248,15 @@ max_sessions(Config, ParallelLogin, Connect0) when is_function(Connect0,2) -> try_to_connect(Connect, Host, Port, Pid) -> - {ok,Tref} = timer:send_after(3000, timeout_no_connection), % give the supervisors some time... + {ok,Tref} = timer:send_after(30000, timeout_no_connection), % give the supervisors some time... try_to_connect(Connect, Host, Port, Pid, Tref, 1). % will take max 3300 ms after 11 tries try_to_connect(Connect, Host, Port, Pid, Tref, N) -> try Connect(Host,Port) of _ConnectionRef1 -> - %% Step 3 ok: could set up one more connection after killing one - %% Thats good. timer:cancel(Tref), + ct:log("Step 3 ok: could set up one more connection after killing one. Thats good.",[]), ssh:stop_daemon(Pid), receive % flush. timeout_no_connection -> ok diff --git a/lib/ssh/test/ssh_peername_sockname_server.erl b/lib/ssh/test/ssh_peername_sockname_server.erl index 8731d80f62..5e35fd6612 100644 --- a/lib/ssh/test/ssh_peername_sockname_server.erl +++ b/lib/ssh/test/ssh_peername_sockname_server.erl @@ -26,7 +26,7 @@ %% ssh connection. --behaviour(ssh_daemon_channel). +-behaviour(ssh_server_channel). -record(state, {}). -export([init/1, handle_msg/2, handle_ssh_msg/2, terminate/2]). diff --git a/lib/ssh/test/ssh_renegotiate_SUITE.erl b/lib/ssh/test/ssh_renegotiate_SUITE.erl deleted file mode 100644 index 74bbc291b2..0000000000 --- a/lib/ssh/test/ssh_renegotiate_SUITE.erl +++ /dev/null @@ -1,237 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2008-2016. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% - --module(ssh_renegotiate_SUITE). - --include_lib("common_test/include/ct.hrl"). --include("ssh_test_lib.hrl"). - -%% Note: This directive should only be used in test suites. --compile(export_all). - --define(REKEY_DATA_TMO, 65000). -%%-------------------------------------------------------------------- -%% Common Test interface functions ----------------------------------- -%%-------------------------------------------------------------------- - -suite() -> [{ct_hooks,[ts_install_cth]}, - {timetrap,{seconds,40}}]. - -all() -> [{group,default_algs}, - {group,aes_gcm} - ]. - -groups() -> [{default_algs, [], tests()}, - {aes_gcm, [], tests()} - ]. - -tests() -> [rekey, rekey_limit, renegotiate1, renegotiate2]. - -%%-------------------------------------------------------------------- -init_per_suite(Config) -> - ?CHECK_CRYPTO(Config). - -end_per_suite(_Config) -> - ssh:stop(). - -%%-------------------------------------------------------------------- -init_per_group(aes_gcm, Config) -> - case lists:member({client2server,['[email protected]']}, - ssh_transport:supported_algorithms(cipher)) of - true -> - [{preferred_algorithms, [{cipher,[{client2server,['[email protected]']}, - {server2client,['[email protected]']}]}]} - | Config]; - false -> - {skip, "aes_gcm not supported"} - end; -init_per_group(_, Config) -> - [{preferred_algorithms, ssh:default_algorithms()} | Config]. - - -end_per_group(_, Config) -> - Config. - -%%-------------------------------------------------------------------- -init_per_testcase(_TestCase, Config) -> - ssh:start(), - Config. - -end_per_testcase(_TestCase, _Config) -> - ssh:stop(), - ok. - -%%-------------------------------------------------------------------- -%% Test Cases -------------------------------------------------------- -%%-------------------------------------------------------------------- - -%%% Idle timeout test -rekey() -> [{timetrap,{seconds,90}}]. - -rekey(Config) -> - {Pid, Host, Port} = - ssh_test_lib:std_daemon(Config, - [{rekey_limit, 0}]), - ConnectionRef = - ssh_test_lib:std_connect(Config, Host, Port, - [{rekey_limit, 0}]), - Kex1 = ssh_test_lib:get_kex_init(ConnectionRef), - receive - after ?REKEY_DATA_TMO -> - %%By this time rekeying would have been done - Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), - false = (Kex2 == Kex1), - ssh:close(ConnectionRef), - ssh:stop_daemon(Pid) - end. - -%%-------------------------------------------------------------------- - -%%% Test rekeying by data volume - -rekey_limit() -> [{timetrap,{seconds,400}}]. - -rekey_limit(Config) -> - UserDir = proplists:get_value(priv_dir, Config), - DataFile = filename:join(UserDir, "rekey.data"), - - Algs = proplists:get_value(preferred_algorithms, Config), - {Pid, Host, Port} = ssh_test_lib:std_daemon(Config,[{max_random_length_padding,0}, - {preferred_algorithms,Algs}]), - - ConnectionRef = ssh_test_lib:std_connect(Config, Host, Port, [{rekey_limit, 6000}, - {max_random_length_padding,0}]), - {ok, SftpPid} = ssh_sftp:start_channel(ConnectionRef), - - Kex1 = ssh_test_lib:get_kex_init(ConnectionRef), - - timer:sleep(?REKEY_DATA_TMO), - Kex1 = ssh_test_lib:get_kex_init(ConnectionRef), - - Data = lists:duplicate(159000,1), - ok = ssh_sftp:write_file(SftpPid, DataFile, Data), - - timer:sleep(?REKEY_DATA_TMO), - Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), - - false = (Kex2 == Kex1), - - timer:sleep(?REKEY_DATA_TMO), - Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), - - ok = ssh_sftp:write_file(SftpPid, DataFile, "hi\n"), - - timer:sleep(?REKEY_DATA_TMO), - Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), - - false = (Kex2 == Kex1), - - timer:sleep(?REKEY_DATA_TMO), - Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), - - ssh_sftp:stop_channel(SftpPid), - ssh:close(ConnectionRef), - ssh:stop_daemon(Pid). - -%%-------------------------------------------------------------------- - -%%% Test rekeying with simulataneous send request - -renegotiate1(Config) -> - UserDir = proplists:get_value(priv_dir, Config), - DataFile = filename:join(UserDir, "renegotiate1.data"), - - Algs = proplists:get_value(preferred_algorithms, Config), - {Pid, Host, DPort} = ssh_test_lib:std_daemon(Config,[{max_random_length_padding,0}, - {preferred_algorithms,Algs}]), - - RPort = ssh_test_lib:inet_port(), - {ok,RelayPid} = ssh_relay:start_link({0,0,0,0}, RPort, Host, DPort), - - - ConnectionRef = ssh_test_lib:std_connect(Config, Host, RPort, [{max_random_length_padding,0}]), - {ok, SftpPid} = ssh_sftp:start_channel(ConnectionRef), - - Kex1 = ssh_test_lib:get_kex_init(ConnectionRef), - - {ok, Handle} = ssh_sftp:open(SftpPid, DataFile, [write]), - - ok = ssh_sftp:write(SftpPid, Handle, "hi\n"), - - ssh_relay:hold(RelayPid, rx, 20, 1000), - ssh_connection_handler:renegotiate(ConnectionRef), - spawn(fun() -> ok=ssh_sftp:write(SftpPid, Handle, "another hi\n") end), - - timer:sleep(2000), - - Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), - - false = (Kex2 == Kex1), - - ssh_relay:stop(RelayPid), - ssh_sftp:stop_channel(SftpPid), - ssh:close(ConnectionRef), - ssh:stop_daemon(Pid). - -%%-------------------------------------------------------------------- - -%%% Test rekeying with inflight messages from peer - -renegotiate2(Config) -> - UserDir = proplists:get_value(priv_dir, Config), - DataFile = filename:join(UserDir, "renegotiate2.data"), - - Algs = proplists:get_value(preferred_algorithms, Config), - {Pid, Host, DPort} = ssh_test_lib:std_daemon(Config,[{max_random_length_padding,0}, - {preferred_algorithms,Algs}]), - - RPort = ssh_test_lib:inet_port(), - {ok,RelayPid} = ssh_relay:start_link({0,0,0,0}, RPort, Host, DPort), - - ConnectionRef = ssh_test_lib:std_connect(Config, Host, RPort, [{max_random_length_padding,0}]), - {ok, SftpPid} = ssh_sftp:start_channel(ConnectionRef), - - Kex1 = ssh_test_lib:get_kex_init(ConnectionRef), - - {ok, Handle} = ssh_sftp:open(SftpPid, DataFile, [write]), - - ok = ssh_sftp:write(SftpPid, Handle, "hi\n"), - - ssh_relay:hold(RelayPid, rx, 20, infinity), - spawn(fun() -> ok=ssh_sftp:write(SftpPid, Handle, "another hi\n") end), - %% need a small pause here to ensure ssh_sftp:write is executed - ct:sleep(10), - ssh_connection_handler:renegotiate(ConnectionRef), - ssh_relay:release(RelayPid, rx), - - timer:sleep(2000), - - Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), - - false = (Kex2 == Kex1), - - ssh_relay:stop(RelayPid), - ssh_sftp:stop_channel(SftpPid), - ssh:close(ConnectionRef), - ssh:stop_daemon(Pid). - -%%-------------------------------------------------------------------- -%% Internal functions ------------------------------------------------ -%%-------------------------------------------------------------------- diff --git a/lib/ssh/test/ssh_renegotiate_SUITE_data/id_dsa b/lib/ssh/test/ssh_renegotiate_SUITE_data/id_dsa deleted file mode 100644 index d306f8b26e..0000000000 --- a/lib/ssh/test/ssh_renegotiate_SUITE_data/id_dsa +++ /dev/null @@ -1,13 +0,0 @@ ------BEGIN DSA PRIVATE KEY----- -MIIBvAIBAAKBgQDfi2flSTZZofwT4yQT0NikX/LGNT7UPeB/XEWe/xovEYCElfaQ -APFixXvEgXwoojmZ5kiQRKzLM39wBP0jPERLbnZXfOOD0PDnw0haMh7dD7XKVMod -/EigVgHf/qBdM2M8yz1s/rRF7n1UpLSypziKjkzCm7JoSQ2zbWIPdmBIXwIVAMgP -kpr7Sq3O7sHdb8D601DRjoExAoGAMOQxDfB2Fd8ouz6G96f/UOzRMI/Kdv8kYYKW -JIGY+pRYrLPyYzUeJznwZreOJgrczAX+luHnKFWJ2Dnk5CyeXk67Wsr7pJ/4MBMD -OKeIS0S8qoSBN8+Krp79fgA+yS3IfqbkJLtLu4EBaCX4mKQIX4++k44d4U5lc8pt -+9hlEI8CgYEAznKxx9kyC6bVo7LUYKaGhofRFt0SYFc5PVmT2VUGRs1R6+6DPD+e -uEO6IhFct7JFSRbP9p0JD4Uk+3zlZF+XX6b2PsZkeV8f/02xlNGUSmEzCSiNg1AX -Cy/WusYhul0MncWCHMcOZB5rIvU/aP5EJJtn3xrRaz6u0SThF6AnT34CFQC63czE -ZU8w8Q+H7z0j+a+70x2iAw== ------END DSA PRIVATE KEY----- - diff --git a/lib/ssh/test/ssh_renegotiate_SUITE_data/id_rsa b/lib/ssh/test/ssh_renegotiate_SUITE_data/id_rsa deleted file mode 100644 index 9d7e0dd5fb..0000000000 --- a/lib/ssh/test/ssh_renegotiate_SUITE_data/id_rsa +++ /dev/null @@ -1,15 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIICXAIBAAKBgQD1OET+3O/Bvj/dtjxDTXmj1oiJt4sIph5kGy0RfjoPrZfaS+CU -DhakCmS6t2ivxWFgtpKWaoGMZMJqWj6F6ZsumyFl3FPBtujwY/35cgifrI9Ns4Tl -zR1uuengNBmV+WRQ5cd9F2qS6Z8aDQihzt0r8JUqLcK+VQbrmNzboCCQQwIDAQAB -AoGAPQEyqPTt8JUT7mRXuaacjFXiweAXhp9NEDpyi9eLOjtFe9lElZCrsUOkq47V -TGUeRKEm9qSodfTbKPoqc8YaBJGJPhUaTAcha+7QcDdfHBvIsgxvU7ePVnlpXRp3 -CCUEMPhlnx6xBoTYP+fRU0e3+xJIPVyVCqX1jAdUMkzfRoECQQD6ux7B1QJAIWyK -SGkbDUbBilNmzCFNgIpOP6PA+bwfi5d16diTpra5AX09keQABAo/KaP1PdV8Vg0p -z4P3A7G3AkEA+l+AKG6m0kQTTBMJDqOdVPYwe+5GxunMaqmhokpEbuGsrZBl5Dvd -WpcBjR7jmenrhKZRIuA+Fz5HPo/UQJPl1QJBAKxstDkeED8j/S2XoFhPKAJ+6t39 -sUVICVTIZQeXdmzHJXCcUSkw8+WEhakqw/3SyW0oaK2FSWQJFWJUZ+8eJj8CQEh3 -xeduB5kKnS9CvzdeghZqX6QvVosSdtlUmfUYW/BgH5PpHKTP8wTaeld3XldZTpMJ -dKiMkUw2+XYROVUrubUCQD+Na1LhULlpn4ISEtIEfqpdlUhxDgO15Wg8USmsng+x -ICliVOSQtwaZjm8kwaFt0W7XnpnDxbRs37vIEbIMWak= ------END RSA PRIVATE KEY----- diff --git a/lib/ssh/test/ssh_sup_SUITE.erl b/lib/ssh/test/ssh_sup_SUITE.erl index 1df55834b1..b81f66948d 100644 --- a/lib/ssh/test/ssh_sup_SUITE.erl +++ b/lib/ssh/test/ssh_sup_SUITE.erl @@ -201,8 +201,6 @@ killed_acceptor_restarts(Config) -> Port2 = ssh_test_lib:daemon_port(DaemonPid2), true = (Port /= Port2), - ct:log("~s",[lists:flatten(ssh_info:string())]), - {ok,[{AccPid,ListenAddr,Port}]} = acceptor_pid(DaemonPid), {ok,[{AccPid2,ListenAddr,Port2}]} = acceptor_pid(DaemonPid2), @@ -216,23 +214,34 @@ killed_acceptor_restarts(Config) -> {user_dir, UserDir}]), [{client_version,_}] = ssh:connection_info(C1,[client_version]), + ct:log("~s",[lists:flatten(ssh_info:string())]), + %% Make acceptor restart: exit(AccPid, kill), ?wait_match(undefined, process_info(AccPid)), - %% Check it is a new acceptor: + %% Check it is a new acceptor and wait if it is not: ?wait_match({ok,[{AccPid1,ListenAddr,Port}]}, AccPid1=/=AccPid, acceptor_pid(DaemonPid), AccPid1, 500, 30), - AccPid1 =/= AccPid2, + + true = (AccPid1 =/= AccPid2), %% Connect second client and check it is alive: - {ok,C2} = ssh:connect("localhost", Port, [{silently_accept_hosts, true}, - {user_interaction, false}, - {user, ?USER}, - {password, ?PASSWD}, - {user_dir, UserDir}]), + C2 = + case ssh:connect("localhost", Port, [{silently_accept_hosts, true}, + {user_interaction, false}, + {user, ?USER}, + {password, ?PASSWD}, + {user_dir, UserDir}]) of + {ok,_C2} -> + _C2; + _Other -> + ct:log("new connect failed: ~p~n~n~s",[_Other,lists:flatten(ssh_info:string())]), + ct:fail("Re-connect failed!", []) + end, + [{client_version,_}] = ssh:connection_info(C2,[client_version]), ct:log("~s",[lists:flatten(ssh_info:string())]), @@ -247,8 +256,8 @@ killed_acceptor_restarts(Config) -> ok = ssh:stop_daemon(DaemonPid), ?wait_match(undefined, process_info(DaemonPid), 1000, 30), - {error,closed} = ssh:connection_info(C1,[client_version]), - {error,closed} = ssh:connection_info(C2,[client_version]). + ?wait_match({error,closed}, ssh:connection_info(C1,[client_version]), 1000, 5), + ?wait_match({error,closed}, ssh:connection_info(C2,[client_version]), 1000, 5). %%------------------------------------------------------------------------- shell_channel_tree(Config) -> @@ -281,7 +290,7 @@ shell_channel_tree(Config) -> {ok, ChannelId0} = ssh_connection:session_channel(ConnectionRef, infinity), ok = ssh_connection:shell(ConnectionRef,ChannelId0), - ?wait_match([{_, GroupPid,worker,[ssh_channel]}], + ?wait_match([{_, GroupPid,worker,[ssh_server_channel]}], supervisor:which_children(ChannelSup), [GroupPid]), {links,GroupLinks} = erlang:process_info(GroupPid, links), @@ -330,9 +339,9 @@ chk_empty_con_daemon(Daemon) -> ?wait_match([{{server,ssh_connection_sup, _,_}, ConnectionSup, supervisor, [ssh_connection_sup]}, - {{server,ssh_channel_sup,_ ,_}, + {{server,ssh_server_channel_sup,_ ,_}, ChannelSup,supervisor, - [ssh_channel_sup]}], + [ssh_server_channel_sup]}], supervisor:which_children(SubSysSup), [ConnectionSup,ChannelSup]), ?wait_match([{{ssh_acceptor_sup,_,_,_},_,worker,[ssh_acceptor]}], @@ -363,9 +372,9 @@ check_sshd_system_tree(Daemon, Config) -> ?wait_match([{{server,ssh_connection_sup, _,_}, ConnectionSup, supervisor, [ssh_connection_sup]}, - {{server,ssh_channel_sup,_ ,_}, + {{server,ssh_server_channel_sup,_ ,_}, ChannelSup,supervisor, - [ssh_channel_sup]}], + [ssh_server_channel_sup]}], supervisor:which_children(SubSysSup), [ConnectionSup,ChannelSup]), @@ -379,7 +388,7 @@ check_sshd_system_tree(Daemon, Config) -> ssh_sftp:start_channel(Client), - ?wait_match([{_, _,worker,[ssh_channel]}], + ?wait_match([{_, _,worker,[ssh_server_channel]}], supervisor:which_children(ChannelSup)), ssh:close(Client). diff --git a/lib/ssh/vsn.mk b/lib/ssh/vsn.mk index 99c5cbd346..f327d2ec11 100644 --- a/lib/ssh/vsn.mk +++ b/lib/ssh/vsn.mk @@ -1,4 +1,4 @@ #-*-makefile-*- ; force emacs to enter makefile-mode -SSH_VSN = 4.6.6 +SSH_VSN = 4.6.8 APP_VSN = "ssh-$(SSH_VSN)" diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index 8c1b1541c7..029f29cdb3 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -197,6 +197,18 @@ | sect193r1 | sect193r2 | secp192k1 | secp192r1 | sect163k1 | sect163r1 | sect163r2 | secp160k1 | secp160r1 | secp160r2</c></p></item> + <tag><c>hello_extensions() =</c></tag> + <item><p><c>#{renegotiation_info => + signature_algs => [{hash(), ecsda| rsa| dsa}] | undefined + alpn => binary() | undefined, + next_protocol_negotiation, + srp => string() | undefined, + ec_point_formats , + elliptic_curves = [oid] | undefined + sni = string()} + }</c></p></item> + + </taglist> </section> @@ -211,8 +223,16 @@ <tag><c>{protocol, tls | dtls}</c></tag> <item><p>Choose TLS or DTLS protocol for the transport layer security. Defaults to <c>tls</c> Introduced in OTP 20, DTLS support is considered - experimental in this release. DTLS over other transports than UDP are not yet supported.</p></item> - + experimental in this release. Other transports than UDP are not yet supported.</p></item> + + <tag><c>{handshake, hello | full}</c></tag> + <item><p> Defaults to <c>full</c>. If hello is specified the handshake will + pause after the hello message and give the user a possibility make decisions + based on hello extensions before continuing or aborting the handshake by calling + <seealso marker="#handshake_continue-3"> handshake_continue/3</seealso> or + <seealso marker="#handshake_cancel-1"> handshake_cancel/1</seealso> + </p></item> + <tag><c>{cert, public_key:der_encoded()}</c></tag> <item><p>The DER-encoded users certificate. If this option is supplied, it overrides option <c>certfile</c>.</p></item> @@ -919,15 +939,16 @@ fun(srp, Username :: string(), UserState :: term()) -> <func> <name>connect(Socket, SslOptions) -> </name> - <name>connect(Socket, SslOptions, Timeout) -> {ok, TLSSocket} + <name>connect(Socket, SslOptions, Timeout) -> {ok, TLSSocket} | {ok, TLSSocket, Ext} | {error, Reason}</name> <fsummary>Upgrades a <c>gen_tcp</c>, or equivalent, connected socket to an TLS socket.</fsummary> <type> <v>Socket = socket()</v> - <v>SslOptions = [ssl_option()]</v> + <v>SslOptions = [{handshake, hello| full} | ssl_option()]</v> <v>Timeout = integer() | infinity</v> <v>TLSSocket = sslsocket()</v> + <v>Ext = hello_extensions()</v> <v>Reason = term()</v> </type> <desc><p>Upgrades a <c>gen_tcp</c>, or equivalent, @@ -938,14 +959,25 @@ fun(srp, Username :: string(), UserState :: term()) -> the option <c>server_name_indication</c> shall also be specified, if it is not no Server Name Indication extension will be sent, and <seealso marker="public_key:public_key#pkix_verify_hostname-2">public_key:pkix_verify_hostname/2</seealso> - will be called with the IP-address of the connection as <c>ReferenceID</c>, which is proably not what you want.</p></note> + will be called with the IP-address of the connection as <c>ReferenceID</c>, which is proably not what you want.</p> + </note> + + <p> If the option <c>{handshake, hello}</c> is used the + handshake is paused after receiving the server hello message + and the success response is <c>{ok, TLSSocket, Ext}</c> + instead of <c>{ok, TLSSocket}</c>. Thereafter the handshake is continued or + canceled by calling <seealso marker="#handshake_continue-3"> + <c>handshake_continue/3</c></seealso> or <seealso + marker="#handshake_cancel-1"><c>handshake_cancel/1</c></seealso>. + </p> + </desc> </func> <func> <name>connect(Host, Port, Options) -></name> <name>connect(Host, Port, Options, Timeout) -> - {ok, SslSocket} | {error, Reason}</name> + {ok, SslSocket}| {ok, TLSSocket, Ext} | {error, Reason}</name> <fsummary>Opens an TLS/DTLS connection to <c>Host</c>, <c>Port</c>.</fsummary> <type> <v>Host = host()</v> @@ -972,6 +1004,16 @@ fun(srp, Username :: string(), UserState :: term()) -> <c>dns_id</c> will be assumed with a fallback to <c>ip</c> if that fails. </p> <note><p>According to good practices certificates should not use IP-addresses as "server names". It would be very surprising if this happen outside a closed network. </p></note> + + + <p> If the option <c>{handshake, hello}</c> is used the + handshake is paused after receiving the server hello message + and the success response is <c>{ok, TLSSocket, Ext}</c> + instead of <c>{ok, TLSSocket}</c>. Thereafter the handshake is continued or + canceled by calling <seealso marker="#handshake_continue-3"> + <c>handshake_continue/3</c></seealso> or <seealso + marker="#handshake_cancel-1"><c>handshake_cancel/1</c></seealso>. + </p> </desc> </func> @@ -1113,6 +1155,85 @@ fun(srp, Username :: string(), UserState :: term()) -> </func> <func> + <name>handshake(Socket) -> </name> + <name>handshake(Socket, Timeout) -> {ok, Socket} | {error, Reason}</name> + <fsummary>Performs server-side SSL/TLS handshake.</fsummary> + <type> + <v>Socket = sslsocket()</v> + <v>Timeout = integer()</v> + <v>Reason = term()</v> + </type> + <desc> + <p>Performs the SSL/TLS/DTLS server-side handshake.</p> + <p><c>Socket</c> is a socket as returned by + <seealso marker="#transport_accept-2">ssl:transport_accept/[1,2]</seealso>. + </p> + </desc> + </func> + + <func> + <name>handshake(Socket, SslOptions) -> </name> + <name>handshake(Socket, SslOptions, Timeout) -> {ok, Socket} | {ok, Socket, Ext} | {error, Reason}</name> + <fsummary>Performs server-side SSL/TLS/DTLS handshake.</fsummary> + <type> + <v>Socket = socket() | sslsocket() </v> + <v>Ext = hello_extensions()</v> + <v>SslOptions = [{handshake, hello| full} | ssl_option()]</v> + <v>Timeout = integer()</v> + <v>Reason = term()</v> + </type> + <desc> + <p>If <c>Socket</c> is a ordinary <c>socket()</c>: upgrades a <c>gen_tcp</c>, + or equivalent, socket to an SSL socket, that is, performs + the SSL/TLS server-side handshake and returns the SSL socket.</p> + + <warning><p>The Socket shall be in passive mode ({active, + false}) before calling this function or the handshake can fail + due to a race condition.</p></warning> + + <p>If <c>Socket</c> is an <c>sslsocket()</c>: provides extra SSL/TLS/DTLS + options to those specified in + <seealso marker="#listen-2">ssl:listen/2 </seealso> and then performs + the SSL/TLS/DTLS handshake.</p> + + <p> + If option <c>{handshake, hello}</c> is specified the handshake is + paused after receiving the client hello message and the + sucess response is <c>{ok, TLSSocket, Ext}</c> instead of <c>{ok, + TLSSocket}</c>. Thereafter the handshake is continued or + canceled by calling <seealso marker="#handshake_continue-3"> + <c>handshake_continue/3</c></seealso> or <seealso + marker="#handshake_cancel-1"><c>handshake_cancel/1</c></seealso>. + </p> + </desc> + </func> + + <func> + <name>handshake_cancel(Socket) -> ok </name> + <fsummary>Cancel handshake with a fatal alert</fsummary> + <type> + <v>Socket = sslsocket()</v> + </type> + <desc> + <p>Cancel the handshake with a fatal <c>USER_CANCELED</c> alert.</p> + </desc> + </func> + + <func> + <name>handshake_continue(Socket, SSLOptions, Timeout) -> {ok, Socket} | {error, Reason}</name> + <fsummary>Continue the SSL/TLS handshake.</fsummary> + <type> + <v>Socket = sslsocket()</v> + <v>SslOptions = [ssl_option()]</v> + <v>Timeout = integer()</v> + <v>Reason = term()</v> + </type> + <desc> + <p>Continue the SSL/TLS handshake possiby with new, additional or changed options.</p> + </desc> + </func> + + <func> <name>listen(Port, Options) -> {ok, ListenSocket} | {error, Reason}</name> <fsummary>Creates an SSL listen socket.</fsummary> diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl index 7bc7fc3fc6..220da71123 100644 --- a/lib/ssl/src/dtls_connection.erl +++ b/lib/ssl/src/dtls_connection.erl @@ -55,7 +55,7 @@ %% gen_statem state functions -export([init/3, error/3, downgrade/3, %% Initiation and take down states - hello/3, certify/3, cipher/3, abbreviated/3, %% Handshake states + hello/3, user_hello/3, certify/3, cipher/3, abbreviated/3, %% Handshake states connection/3]). %% gen_statem callbacks -export([callback_mode/0, terminate/3, code_change/4, format_status/2]). @@ -73,8 +73,7 @@ start_fsm(Role, Host, Port, Socket, {#ssl_options{erl_dist = false},_, Tracker} {ok, Pid} = dtls_connection_sup:start_child([Role, Host, Port, Socket, Opts, User, CbInfo]), {ok, SslSocket} = ssl_connection:socket_control(?MODULE, Socket, Pid, CbModule, Tracker), - ok = ssl_connection:handshake(SslSocket, Timeout), - {ok, SslSocket} + ssl_connection:handshake(SslSocket, Timeout) catch error:{badmatch, {error, _} = Error} -> Error @@ -492,10 +491,11 @@ hello(enter, _, #state{role = client} = State0) -> {State, Actions} = handle_flight_timer(State0), {keep_state, State, Actions}; hello(internal, #client_hello{cookie = <<>>, - client_version = Version} = Hello, #state{role = server, - transport_cb = Transport, - socket = Socket, - protocol_specific = #{current_cookie_secret := Secret}} = State0) -> + client_version = Version} = Hello, + #state{role = server, + transport_cb = Transport, + socket = Socket, + protocol_specific = #{current_cookie_secret := Secret}} = State0) -> {ok, {IP, Port}} = dtls_socket:peername(Transport, Socket), Cookie = dtls_handshake:cookie(Secret, IP, Port, Hello), %% FROM RFC 6347 regarding HelloVerifyRequest message: @@ -509,24 +509,6 @@ hello(internal, #client_hello{cookie = <<>>, {State2, Actions} = send_handshake(VerifyRequest, State1), {Record, State} = next_record(State2), next_event(?FUNCTION_NAME, Record, State#state{tls_handshake_history = ssl_handshake:init_handshake_history()}, Actions); -hello(internal, #client_hello{cookie = Cookie} = Hello, #state{role = server, - transport_cb = Transport, - socket = Socket, - protocol_specific = #{current_cookie_secret := Secret, - previous_cookie_secret := PSecret}} = State0) -> - {ok, {IP, Port}} = dtls_socket:peername(Transport, Socket), - case dtls_handshake:cookie(Secret, IP, Port, Hello) of - Cookie -> - handle_client_hello(Hello, State0); - _ -> - case dtls_handshake:cookie(PSecret, IP, Port, Hello) of - Cookie -> - handle_client_hello(Hello, State0); - _ -> - %% Handle bad cookie as new cookie request RFC 6347 4.1.2 - hello(internal, Hello#client_hello{cookie = <<>>}, State0) - end - end; hello(internal, #hello_verify_request{cookie = Cookie}, #state{role = client, host = Host, port = Port, ssl_options = SslOpts, @@ -551,6 +533,34 @@ hello(internal, #hello_verify_request{cookie = Cookie}, #state{role = client, Hello#client_hello.session_id}}, {Record, State} = next_record(State3), next_event(?FUNCTION_NAME, Record, State, Actions); +hello(internal, #client_hello{extensions = Extensions} = Hello, #state{ssl_options = #ssl_options{handshake = hello}, + start_or_recv_from = From} = State) -> + {next_state, user_hello, State#state{start_or_recv_from = undefined, + hello = Hello}, + [{reply, From, {ok, ssl_connection:map_extensions(Extensions)}}]}; +hello(internal, #server_hello{extensions = Extensions} = Hello, #state{ssl_options = #ssl_options{handshake = hello}, + start_or_recv_from = From} = State) -> + {next_state, user_hello, State#state{start_or_recv_from = undefined, + hello = Hello}, + [{reply, From, {ok, ssl_connection:map_extensions(Extensions)}}]}; +hello(internal, #client_hello{cookie = Cookie} = Hello, #state{role = server, + transport_cb = Transport, + socket = Socket, + protocol_specific = #{current_cookie_secret := Secret, + previous_cookie_secret := PSecret}} = State0) -> + {ok, {IP, Port}} = dtls_socket:peername(Transport, Socket), + case dtls_handshake:cookie(Secret, IP, Port, Hello) of + Cookie -> + handle_client_hello(Hello, State0); + _ -> + case dtls_handshake:cookie(PSecret, IP, Port, Hello) of + Cookie -> + handle_client_hello(Hello, State0); + _ -> + %% Handle bad cookie as new cookie request RFC 6347 4.1.2 + hello(internal, Hello#client_hello{cookie = <<>>}, State0) + end + end; hello(internal, #server_hello{} = Hello, #state{connection_states = ConnectionStates0, negotiated_version = ReqVersion, @@ -577,6 +587,11 @@ hello(state_timeout, Event, State) -> hello(Type, Event, State) -> gen_handshake(?FUNCTION_NAME, Type, Event, State). +user_hello(enter, _, State) -> + {keep_state, State}; +user_hello(Type, Event, State) -> + gen_handshake(?FUNCTION_NAME, Type, Event, State). + %%-------------------------------------------------------------------- -spec abbreviated(gen_statem:event_type(), term(), #state{}) -> gen_statem:state_function_result(). diff --git a/lib/ssl/src/dtls_handshake.erl b/lib/ssl/src/dtls_handshake.erl index 6071eece13..1a415a5f76 100644 --- a/lib/ssl/src/dtls_handshake.erl +++ b/lib/ssl/src/dtls_handshake.erl @@ -174,7 +174,9 @@ handle_client_hello(Version, signature_algs = ClientHashSigns} = HelloExt}, #ssl_options{versions = Versions, - signature_algs = SupportedHashSigns} = SslOpts, + signature_algs = SupportedHashSigns, + eccs = SupportedECCs, + honor_ecc_order = ECCOrder} = SslOpts, {Port, Session0, Cache, CacheCb, ConnectionStates0, Cert, _}, Renegotiation) -> case dtls_record:is_acceptable_version(Version, Versions) of @@ -182,7 +184,7 @@ handle_client_hello(Version, TLSVersion = dtls_v1:corresponding_tls_version(Version), AvailableHashSigns = ssl_handshake:available_signature_algs( ClientHashSigns, SupportedHashSigns, Cert,TLSVersion), - ECCCurve = ssl_handshake:select_curve(Curves, ssl_handshake:supported_ecc(TLSVersion)), + ECCCurve = ssl_handshake:select_curve(Curves, SupportedECCs, ECCOrder), {Type, #session{cipher_suite = CipherSuite} = Session1} = ssl_handshake:select_session(SugesstedId, CipherSuites, AvailableHashSigns, Compressions, diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index 4efd13a6fa..5b6d92ebf4 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -32,7 +32,9 @@ %% Socket handling -export([connect/3, connect/2, connect/4, listen/2, transport_accept/1, transport_accept/2, - ssl_accept/1, ssl_accept/2, ssl_accept/3, + handshake/1, handshake/2, handshake/3, + handshake_continue/3, handshake_cancel/1, + ssl_accept/1, ssl_accept/2, ssl_accept/3, controlling_process/2, peername/1, peercert/1, sockname/1, close/1, close/2, shutdown/2, recv/2, recv/3, send/2, getopts/2, setopts/2, getstat/1, getstat/2 @@ -45,7 +47,7 @@ format_error/1, renegotiate/1, prf/5, negotiated_protocol/1, connection_information/1, connection_information/2]). %% Misc --export([handle_options/2, tls_version/1]). +-export([handle_options/2, tls_version/1, new_ssl_options/3]). -include("ssl_api.hrl"). -include("ssl_internal.hrl"). @@ -170,23 +172,54 @@ transport_accept(#sslsocket{pid = {ListenSocket, ok | {ok, #sslsocket{}} | {error, reason()}. -spec ssl_accept(#sslsocket{} | port(), [ssl_option()] | [ssl_option()| transport_option()], timeout()) -> - {ok, #sslsocket{}} | {error, reason()}. + ok | {ok, #sslsocket{}} | {error, reason()}. %% %% Description: Performs accept on an ssl listen socket. e.i. performs %% ssl handshake. %%-------------------------------------------------------------------- ssl_accept(ListenSocket) -> - ssl_accept(ListenSocket, infinity). + ssl_accept(ListenSocket, [], infinity). +ssl_accept(Socket, Timeout) when (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> + ssl_accept(Socket, [], Timeout); +ssl_accept(ListenSocket, SslOptions) when is_port(ListenSocket) -> + ssl_accept(ListenSocket, SslOptions, infinity); +ssl_accept(Socket, Timeout) -> + ssl_accept(Socket, [], Timeout). +ssl_accept(Socket, SslOptions, Timeout) when is_port(Socket) -> + handshake(Socket, SslOptions, Timeout); +ssl_accept(Socket, SslOptions, Timeout) -> + case handshake(Socket, SslOptions, Timeout) of + {ok, _} -> + ok; + Error -> + Error + end. +%%-------------------------------------------------------------------- +-spec handshake(#sslsocket{}) -> {ok, #sslsocket{}} | {error, reason()}. +-spec handshake(#sslsocket{} | port(), timeout()| [ssl_option() + | transport_option()]) -> + {ok, #sslsocket{}} | {error, reason()}. -ssl_accept(#sslsocket{} = Socket, Timeout) when (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> +-spec handshake(#sslsocket{} | port(), [ssl_option()] | [ssl_option()| transport_option()], timeout()) -> + {ok, #sslsocket{}} | {error, reason()}. +%% +%% Description: Performs accept on an ssl listen socket. e.i. performs +%% ssl handshake. +%%-------------------------------------------------------------------- +handshake(ListenSocket) -> + handshake(ListenSocket, infinity). + +handshake(#sslsocket{} = Socket, Timeout) when (is_integer(Timeout) andalso Timeout >= 0) or + (Timeout == infinity) -> ssl_connection:handshake(Socket, Timeout); -ssl_accept(ListenSocket, SslOptions) when is_port(ListenSocket) -> - ssl_accept(ListenSocket, SslOptions, infinity). +handshake(ListenSocket, SslOptions) when is_port(ListenSocket) -> + handshake(ListenSocket, SslOptions, infinity). -ssl_accept(#sslsocket{} = Socket, [], Timeout) when (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity)-> - ssl_accept(Socket, Timeout); -ssl_accept(#sslsocket{fd = {_, _, _, Tracker}} = Socket, SslOpts, Timeout) when +handshake(#sslsocket{} = Socket, [], Timeout) when (is_integer(Timeout) andalso Timeout >= 0) or + (Timeout == infinity)-> + handshake(Socket, Timeout); +handshake(#sslsocket{fd = {_, _, _, Tracker}} = Socket, SslOpts, Timeout) when (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity)-> try {ok, EmOpts, _} = tls_socket:get_all_opts(Tracker), @@ -195,7 +228,7 @@ ssl_accept(#sslsocket{fd = {_, _, _, Tracker}} = Socket, SslOpts, Timeout) when catch Error = {error, _Reason} -> Error end; -ssl_accept(#sslsocket{pid = Pid, fd = {_, _, _}} = Socket, SslOpts, Timeout) when +handshake(#sslsocket{pid = Pid, fd = {_, _, _}} = Socket, SslOpts, Timeout) when (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity)-> try {ok, EmOpts, _} = dtls_udp_listener:get_all_opts(Pid), @@ -204,8 +237,8 @@ ssl_accept(#sslsocket{pid = Pid, fd = {_, _, _}} = Socket, SslOpts, Timeout) whe catch Error = {error, _Reason} -> Error end; -ssl_accept(Socket, SslOptions, Timeout) when is_port(Socket), - (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> +handshake(Socket, SslOptions, Timeout) when is_port(Socket), + (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> {Transport,_,_,_} = proplists:get_value(cb_info, SslOptions, {gen_tcp, tcp, tcp_closed, tcp_error}), EmulatedOptions = tls_socket:emulated_options(), @@ -215,13 +248,31 @@ ssl_accept(Socket, SslOptions, Timeout) when is_port(Socket), {ok, #config{transport_info = CbInfo, ssl = SslOpts, emulated = EmOpts}} -> ok = tls_socket:setopts(Transport, Socket, tls_socket:internal_inet_values()), {ok, Port} = tls_socket:port(Transport, Socket), - ssl_connection:ssl_accept(ConnetionCb, Port, Socket, - {SslOpts, - tls_socket:emulated_socket_options(EmOpts, #socket_options{}), undefined}, - self(), CbInfo, Timeout) + ssl_connection:handshake(ConnetionCb, Port, Socket, + {SslOpts, + tls_socket:emulated_socket_options(EmOpts, #socket_options{}), undefined}, + self(), CbInfo, Timeout) catch Error = {error, _Reason} -> Error end. + +%%-------------------------------------------------------------------- +-spec handshake_continue(#sslsocket{}, [ssl_option()], timeout()) -> + {ok, #sslsocket{}} | {error, reason()}. +%% +%% +%% Description: Continues the handshke possible with newly supplied options. +%%-------------------------------------------------------------------- +handshake_continue(Socket, SSLOptions, Timeout) -> + ssl_connection:handshake_continue(Socket, SSLOptions, Timeout). +%%-------------------------------------------------------------------- +-spec handshake_cancel(#sslsocket{}) -> term(). +%% +%% Description: Cancels the handshakes sending a close alert. +%%-------------------------------------------------------------------- +handshake_cancel(Socket) -> + ssl_connection:handshake_cancel(Socket). + %%-------------------------------------------------------------------- -spec close(#sslsocket{}) -> term(). %% @@ -476,8 +527,9 @@ eccs() -> eccs_filter_supported(Curves). %%-------------------------------------------------------------------- --spec eccs(tls_record:tls_version() | tls_record:tls_atom_version()) -> - tls_v1:curves(). +-spec eccs(tls_record:tls_version() | tls_record:tls_atom_version() | + dtls_record:dtls_version() | dtls_record:dtls_atom_version()) -> + tls_v1:curves(). %% Description: returns the curves supported for a given version of %% ssl/tls. %%-------------------------------------------------------------------- @@ -486,13 +538,16 @@ eccs({3,0}) -> eccs({3,_}) -> Curves = tls_v1:ecc_curves(all), eccs_filter_supported(Curves); -eccs({_,_} = DTLSVersion) -> - eccs(dtls_v1:corresponding_tls_version(DTLSVersion)); -eccs(DTLSAtomVersion) when DTLSAtomVersion == 'dtlsv1'; - DTLSAtomVersion == 'dtlsv2' -> - eccs(dtls_record:protocol_version(DTLSAtomVersion)); -eccs(AtomVersion) when is_atom(AtomVersion) -> - eccs(tls_record:protocol_version(AtomVersion)). +eccs({254,_} = Version) -> + eccs(dtls_v1:corresponding_tls_version(Version)); +eccs(Version) when Version == 'tlsv1.2'; + Version == 'tlsv1.1'; + Version == tlsv1; + Version == sslv3 -> + eccs(tls_record:protocol_version(Version)); +eccs(Version) when Version == 'dtlsv1.2'; + Version == 'dtlsv1'-> + eccs(dtls_v1:corresponding_tls_version(dtls_record:protocol_version(Version))). eccs_filter_supported(Curves) -> CryptoCurves = crypto:ec_curves(), @@ -881,7 +936,8 @@ handle_options(Opts0, Role, Host) -> client, Role), crl_check = handle_option(crl_check, Opts, false), crl_cache = handle_option(crl_cache, Opts, {ssl_crl_cache, {internal, []}}), - max_handshake_size = handle_option(max_handshake_size, Opts, ?DEFAULT_MAX_HANDSHAKE_SIZE) + max_handshake_size = handle_option(max_handshake_size, Opts, ?DEFAULT_MAX_HANDSHAKE_SIZE), + handshake = handle_option(handshake, Opts, full) }, CbInfo = proplists:get_value(cb_info, Opts, default_cb_info(Protocol)), @@ -897,8 +953,7 @@ handle_options(Opts0, Role, Host) -> client_preferred_next_protocols, log_alert, server_name_indication, honor_cipher_order, padding_check, crl_check, crl_cache, fallback, signature_algs, eccs, honor_ecc_order, beast_mitigation, - max_handshake_size], - + max_handshake_size, handshake], SockOpts = lists:foldl(fun(Key, PropList) -> proplists:delete(Key, PropList) end, Opts, SslOptions), @@ -910,8 +965,6 @@ handle_options(Opts0, Role, Host) -> inet_user = Sock, transport_info = CbInfo, connection_cb = ConnetionCb }}. - - handle_option(OptionName, Opts, Default, Role, Role) -> handle_option(OptionName, Opts, Default); handle_option(_, _, undefined = Value, _, _) -> @@ -1139,6 +1192,10 @@ validate_option(protocol, Value = tls) -> Value; validate_option(protocol, Value = dtls) -> Value; +validate_option(handshake, hello = Value) -> + Value; +validate_option(handshake, full = Value) -> + Value; validate_option(Opt, Value) -> throw({error, {options, {Opt, Value}}}). diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index f493c93726..3f8c1f97f9 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -37,7 +37,9 @@ -include_lib("public_key/include/public_key.hrl"). %% Setup --export([connect/8, ssl_accept/7, handshake/2, handshake/3, + +-export([connect/8, handshake/7, handshake/2, handshake/3, + handshake_continue/3, handshake_cancel/1, socket_control/4, socket_control/5, start_or_recv_cancel_timer/2]). %% User Events @@ -57,11 +59,11 @@ %% Help functions for tls|dtls_connection.erl -export([handle_session/7, ssl_config/3, - prepare_connection/2, hibernate_after/3]). + prepare_connection/2, hibernate_after/3, map_extensions/1]). %% General gen_statem state functions with extra callback argument %% to determine if it is an SSL/TLS or DTLS gen_statem machine --export([init/4, error/4, hello/4, abbreviated/4, certify/4, cipher/4, +-export([init/4, error/4, hello/4, user_hello/4, abbreviated/4, certify/4, cipher/4, connection/4, death_row/4, downgrade/4]). %% gen_statem callbacks @@ -93,7 +95,7 @@ connect(Connection, Host, Port, Socket, Options, User, CbInfo, Timeout) -> {error, ssl_not_started} end. %%-------------------------------------------------------------------- --spec ssl_accept(tls_connection | dtls_connection, +-spec handshake(tls_connection | dtls_connection, inet:port_number(), port(), {#ssl_options{}, #socket_options{}, undefined | pid()}, pid(), tuple(), timeout()) -> @@ -102,7 +104,7 @@ connect(Connection, Host, Port, Socket, Options, User, CbInfo, Timeout) -> %% Description: Performs accept on an ssl listen socket. e.i. performs %% ssl handshake. %%-------------------------------------------------------------------- -ssl_accept(Connection, Port, Socket, Opts, User, CbInfo, Timeout) -> +handshake(Connection, Port, Socket, Opts, User, CbInfo, Timeout) -> try Connection:start_fsm(server, "localhost", Port, Socket, Opts, User, CbInfo, Timeout) catch @@ -111,32 +113,60 @@ ssl_accept(Connection, Port, Socket, Opts, User, CbInfo, Timeout) -> end. %%-------------------------------------------------------------------- --spec handshake(#sslsocket{}, timeout()) -> ok | {error, reason()}. +-spec handshake(#sslsocket{}, timeout()) -> {ok, #sslsocket{}} | + {ok, #sslsocket{}, map()}| {error, reason()}. %% %% Description: Starts ssl handshake. %%-------------------------------------------------------------------- -handshake(#sslsocket{pid = Pid}, Timeout) -> +handshake(#sslsocket{pid = Pid} = Socket, Timeout) -> case call(Pid, {start, Timeout}) of connected -> - ok; + {ok, Socket}; + {ok, Ext} -> + {ok, Socket, Ext}; Error -> Error end. %%-------------------------------------------------------------------- -spec handshake(#sslsocket{}, {#ssl_options{},#socket_options{}}, - timeout()) -> ok | {error, reason()}. + timeout()) -> {ok, #sslsocket{}} | {error, reason()}. %% %% Description: Starts ssl handshake with some new options %%-------------------------------------------------------------------- -handshake(#sslsocket{pid = Pid}, SslOptions, Timeout) -> +handshake(#sslsocket{pid = Pid} = Socket, SslOptions, Timeout) -> case call(Pid, {start, SslOptions, Timeout}) of connected -> - ok; + {ok, Socket}; Error -> Error end. +%%-------------------------------------------------------------------- +-spec handshake_continue(#sslsocket{}, [ssl_option()], + timeout()) -> {ok, #sslsocket{}}| {error, reason()}. +%% +%% Description: Continues handshake with new options +%%-------------------------------------------------------------------- +handshake_continue(#sslsocket{pid = Pid} = Socket, SslOptions, Timeout) -> + case call(Pid, {handshake_continue, SslOptions, Timeout}) of + connected -> + {ok, Socket}; + Error -> + Error + end. +%%-------------------------------------------------------------------- +-spec handshake_cancel(#sslsocket{}) -> ok | {error, reason()}. +%% +%% Description: Cancels connection +%%-------------------------------------------------------------------- +handshake_cancel(#sslsocket{pid = Pid}) -> + case call(Pid, cancel) of + closed -> + ok; + Error -> + Error + end. %-------------------------------------------------------------------- -spec socket_control(tls_connection | dtls_connection, port(), pid(), atom()) -> {ok, #sslsocket{}} | {error, reason()}. @@ -527,6 +557,9 @@ handle_session(#server_hello{cipher_suite = CipherSuite, -spec ssl_config(#ssl_options{}, client | server, #state{}) -> #state{}. %%-------------------------------------------------------------------- ssl_config(Opts, Role, State) -> + ssl_config(Opts, Role, State, new). + +ssl_config(Opts, Role, State0, Type) -> {ok, #{cert_db_ref := Ref, cert_db_handle := CertDbHandle, fileref_db_handle := FileRefHandle, @@ -536,20 +569,26 @@ ssl_config(Opts, Role, State) -> dh_params := DHParams, own_certificate := OwnCert}} = ssl_config:init(Opts, Role), - Handshake = ssl_handshake:init_handshake_history(), TimeStamp = erlang:monotonic_time(), - Session = State#state.session, - State#state{tls_handshake_history = Handshake, - session = Session#session{own_certificate = OwnCert, - time_stamp = TimeStamp}, - file_ref_db = FileRefHandle, - cert_db_ref = Ref, - cert_db = CertDbHandle, - crl_db = CRLDbHandle, - session_cache = CacheHandle, - private_key = Key, - diffie_hellman_params = DHParams, - ssl_options = Opts}. + Session = State0#state.session, + State = State0#state{session = Session#session{own_certificate = OwnCert, + time_stamp = TimeStamp}, + file_ref_db = FileRefHandle, + cert_db_ref = Ref, + cert_db = CertDbHandle, + crl_db = CRLDbHandle, + session_cache = CacheHandle, + private_key = Key, + diffie_hellman_params = DHParams, + ssl_options = Opts}, + case Type of + new -> + Handshake = ssl_handshake:init_handshake_history(), + State#state{tls_handshake_history = Handshake}; + continue -> + State + end. + %%==================================================================== %% gen_statem general state functions with connection cb argument @@ -579,7 +618,8 @@ init({call, From}, {start, {Opts, EmOpts}, Timeout}, end, State = ssl_config(SslOpts, Role, State0), init({call, From}, {start, Timeout}, - State#state{ssl_options = SslOpts, socket_options = new_emulated(EmOpts, SockOpts)}, Connection) + State#state{ssl_options = SslOpts, + socket_options = new_emulated(EmOpts, SockOpts)}, Connection) catch throw:Error -> stop_and_reply(normal, {reply, From, {error, Error}}, State0) end; @@ -612,6 +652,23 @@ hello(info, Msg, State, _) -> hello(Type, Msg, State, Connection) -> handle_common_event(Type, Msg, ?FUNCTION_NAME, State, Connection). +user_hello({call, From}, cancel, #state{negotiated_version = Version} = State, _) -> + gen_statem:reply(From, ok), + handle_own_alert(?ALERT_REC(?FATAL, ?USER_CANCELED, user_canceled), + Version, ?FUNCTION_NAME, State); +user_hello({call, From}, {handshake_continue, NewOptions, Timeout}, #state{hello = Hello, + role = Role, + start_or_recv_from = RecvFrom, + ssl_options = Options0} = State0, _Connection) -> + Timer = start_or_recv_cancel_timer(Timeout, RecvFrom), + Options = ssl:handle_options(NewOptions, Options0#ssl_options{handshake = full}), + State = ssl_config(Options, Role, State0, continue), + {next_state, hello, State#state{start_or_recv_from = From, + timer = Timer}, + [{next_event, internal, Hello}]}; +user_hello(_, _, _, _) -> + {keep_state_and_data, [postpone]}. + %%-------------------------------------------------------------------- -spec abbreviated(gen_statem:event_type(), #hello_request{} | #finished{} | term(), @@ -770,6 +827,14 @@ certify(internal, #certificate_request{}, Alg == srp_dss; Alg == srp_rsa; Alg == srp_anon -> handle_own_alert(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE), Version, ?FUNCTION_NAME, State); +certify(internal, #certificate_request{}, + #state{session = #session{own_certificate = undefined}, + role = client} = State0, Connection) -> + %% The client does not have a certificate and will send an empty reply, the server may fail + %% or accept the connection by its own preference. No signature algorihms needed as there is + %% no certificate to verify. + {Record, State} = Connection:next_record(State0), + Connection:next_event(?FUNCTION_NAME, Record, State#state{client_certificate_requested = true}); certify(internal, #certificate_request{} = CertRequest, #state{session = #session{own_certificate = Cert}, role = client, @@ -2285,7 +2350,24 @@ hibernate_after(connection = StateName, {next_state, StateName, State, [{timeout, HibernateAfter, hibernate} | Actions]}; hibernate_after(StateName, State, Actions) -> {next_state, StateName, State, Actions}. - + +map_extensions(#hello_extensions{renegotiation_info = RenegotiationInfo, + signature_algs = SigAlg, + alpn = Alpn, + next_protocol_negotiation = Next, + srp = SRP, + ec_point_formats = ECPointFmt, + elliptic_curves = ECCCurves, + sni = SNI}) -> + #{renegotiation_info => ssl_handshake:extension_value(RenegotiationInfo), + signature_algs => ssl_handshake:extension_value(SigAlg), + alpn => ssl_handshake:extension_value(Alpn), + srp => ssl_handshake:extension_value(SRP), + next_protocol => ssl_handshake:extension_value(Next), + ec_point_formats => ssl_handshake:extension_value(ECPointFmt), + elliptic_curves => ssl_handshake:extension_value(ECCCurves), + sni => ssl_handshake:extension_value(SNI)}. + terminate_alert(normal, Version, ConnectionStates, Connection) -> Connection:encode_alert(?ALERT_REC(?WARNING, ?CLOSE_NOTIFY), Version, ConnectionStates); diff --git a/lib/ssl/src/ssl_connection.hrl b/lib/ssl/src/ssl_connection.hrl index f9d2149170..a61ff747ae 100644 --- a/lib/ssl/src/ssl_connection.hrl +++ b/lib/ssl/src/ssl_connection.hrl @@ -61,7 +61,7 @@ client_certificate_requested = false :: boolean(), key_algorithm :: ssl_cipher:key_algo(), hashsign_algorithm = {undefined, undefined}, - cert_hashsign_algorithm, + cert_hashsign_algorithm = {undefined, undefined}, public_key_info :: ssl_handshake:public_key_info() | 'undefined', private_key :: public_key:private_key() | secret_printout() | 'undefined', diffie_hellman_params:: #'DHParameter'{} | undefined | secret_printout(), @@ -77,7 +77,8 @@ renegotiation :: undefined | {boolean(), From::term() | internal | peer}, start_or_recv_from :: term(), timer :: undefined | reference(), % start_or_recive_timer - %%send_queue :: queue:queue(), + %%send_queue :: queue:queue(), + hello, %%:: #client_hello{} | #server_hello{}, terminated = false ::boolean(), allow_renegotiate = true ::boolean(), expecting_next_protocol_negotiation = false ::boolean(), @@ -88,11 +89,11 @@ sni_hostname = undefined, downgrade, flight_buffer = [] :: list() | map(), %% Buffer of TLS/DTLS records, used during the TLS handshake - %% to when possible pack more than on TLS record into the - %% underlaying packet format. Introduced by DTLS - RFC 4347. - %% The mecahnism is also usefull in TLS although we do not - %% need to worry about packet loss in TLS. In DTLS we need to track DTLS handshake seqnr - flight_state = reliable, %% reliable | {retransmit, integer()}| {waiting, ref(), integer()} - last two is used in DTLS over udp. + %% to when possible pack more than one TLS record into the + %% underlaying packet format. Introduced by DTLS - RFC 4347. + %% The mecahnism is also usefull in TLS although we do not + %% need to worry about packet loss in TLS. In DTLS we need to track DTLS handshake seqnr + flight_state = reliable, %% reliable | {retransmit, integer()}| {waiting, ref(), integer()} - last two is used in DTLS over udp. protocol_specific = #{} :: map() }). -define(DEFAULT_DIFFIE_HELLMAN_PARAMS, diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index e1f813ea95..8ddd4623c1 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -53,7 +53,7 @@ -export([certify/7, certificate_verify/6, verify_signature/5, master_secret/4, server_key_exchange_hash/2, verify_connection/6, init_handshake_history/0, update_handshake_history/2, verify_server_key/5, - select_version/3 + select_version/3, extension_value/1 ]). %% Encode @@ -139,8 +139,8 @@ certificate(OwnCert, CertDbHandle, CertDbRef, server) -> case ssl_certificate:certificate_chain(OwnCert, CertDbHandle, CertDbRef) of {ok, _, Chain} -> #certificate{asn1_certificates = Chain}; - {error, _} -> - ?ALERT_REC(?FATAL, ?INTERNAL_ERROR, server_has_no_suitable_certificates) + {error, Error} -> + ?ALERT_REC(?FATAL, ?INTERNAL_ERROR, {server_has_no_suitable_certificates, Error}) end. %%-------------------------------------------------------------------- @@ -1091,12 +1091,6 @@ select_hashsign(_, Cert, _, _, Version) -> %% %% Description: Handles signature algorithms selection for certificate requests (client) %%-------------------------------------------------------------------- -select_hashsign(#certificate_request{}, undefined, _, {Major, Minor}) when Major >= 3 andalso Minor >= 3-> - %% There client does not have a certificate and will send an empty reply, the server may fail - %% or accept the connection by its own preference. No signature algorihms needed as there is - %% no certificate to verify. - {undefined, undefined}; - select_hashsign(#certificate_request{hashsign_algorithms = #hash_sign_algos{hash_sign_algos = HashSigns}, certificate_types = Types}, Cert, SupportedHashSigns, {Major, Minor}) when Major >= 3 andalso Minor >= 3-> @@ -1166,6 +1160,25 @@ srp_user(#ssl_options{srp_identity = {UserName, _}}) -> srp_user(_) -> undefined. +extension_value(undefined) -> + undefined; +extension_value(#sni{hostname = HostName}) -> + HostName; +extension_value(#ec_point_formats{ec_point_format_list = List}) -> + List; +extension_value(#elliptic_curves{elliptic_curve_list = List}) -> + List; +extension_value(#hash_sign_algos{hash_sign_algos = Algos}) -> + Algos; +extension_value(#alpn{extension_data = Data}) -> + Data; +extension_value(#next_protocol_negotiation{extension_data = Data}) -> + Data; +extension_value(#srp{username = Name}) -> + Name; +extension_value(#renegotiation_info{renegotiated_connection = Data}) -> + Data. + %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- diff --git a/lib/ssl/src/ssl_internal.hrl b/lib/ssl/src/ssl_internal.hrl index d354910f33..5df00de0e5 100644 --- a/lib/ssl/src/ssl_internal.hrl +++ b/lib/ssl/src/ssl_internal.hrl @@ -144,7 +144,8 @@ signature_algs, eccs, honor_ecc_order :: boolean(), - max_handshake_size :: integer() + max_handshake_size :: integer(), + handshake }). -record(socket_options, diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index c35378f18f..ef84c5320e 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -62,7 +62,7 @@ %% gen_statem state functions -export([init/3, error/3, downgrade/3, %% Initiation and take down states - hello/3, certify/3, cipher/3, abbreviated/3, %% Handshake states + hello/3, user_hello/3, certify/3, cipher/3, abbreviated/3, %% Handshake states connection/3, death_row/3]). %% gen_statem callbacks -export([callback_mode/0, terminate/3, code_change/4, format_status/2]). @@ -80,8 +80,7 @@ start_fsm(Role, Host, Port, Socket, {#ssl_options{erl_dist = false},_, Tracker} {ok, Pid} = tls_connection_sup:start_child([Role, Host, Port, Socket, Opts, User, CbInfo]), {ok, SslSocket} = ssl_connection:socket_control(?MODULE, Socket, Pid, CbModule, Tracker), - ok = ssl_connection:handshake(SslSocket, Timeout), - {ok, SslSocket} + ssl_connection:handshake(SslSocket, Timeout) catch error:{badmatch, {error, _} = Error} -> Error @@ -94,8 +93,7 @@ start_fsm(Role, Host, Port, Socket, {#ssl_options{erl_dist = true},_, Tracker} = {ok, Pid} = tls_connection_sup:start_child_dist([Role, Host, Port, Socket, Opts, User, CbInfo]), {ok, SslSocket} = ssl_connection:socket_control(?MODULE, Socket, Pid, CbModule, Tracker), - ok = ssl_connection:handshake(SslSocket, Timeout), - {ok, SslSocket} + ssl_connection:handshake(SslSocket, Timeout) catch error:{badmatch, {error, _} = Error} -> Error @@ -454,6 +452,16 @@ error(_, _, _) -> #state{}) -> gen_statem:state_function_result(). %%-------------------------------------------------------------------- +hello(internal, #client_hello{extensions = Extensions} = Hello, #state{ssl_options = #ssl_options{handshake = hello}, + start_or_recv_from = From} = State) -> + {next_state, user_hello, State#state{start_or_recv_from = undefined, + hello = Hello}, + [{reply, From, {ok, ssl_connection:map_extensions(Extensions)}}]}; +hello(internal, #server_hello{extensions = Extensions} = Hello, #state{ssl_options = #ssl_options{handshake = hello}, + start_or_recv_from = From} = State) -> + {next_state, user_hello, State#state{start_or_recv_from = undefined, + hello = Hello}, + [{reply, From, {ok, ssl_connection:map_extensions(Extensions)}}]}; hello(internal, #client_hello{client_version = ClientVersion} = Hello, #state{connection_states = ConnectionStates0, port = Port, session = #session{own_certificate = Cert} = Session0, @@ -463,7 +471,6 @@ hello(internal, #client_hello{client_version = ClientVersion} = Hello, negotiated_protocol = CurrentProtocol, key_algorithm = KeyExAlg, ssl_options = SslOpts} = State) -> - case tls_handshake:hello(Hello, SslOpts, {Port, Session0, Cache, CacheCb, ConnectionStates0, Cert, KeyExAlg}, Renegotiation) of #alert{} = Alert -> @@ -482,7 +489,7 @@ hello(internal, #client_hello{client_version = ClientVersion} = Hello, session = Session, negotiated_protocol = Protocol}) end; -hello(internal, #server_hello{} = Hello, +hello(internal, #server_hello{} = Hello, #state{connection_states = ConnectionStates0, negotiated_version = ReqVersion, role = client, @@ -500,6 +507,9 @@ hello(info, Event, State) -> hello(Type, Event, State) -> gen_handshake(?FUNCTION_NAME, Type, Event, State). +user_hello(Type, Event, State) -> + gen_handshake(?FUNCTION_NAME, Type, Event, State). + %%-------------------------------------------------------------------- -spec abbreviated(gen_statem:event_type(), term(), #state{}) -> gen_statem:state_function_result(). @@ -746,7 +756,7 @@ gen_handshake(StateName, Type, Event, malformed_handshake_data), Version, StateName, State) end. - + gen_info(Event, connection = StateName, #state{negotiated_version = Version} = State) -> try handle_info(Event, StateName, State) of Result -> diff --git a/lib/ssl/test/Makefile b/lib/ssl/test/Makefile index 9347b56f39..845f5bee2e 100644 --- a/lib/ssl/test/Makefile +++ b/lib/ssl/test/Makefile @@ -57,6 +57,8 @@ MODULES = \ ssl_session_cache_SUITE \ ssl_to_openssl_SUITE \ ssl_ECC_SUITE \ + ssl_ECC_openssl_SUITE \ + ssl_ECC\ ssl_upgrade_SUITE\ ssl_sni_SUITE \ make_certs\ diff --git a/lib/ssl/test/ssl_ECC.erl b/lib/ssl/test/ssl_ECC.erl new file mode 100644 index 0000000000..489a72e50e --- /dev/null +++ b/lib/ssl/test/ssl_ECC.erl @@ -0,0 +1,154 @@ + +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% + +-module(ssl_ECC). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("public_key/include/public_key.hrl"). + +%% Test diffrent certificate chain types, note that it is the servers +%% chain that affect what cipher suit that will be choosen + +%% ECDH_RSA +client_ecdh_rsa_server_ecdh_rsa(Config) when is_list(Config) -> + Suites = all_rsa_suites(Config), + Default = ssl_test_lib:default_cert_chain_conf(), + {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {client_chain, Default}], + ecdh_rsa, ecdh_rsa, Config), + ssl_test_lib:basic_test(ssl_test_lib:ssl_options(COpts, Config), + ssl_test_lib:ssl_options(SOpts, Config), + [{check_keyex, ecdh_rsa}, {ciphers, Suites} | proplists:delete(check_keyex, Config)]). +client_ecdhe_rsa_server_ecdh_rsa(Config) when is_list(Config) -> + Suites = all_rsa_suites(Config), + Default = ssl_test_lib:default_cert_chain_conf(), + {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {client_chain, Default}], + ecdhe_rsa, ecdh_rsa, Config), + ssl_test_lib:basic_test(ssl_test_lib:ssl_options(COpts, Config), + ssl_test_lib:ssl_options(SOpts, Config), + [{check_keyex, ecdh_rsa}, {ciphers, Suites} | proplists:delete(check_keyex, Config)]). +client_ecdhe_ecdsa_server_ecdh_rsa(Config) when is_list(Config) -> + Suites = all_rsa_suites(Config), + Default = ssl_test_lib:default_cert_chain_conf(), + {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {client_chain, Default}], + ecdhe_ecdsa, ecdh_rsa, Config), + ssl_test_lib:basic_test(ssl_test_lib:ssl_options(COpts, Config), + ssl_test_lib:ssl_options(SOpts, Config), + [{check_keyex, ecdh_rsa}, {ciphers, Suites} | proplists:delete(check_keyex, Config)]). + +%% ECDHE_RSA +client_ecdh_rsa_server_ecdhe_rsa(Config) when is_list(Config) -> + Default = ssl_test_lib:default_cert_chain_conf(), + {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {client_chain, Default}], + ecdh_rsa, ecdhe_rsa, Config), + ssl_test_lib:basic_test(ssl_test_lib:ssl_options(COpts, Config), + ssl_test_lib:ssl_options(SOpts, Config), + [{check_keyex, ecdhe_rsa} | proplists:delete(check_keyex, Config)]). +client_ecdhe_rsa_server_ecdhe_rsa(Config) when is_list(Config) -> + Default = ssl_test_lib:default_cert_chain_conf(), + {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {client_chain, Default}], + ecdhe_rsa, ecdhe_rsa, Config), + ssl_test_lib:basic_test(ssl_test_lib:ssl_options(COpts, Config), + ssl_test_lib:ssl_options(SOpts, Config), + [{check_keyex, ecdhe_rsa} | proplists:delete(check_keyex, Config)]). +client_ecdhe_ecdsa_server_ecdhe_rsa(Config) when is_list(Config) -> + Default = ssl_test_lib:default_cert_chain_conf(), + {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {client_chain, Default}], + ecdh_ecdsa, ecdhe_rsa, Config), + ssl_test_lib:basic_test(ssl_test_lib:ssl_options(COpts, Config), + ssl_test_lib:ssl_options(SOpts, Config), + [{check_keyex, ecdhe_rsa} | proplists:delete(check_keyex, Config)]). + +%% ECDH_ECDSA +client_ecdh_ecdsa_server_ecdh_ecdsa(Config) when is_list(Config) -> + Ext = x509_test:extensions([{key_usage, [keyEncipherment]}]), + {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, + [[], [], [{extensions, Ext}]]}, + {client_chain, + ssl_test_lib:default_cert_chain_conf()}], + ecdh_ecdsa, ecdh_ecdsa, Config), + ssl_test_lib:basic_test(ssl_test_lib:ssl_options(COpts, Config), + ssl_test_lib:ssl_options(SOpts, Config), + [{check_keyex, ecdh_ecdsa} | proplists:delete(check_keyex, Config)]). +client_ecdhe_rsa_server_ecdh_ecdsa(Config) when is_list(Config) -> + Ext = x509_test:extensions([{key_usage, [keyEncipherment]}]), + {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, + [[], [], [{extensions, Ext}]]}, + {client_chain, + ssl_test_lib:default_cert_chain_conf()}], + ecdhe_rsa, ecdh_ecdsa, Config), + ssl_test_lib:basic_test(ssl_test_lib:ssl_options(COpts, Config), + ssl_test_lib:ssl_options(SOpts, Config), + [{check_keyex, ecdh_ecdsa} | proplists:delete(check_keyex, Config)]). + +client_ecdhe_ecdsa_server_ecdh_ecdsa(Config) when is_list(Config) -> + Ext = x509_test:extensions([{key_usage, [keyEncipherment]}]), + {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, + [[], [], [{extensions, Ext}]]}, + {client_chain, + ssl_test_lib:default_cert_chain_conf()}], + ecdhe_ecdsa, ecdh_ecdsa, Config), + ssl_test_lib:basic_test(ssl_test_lib:ssl_options(COpts, Config), + ssl_test_lib:ssl_options(SOpts, Config), + [{check_keyex, ecdh_ecdsa} | proplists:delete(check_keyex, Config)]). + +%% ECDHE_ECDSA +client_ecdh_rsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> + Default = ssl_test_lib:default_cert_chain_conf(), + {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {client_chain, Default}], + ecdh_rsa, ecdhe_ecdsa, Config), + ssl_test_lib:basic_test(ssl_test_lib:ssl_options(COpts, Config), + ssl_test_lib:ssl_options(SOpts, Config), + [{check_keyex, ecdhe_ecdsa} | proplists:delete(check_keyex, Config)]). +client_ecdh_ecdsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> + Default = ssl_test_lib:default_cert_chain_conf(), + {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {client_chain, Default}], + ecdh_ecdsa, ecdhe_ecdsa, Config), + ssl_test_lib:basic_test(ssl_test_lib:ssl_options(COpts, Config), + ssl_test_lib:ssl_options(SOpts, Config), + [{check_keyex, ecdhe_ecdsa} | proplists:delete(check_keyex, Config)]). +client_ecdhe_ecdsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> + Default = ssl_test_lib:default_cert_chain_conf(), + {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {client_chain, Default}], + ecdhe_ecdsa, ecdhe_ecdsa, Config), + ssl_test_lib:basic_test(ssl_test_lib:ssl_options(COpts, Config), + ssl_test_lib:ssl_options(SOpts, Config), + [{check_keyex, ecdhe_ecdsa} | proplists:delete(check_keyex, Config)]). + +all_rsa_suites(Config) -> + Version = proplists:get_value(tls_version, Config), + All = ssl:cipher_suites(all, Version), + Default = ssl:cipher_suites(default, Version), + RSASuites = ssl:filter_cipher_suites(All,[{key_exchange, fun(rsa) -> true;(_) -> false end}]), + ssl:append_cipher_suites(RSASuites, Default). diff --git a/lib/ssl/test/ssl_ECC_SUITE.erl b/lib/ssl/test/ssl_ECC_SUITE.erl index f38c0a7416..6e2d86571a 100644 --- a/lib/ssl/test/ssl_ECC_SUITE.erl +++ b/lib/ssl/test/ssl_ECC_SUITE.erl @@ -43,52 +43,17 @@ all() -> groups() -> [ - {'tlsv1.2', [], all_versions_groups()}, - {'tlsv1.1', [], all_versions_groups()}, - {'tlsv1', [], all_versions_groups()}, - {'dtlsv1.2', [], all_versions_groups()}, - {'dtlsv1', [], all_versions_groups()}, - {'erlang_server', [], openssl_key_cert_combinations()}, - %%{'erlang_client', [], openssl_key_cert_combinations()}, - {'erlang', [], key_cert_combinations() ++ misc() - ++ ecc_negotiation()} + {'tlsv1.2', [], test_cases()}, + {'tlsv1.1', [], test_cases()}, + {'tlsv1', [], test_cases()}, + {'dtlsv1.2', [], test_cases()}, + {'dtlsv1', [], test_cases()} ]. -all_versions_groups ()-> - [{group, 'erlang_server'}, - %%{group, 'erlang_client'}, - {group, 'erlang'} - ]. - - -openssl_key_cert_combinations() -> - ECDH_RSA = case ssl_test_lib:openssl_filter("ECDH-RSA") of - [] -> - []; - _ -> - server_ecdh_rsa() - end, - - ECDHE_RSA = case ssl_test_lib:openssl_filter("ECDHE-RSA") of - [] -> - []; - _ -> - server_ecdhe_rsa() - end, - ECDH_ECDSA = case ssl_test_lib:openssl_filter("ECDH-ECDSA") of - [] -> - []; - _ -> - server_ecdhe_ecdsa() - end, - - ECDHE_ECDSA = case ssl_test_lib:openssl_filter("ECDHE-ECDSA") of - [] -> - []; - _ -> - server_ecdhe_ecdsa() - end, - ECDH_RSA ++ ECDHE_RSA ++ ECDH_ECDSA ++ ECDHE_ECDSA. +test_cases()-> + key_cert_combinations() + ++ misc() + ++ ecc_negotiation(). key_cert_combinations() -> server_ecdh_rsa() ++ @@ -116,7 +81,6 @@ server_ecdhe_ecdsa() -> client_ecdh_ecdsa_server_ecdhe_ecdsa, client_ecdhe_ecdsa_server_ecdhe_ecdsa]. - misc()-> [client_ecdsa_server_ecdsa_with_raw_key]. @@ -142,9 +106,14 @@ init_per_suite(Config0) -> end_per_suite(Config0), try crypto:start() of ok -> - Config0 + case ssl_test_lib:sufficient_crypto_support(cipher_ec) of + true -> + Config0; + false -> + {skip, "Crypto does not support ECC"} + end catch _:_ -> - {skip, "Crypto did not start"} + {skip, "Crypto did not start"} end. end_per_suite(_Config) -> @@ -152,52 +121,14 @@ end_per_suite(_Config) -> application:stop(crypto). %%-------------------------------------------------------------------- -init_per_group(erlang_client = Group, Config) -> - case ssl_test_lib:is_sane_ecc(openssl) of - true -> - common_init_per_group(Group, [{server_type, openssl}, - {client_type, erlang} | Config]); - false -> - {skip, "Known ECC bug in openssl"} - end; - -init_per_group(erlang_server = Group, Config) -> - case ssl_test_lib:is_sane_ecc(openssl) of - true -> - common_init_per_group(Group, [{server_type, erlang}, - {client_type, openssl} | Config]); - false -> - {skip, "Known ECC bug in openssl"} - end; - -init_per_group(erlang = Group, Config) -> - case ssl_test_lib:sufficient_crypto_support(Group) of - true -> - common_init_per_group(Group, [{server_type, erlang}, - {client_type, erlang} | Config]); - false -> - {skip, "Crypto does not support ECC"} - end; - -init_per_group(openssl = Group, Config) -> - case ssl_test_lib:sufficient_crypto_support(Group) of - true -> - common_init_per_group(Group, [{server_type, openssl}, - {client_type, openssl} | Config]); - false -> - {skip, "Crypto does not support ECC"} - end; - -init_per_group(Group, Config) -> - common_init_per_group(Group, Config). - -common_init_per_group(GroupName, Config) -> +init_per_group(GroupName, Config) -> case ssl_test_lib:is_tls_version(GroupName) of true -> - Config0 = ssl_test_lib:init_tls_version(GroupName, Config), - [{tls_version, GroupName} | Config0]; - _ -> - openssl_check(GroupName, Config) + [{tls_version, GroupName}, + {server_type, erlang}, + {client_type, erlang} | ssl_test_lib:init_tls_version(GroupName, Config)]; + _ -> + Config end. end_per_group(GroupName, Config0) -> @@ -215,7 +146,7 @@ init_per_testcase(TestCase, Config) -> ssl_test_lib:ct_log_supported_protocol_versions(Config), ct:log("Ciphers: ~p~n ", [ ssl:cipher_suites()]), end_per_testcase(TestCase, Config), - ssl_test_lib:clean_start(), + ssl:start(), ct:timetrap({seconds, 15}), Config. @@ -226,104 +157,45 @@ end_per_testcase(_TestCase, Config) -> %%-------------------------------------------------------------------- %% Test Cases -------------------------------------------------------- %%-------------------------------------------------------------------- - %% Test diffrent certificate chain types, note that it is the servers %% chain that affect what cipher suit that will be choosen %% ECDH_RSA client_ecdh_rsa_server_ecdh_rsa(Config) when is_list(Config) -> - Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, - {client_chain, Default}], - ecdh_rsa, ecdh_rsa, Config), - basic_test(COpts, SOpts, [{check_keyex, ecdh_rsa} | proplists:delete(check_keyex, Config)]). -client_ecdhe_rsa_server_ecdh_rsa(Config) when is_list(Config) -> - Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, - {client_chain, Default}], - ecdhe_rsa, ecdh_rsa, Config), - basic_test(COpts, SOpts, [{check_keyex, ecdh_rsa} | proplists:delete(check_keyex, Config)]). -client_ecdhe_ecdsa_server_ecdh_rsa(Config) when is_list(Config) -> - Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, - {client_chain, Default}], - ecdhe_ecdsa, ecdh_rsa, Config), - basic_test(COpts, SOpts, [{check_keyex, ecdh_rsa} | proplists:delete(check_keyex, Config)]). - + ssl_ECC:client_ecdh_rsa_server_ecdh_rsa(Config). +client_ecdhe_rsa_server_ecdh_rsa(Config) when is_list(Config) -> + ssl_ECC:client_ecdhe_rsa_server_ecdh_rsa(Config). +client_ecdhe_ecdsa_server_ecdh_rsa(Config) when is_list(Config) -> + ssl_ECC:client_ecdhe_ecdsa_server_ecdh_rsa(Config). %% ECDHE_RSA client_ecdh_rsa_server_ecdhe_rsa(Config) when is_list(Config) -> - Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, - {client_chain, Default}], - ecdh_rsa, ecdhe_rsa, Config), - basic_test(COpts, SOpts, [{check_keyex, ecdhe_rsa} | proplists:delete(check_keyex, Config)]). -client_ecdhe_rsa_server_ecdhe_rsa(Config) when is_list(Config) -> - Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, - {client_chain, Default}], - ecdhe_rsa, ecdhe_rsa, Config), - basic_test(COpts, SOpts, [{check_keyex, ecdhe_rsa} | proplists:delete(check_keyex, Config)]). -client_ecdhe_ecdsa_server_ecdhe_rsa(Config) when is_list(Config) -> - Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, - {client_chain, Default}], - ecdh_ecdsa, ecdhe_rsa, Config), - basic_test(COpts, SOpts, [{check_keyex, ecdhe_rsa} | proplists:delete(check_keyex, Config)]). - + ssl_ECC:client_ecdh_rsa_server_ecdhe_rsa(Config). +client_ecdhe_rsa_server_ecdhe_rsa(Config) when is_list(Config) -> + ssl_ECC:client_ecdhe_rsa_server_ecdhe_rsa(Config). +client_ecdhe_ecdsa_server_ecdhe_rsa(Config) when is_list(Config) -> + ssl_ECC:client_ecdhe_ecdsa_server_ecdhe_rsa(Config). %% ECDH_ECDSA -client_ecdh_ecdsa_server_ecdh_ecdsa(Config) when is_list(Config) -> - Ext = x509_test:extensions([{key_usage, [keyEncipherment]}]), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, - [[], [], [{extensions, Ext}]]}, - {client_chain, - ssl_test_lib:default_cert_chain_conf()}], - ecdh_ecdsa, ecdh_ecdsa, Config), - basic_test(COpts, SOpts, - [{check_keyex, ecdh_ecdsa} | proplists:delete(check_keyex, Config)]). -client_ecdhe_rsa_server_ecdh_ecdsa(Config) when is_list(Config) -> - Ext = x509_test:extensions([{key_usage, [keyEncipherment]}]), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, - [[], [], [{extensions, Ext}]]}, - {client_chain, - ssl_test_lib:default_cert_chain_conf()}], - ecdhe_rsa, ecdh_ecdsa, Config), - basic_test(COpts, SOpts, [{check_keyex, ecdh_ecdsa} | proplists:delete(check_keyex, Config)]). - -client_ecdhe_ecdsa_server_ecdh_ecdsa(Config) when is_list(Config) -> - Ext = x509_test:extensions([{key_usage, [keyEncipherment]}]), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, - [[], [], [{extensions, Ext}]]}, - {client_chain, - ssl_test_lib:default_cert_chain_conf()}], - ecdhe_ecdsa, ecdh_ecdsa, Config), - basic_test(COpts, SOpts, - [{check_keyex, ecdh_ecdsa} | proplists:delete(check_keyex, Config)]). - +client_ecdh_ecdsa_server_ecdh_ecdsa(Config) when is_list(Config) -> + ssl_ECC:client_ecdh_ecdsa_server_ecdh_ecdsa(Config). +client_ecdhe_rsa_server_ecdh_ecdsa(Config) when is_list(Config) -> + ssl_ECC:client_ecdhe_rsa_server_ecdh_ecdsa(Config). +client_ecdhe_ecdsa_server_ecdh_ecdsa(Config) when is_list(Config) -> + ssl_ECC:client_ecdhe_ecdsa_server_ecdh_ecdsa(Config). %% ECDHE_ECDSA -client_ecdh_rsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> - Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, - {client_chain, Default}], - ecdh_rsa, ecdhe_ecdsa, Config), - basic_test(COpts, SOpts, [{check_keyex, ecdhe_ecdsa} | proplists:delete(check_keyex, Config)]). -client_ecdh_ecdsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> - Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, - {client_chain, Default}], - ecdh_ecdsa, ecdhe_ecdsa, Config), - basic_test(COpts, SOpts, [{check_keyex, ecdhe_ecdsa} | proplists:delete(check_keyex, Config)]). -client_ecdhe_ecdsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> - Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, - {client_chain, Default}], - ecdhe_ecdsa, ecdhe_ecdsa, Config), - basic_test(COpts, SOpts, [{check_keyex, ecdhe_ecdsa} | proplists:delete(check_keyex, Config)]). +client_ecdh_rsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> + ssl_ECC:client_ecdh_rsa_server_ecdhe_ecdsa(Config). +client_ecdh_ecdsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> + ssl_ECC:client_ecdh_ecdsa_server_ecdhe_ecdsa(Config). +client_ecdhe_ecdsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> + ssl_ECC:client_ecdhe_ecdsa_server_ecdhe_ecdsa(Config). client_ecdsa_server_ecdsa_with_raw_key(Config) when is_list(Config) -> Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {COpts0, SOpts0} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, {client_chain, Default}] , ecdhe_ecdsa, ecdhe_ecdsa, Config), + COpts = ssl_test_lib:ssl_options(COpts0, Config), + SOpts = ssl_test_lib:ssl_options(SOpts0, Config), ServerKeyFile = proplists:get_value(keyfile, SOpts), {ok, PemBin} = file:read_file(ServerKeyFile), PemEntries = public_key:pem_decode(PemBin), @@ -331,331 +203,192 @@ client_ecdsa_server_ecdsa_with_raw_key(Config) when is_list(Config) -> ServerKey = {'ECPrivateKey', Key}, SType = proplists:get_value(server_type, Config), CType = proplists:get_value(client_type, Config), - {Server, Port} = start_server_with_raw_key(SType, - [{key, ServerKey} | proplists:delete(keyfile, SOpts)], - Config), - Client = start_client(CType, Port, COpts, Config), - check_result(Server, SType, Client, CType), - close(Server, Client). + {Server, Port} = ssl_test_lib:start_server_with_raw_key(SType, + [{key, ServerKey} | proplists:delete(keyfile, SOpts)], + Config), + Client = ssl_test_lib:start_client(CType, Port, COpts, Config), + ssl_test_lib:gen_check_result(Server, SType, Client, CType), + ssl_test_lib:stop(Server, Client). ecc_default_order(Config) -> Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, - {client_chain, Default}], - ecdhe_ecdsa, ecdhe_ecdsa, Config), + {COpts0, SOpts0} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {client_chain, Default}], + ecdhe_ecdsa, ecdhe_ecdsa, Config), + COpts = ssl_test_lib:ssl_options(COpts0, Config), + SOpts = ssl_test_lib:ssl_options(SOpts0, Config), ECCOpts = [], - case supported_eccs([{eccs, [sect571r1]}]) of - true -> ecc_test(sect571r1, COpts, SOpts, [], ECCOpts, Config); + case ssl_test_lib:supported_eccs([{eccs, [sect571r1]}]) of + true -> ssl_test_lib:ecc_test(sect571r1, COpts, SOpts, [], ECCOpts, Config); false -> {skip, "unsupported named curves"} end. ecc_default_order_custom_curves(Config) -> Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, - {client_chain, Default}], - ecdhe_ecdsa, ecdhe_ecdsa, Config), + {COpts0, SOpts0} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {client_chain, Default}], + ecdhe_ecdsa, ecdhe_ecdsa, Config), + COpts = ssl_test_lib:ssl_options(COpts0, Config), + SOpts = ssl_test_lib:ssl_options(SOpts0, Config), ECCOpts = [{eccs, [secp256r1, sect571r1]}], - case supported_eccs(ECCOpts) of - true -> ecc_test(sect571r1, COpts, SOpts, [], ECCOpts, Config); + case ssl_test_lib:supported_eccs(ECCOpts) of + true -> ssl_test_lib:ecc_test(sect571r1, COpts, SOpts, [], ECCOpts, Config); false -> {skip, "unsupported named curves"} end. ecc_client_order(Config) -> Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, - {client_chain, Default}], - ecdhe_ecdsa, ecdhe_ecdsa, Config), + {COpts0, SOpts0} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {client_chain, Default}], + ecdhe_ecdsa, ecdhe_ecdsa, Config), + COpts = ssl_test_lib:ssl_options(COpts0, Config), + SOpts = ssl_test_lib:ssl_options(SOpts0, Config), ECCOpts = [{honor_ecc_order, false}], - case supported_eccs([{eccs, [sect571r1]}]) of - true -> ecc_test(sect571r1, COpts, SOpts, [], ECCOpts, Config); + case ssl_test_lib:supported_eccs([{eccs, [sect571r1]}]) of + true -> ssl_test_lib:ecc_test(sect571r1, COpts, SOpts, [], ECCOpts, Config); false -> {skip, "unsupported named curves"} end. ecc_client_order_custom_curves(Config) -> Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {COpts0, SOpts0} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, {client_chain, Default}], ecdhe_ecdsa, ecdhe_ecdsa, Config), + COpts = ssl_test_lib:ssl_options(COpts0, Config), + SOpts = ssl_test_lib:ssl_options(SOpts0, Config), ECCOpts = [{honor_ecc_order, false}, {eccs, [secp256r1, sect571r1]}], - case supported_eccs(ECCOpts) of - true -> ecc_test(sect571r1, COpts, SOpts, [], ECCOpts, Config); + case ssl_test_lib:supported_eccs(ECCOpts) of + true -> ssl_test_lib:ecc_test(sect571r1, COpts, SOpts, [], ECCOpts, Config); false -> {skip, "unsupported named curves"} end. ecc_unknown_curve(Config) -> Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {COpts0, SOpts0} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, {client_chain, Default}], ecdhe_ecdsa, ecdhe_ecdsa, Config), + COpts = ssl_test_lib:ssl_options(COpts0, Config), + SOpts = ssl_test_lib:ssl_options(SOpts0, Config), ECCOpts = [{eccs, ['123_fake_curve']}], - ecc_test_error(COpts, SOpts, [], ECCOpts, Config). + ssl_test_lib:ecc_test_error(COpts, SOpts, [], ECCOpts, Config). client_ecdh_rsa_server_ecdhe_ecdsa_server_custom(Config) -> Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {COpts0, SOpts0} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, {client_chain, Default}], - ecdh_rsa, ecdhe_ecdsa, Config), + ecdh_rsa, ecdhe_ecdsa, Config), + COpts = ssl_test_lib:ssl_options(COpts0, Config), + SOpts = ssl_test_lib:ssl_options(SOpts0, Config), ECCOpts = [{honor_ecc_order, true}, {eccs, [secp256r1, sect571r1]}], - case supported_eccs(ECCOpts) of - true -> ecc_test(secp256r1, COpts, SOpts, [], ECCOpts, Config); + case ssl_test_lib:supported_eccs(ECCOpts) of + true -> ssl_test_lib:ecc_test(secp256r1, COpts, SOpts, [], ECCOpts, Config); false -> {skip, "unsupported named curves"} end. client_ecdh_rsa_server_ecdhe_rsa_server_custom(Config) -> Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {COpts0, SOpts0} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, {client_chain, Default}], ecdh_rsa, ecdhe_rsa, Config), + COpts = ssl_test_lib:ssl_options(COpts0, Config), + SOpts = ssl_test_lib:ssl_options(SOpts0, Config), ECCOpts = [{honor_ecc_order, true}, {eccs, [secp256r1, sect571r1]}], - case supported_eccs(ECCOpts) of - true -> ecc_test(undefined, COpts, SOpts, [], ECCOpts, Config); + + case ssl_test_lib:supported_eccs(ECCOpts) of + true -> ssl_test_lib:ecc_test(undefined, COpts, SOpts, [], ECCOpts, Config); false -> {skip, "unsupported named curves"} end. client_ecdhe_rsa_server_ecdhe_ecdsa_server_custom(Config) -> Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {COpts0, SOpts0} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, {client_chain, Default}], ecdhe_rsa, ecdhe_ecdsa, Config), + COpts = ssl_test_lib:ssl_options(COpts0, Config), + SOpts = ssl_test_lib:ssl_options(SOpts0, Config), ECCOpts = [{honor_ecc_order, true}, {eccs, [secp256r1, sect571r1]}], - case supported_eccs(ECCOpts) of - true -> ecc_test(secp256r1, COpts, SOpts, [], ECCOpts, Config); + case ssl_test_lib:supported_eccs(ECCOpts) of + true -> ssl_test_lib:ecc_test(secp256r1, COpts, SOpts, [], ECCOpts, Config); false -> {skip, "unsupported named curves"} end. client_ecdhe_rsa_server_ecdhe_rsa_server_custom(Config) -> Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {COpts0, SOpts0} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, {client_chain, Default}], ecdhe_rsa, ecdhe_rsa, Config), + + COpts = ssl_test_lib:ssl_options(COpts0, Config), + SOpts = ssl_test_lib:ssl_options(SOpts0, Config), ECCOpts = [{honor_ecc_order, true}, {eccs, [secp256r1, sect571r1]}], - case supported_eccs(ECCOpts) of - true -> ecc_test(undefined, COpts, SOpts, [], ECCOpts, Config); + case ssl_test_lib:supported_eccs(ECCOpts) of + true -> ssl_test_lib:ecc_test(undefined, COpts, SOpts, [], ECCOpts, Config); false -> {skip, "unsupported named curves"} end. client_ecdhe_rsa_server_ecdh_rsa_server_custom(Config) -> Default = ssl_test_lib:default_cert_chain_conf(), Ext = x509_test:extensions([{key_usage, [keyEncipherment]}]), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, [[], [], [{extensions, Ext}]]}, + {COpts0, SOpts0} = ssl_test_lib:make_ec_cert_chains([{server_chain, [[], [], [{extensions, Ext}]]}, {client_chain, Default}], ecdhe_rsa, ecdh_rsa, Config), + + COpts = ssl_test_lib:ssl_options(COpts0, Config), + SOpts = ssl_test_lib:ssl_options(SOpts0, Config), ECCOpts = [{honor_ecc_order, true}, {eccs, [secp256r1, sect571r1]}], - case supported_eccs(ECCOpts) of - true -> ecc_test(undefined, COpts, SOpts, [], ECCOpts, Config); + + case ssl_test_lib:supported_eccs(ECCOpts) of + true -> ssl_test_lib:ecc_test(undefined, COpts, SOpts, [], ECCOpts, Config); false -> {skip, "unsupported named curves"} end. client_ecdhe_ecdsa_server_ecdhe_ecdsa_server_custom(Config) -> Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {COpts0, SOpts0} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, {client_chain, Default}], ecdhe_ecdsa, ecdhe_ecdsa, Config), + COpts = ssl_test_lib:ssl_options(COpts0, Config), + SOpts = ssl_test_lib:ssl_options(SOpts0, Config), ECCOpts = [{honor_ecc_order, true}, {eccs, [secp256r1, sect571r1]}], - case supported_eccs(ECCOpts) of - true -> ecc_test(secp256r1, COpts, SOpts, [], ECCOpts, Config); + case ssl_test_lib:supported_eccs(ECCOpts) of + true -> ssl_test_lib:ecc_test(secp256r1, COpts, SOpts, [], ECCOpts, Config); false -> {skip, "unsupported named curves"} end. client_ecdhe_ecdsa_server_ecdhe_rsa_server_custom(Config) -> Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {COpts0, SOpts0} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, {client_chain, Default}], ecdhe_ecdsa, ecdhe_rsa, Config), + COpts = ssl_test_lib:ssl_options(COpts0, Config), + SOpts = ssl_test_lib:ssl_options(SOpts0, Config), ECCOpts = [{honor_ecc_order, true}, {eccs, [secp256r1, sect571r1]}], - case supported_eccs(ECCOpts) of - true -> ecc_test(undefined, COpts, SOpts, [], ECCOpts, Config); + case ssl_test_lib:supported_eccs(ECCOpts) of + true -> ssl_test_lib:ecc_test(undefined, COpts, SOpts, [], ECCOpts, Config); false -> {skip, "unsupported named curves"} end. client_ecdhe_ecdsa_server_ecdhe_ecdsa_client_custom(Config) -> Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {COpts0, SOpts0} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, {client_chain, Default}], ecdhe_ecdsa, ecdhe_ecdsa, Config), + COpts = ssl_test_lib:ssl_options(COpts0, Config), + SOpts = ssl_test_lib:ssl_options(SOpts0, Config), ECCOpts = [{eccs, [secp256r1, sect571r1]}], - case supported_eccs(ECCOpts) of - true -> ecc_test(secp256r1, COpts, SOpts, ECCOpts, [], Config); + case ssl_test_lib:supported_eccs(ECCOpts) of + true -> ssl_test_lib:ecc_test(secp256r1, COpts, SOpts, ECCOpts, [], Config); false -> {skip, "unsupported named curves"} end. client_ecdhe_rsa_server_ecdhe_ecdsa_client_custom(Config) -> Default = ssl_test_lib:default_cert_chain_conf(), - {COpts, SOpts} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, + {COpts0, SOpts0} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, {client_chain, Default}], ecdhe_rsa, ecdhe_ecdsa, Config), + COpts = ssl_test_lib:ssl_options(COpts0, Config), + SOpts = ssl_test_lib:ssl_options(SOpts0, Config), ECCOpts = [{eccs, [secp256r1, sect571r1]}], - case supported_eccs(ECCOpts) of - true -> ecc_test(secp256r1, COpts, SOpts, ECCOpts, [], Config); + case ssl_test_lib:supported_eccs(ECCOpts) of + true -> ssl_test_lib:ecc_test(secp256r1, COpts, SOpts, ECCOpts, [], Config); false -> {skip, "unsupported named curves"} end. - -%%-------------------------------------------------------------------- -%% Internal functions ------------------------------------------------ -%%-------------------------------------------------------------------- -basic_test(COpts, SOpts, Config) -> - SType = proplists:get_value(server_type, Config), - CType = proplists:get_value(client_type, Config), - {Server, Port} = start_server(SType, SOpts, Config), - Client = start_client(CType, Port, COpts, Config), - check_result(Server, SType, Client, CType), - close(Server, Client). - - -ecc_test(Expect, COpts, SOpts, CECCOpts, SECCOpts, Config) -> - {Server, Port} = start_server_ecc(erlang, SOpts, Expect, SECCOpts, Config), - Client = start_client_ecc(erlang, Port, COpts, Expect, CECCOpts, Config), - ssl_test_lib:check_result(Server, ok, Client, ok), - close(Server, Client). - -ecc_test_error(COpts, SOpts, CECCOpts, SECCOpts, Config) -> - {Server, Port} = start_server_ecc_error(erlang, SOpts, SECCOpts, Config), - Client = start_client_ecc_error(erlang, Port, COpts, CECCOpts, Config), - Error = {error, {tls_alert, "insufficient security"}}, - ssl_test_lib:check_result(Server, Error, Client, Error). - - -start_client(openssl, Port, ClientOpts, _Config) -> - Cert = proplists:get_value(certfile, ClientOpts), - Key = proplists:get_value(keyfile, ClientOpts), - CA = proplists:get_value(cacertfile, ClientOpts), - Version = tls_record:protocol_version(tls_record:highest_protocol_version([])), - Exe = "openssl", - Args = ["s_client", "-verify", "2", "-port", integer_to_list(Port), - ssl_test_lib:version_flag(Version), - "-cert", Cert, "-CAfile", CA, - "-key", Key, "-host","localhost", "-msg", "-debug"], - - OpenSslPort = ssl_test_lib:portable_open_port(Exe, Args), - true = port_command(OpenSslPort, "Hello world"), - OpenSslPort; - -start_client(erlang, Port, ClientOpts, Config) -> - {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), - KeyEx = proplists:get_value(check_keyex, Config, false), - ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, - {host, Hostname}, - {from, self()}, - {mfa, {ssl_test_lib, check_key_exchange_send_active, [KeyEx]}}, - {options, [{verify, verify_peer} | ClientOpts]}]). - - -start_client_ecc(erlang, Port, ClientOpts, Expect, ECCOpts, Config) -> - {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), - ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, - {host, Hostname}, - {from, self()}, - {mfa, {?MODULE, check_ecc, [client, Expect]}}, - {options, - ECCOpts ++ - [{verify, verify_peer} | ClientOpts]}]). - -start_client_ecc_error(erlang, Port, ClientOpts, ECCOpts, Config) -> - {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), - ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port}, - {host, Hostname}, - {from, self()}, - {options, - ECCOpts ++ - [{verify, verify_peer} | ClientOpts]}]). - - -start_server(openssl, ServerOpts, _Config) -> - Cert = proplists:get_value(certfile, ServerOpts), - Key = proplists:get_value(keyfile, ServerOpts), - CA = proplists:get_value(cacertfile, ServerOpts), - Port = ssl_test_lib:inet_port(node()), - Version = tls_record:protocol_version(tls_record:highest_protocol_version([])), - Exe = "openssl", - Args = ["s_server", "-accept", integer_to_list(Port), ssl_test_lib:version_flag(Version), - "-verify", "2", "-cert", Cert, "-CAfile", CA, - "-key", Key, "-msg", "-debug"], - OpenSslPort = ssl_test_lib:portable_open_port(Exe, Args), - true = port_command(OpenSslPort, "Hello world"), - {OpenSslPort, Port}; -start_server(erlang, ServerOpts, Config) -> - {_, ServerNode, _} = ssl_test_lib:run_where(Config), - KeyEx = proplists:get_value(check_keyex, Config, false), - Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, - {from, self()}, - {mfa, {ssl_test_lib, - check_key_exchange_send_active, - [KeyEx]}}, - {options, [{verify, verify_peer} | ServerOpts]}]), - {Server, ssl_test_lib:inet_port(Server)}. - -start_server_with_raw_key(erlang, ServerOpts, Config) -> - {_, ServerNode, _} = ssl_test_lib:run_where(Config), - Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, - {from, self()}, - {mfa, {ssl_test_lib, - send_recv_result_active, - []}}, - {options, - [{verify, verify_peer} | ServerOpts]}]), - {Server, ssl_test_lib:inet_port(Server)}. - -start_server_ecc(erlang, ServerOpts, Expect, ECCOpts, Config) -> - {_, ServerNode, _} = ssl_test_lib:run_where(Config), - Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, - {from, self()}, - {mfa, {?MODULE, check_ecc, [server, Expect]}}, - {options, - ECCOpts ++ - [{verify, verify_peer} | ServerOpts]}]), - {Server, ssl_test_lib:inet_port(Server)}. - -start_server_ecc_error(erlang, ServerOpts, ECCOpts, Config) -> - {_, ServerNode, _} = ssl_test_lib:run_where(Config), - Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, - {from, self()}, - {options, - ECCOpts ++ - [{verify, verify_peer} | ServerOpts]}]), - {Server, ssl_test_lib:inet_port(Server)}. - -check_result(Server, erlang, Client, erlang) -> - ssl_test_lib:check_result(Server, ok, Client, ok); -check_result(Server, erlang, _, _) -> - ssl_test_lib:check_result(Server, ok); -check_result(_, _, Client, erlang) -> - ssl_test_lib:check_result(Client, ok); -check_result(_,openssl, _, openssl) -> - ok. - -openssl_check(erlang, Config) -> - Config; -openssl_check(_, Config) -> - TLSVersion = proplists:get_value(tls_version, Config), - case ssl_test_lib:check_sane_openssl_version(TLSVersion) of - true -> - Config; - false -> - {skip, "TLS version not supported by openssl"} - end. - -close(Port1, Port2) when is_port(Port1), is_port(Port2) -> - ssl_test_lib:close_port(Port1), - ssl_test_lib:close_port(Port2); -close(Port, Pid) when is_port(Port) -> - ssl_test_lib:close_port(Port), - ssl_test_lib:close(Pid); -close(Pid, Port) when is_port(Port) -> - ssl_test_lib:close_port(Port), - ssl_test_lib:close(Pid); -close(Client, Server) -> - ssl_test_lib:close(Server), - ssl_test_lib:close(Client). - -supported_eccs(Opts) -> - ToCheck = proplists:get_value(eccs, Opts, []), - Supported = ssl:eccs(), - lists:all(fun(Curve) -> lists:member(Curve, Supported) end, ToCheck). - -check_ecc(SSL, Role, Expect) -> - {ok, Data} = ssl:connection_information(SSL), - case lists:keyfind(ecc, 1, Data) of - {ecc, {named_curve, Expect}} -> ok; - false when Expect =:= undefined -> ok; - Other -> {error, Role, Expect, Other} - end. - diff --git a/lib/ssl/test/ssl_ECC_openssl_SUITE.erl b/lib/ssl/test/ssl_ECC_openssl_SUITE.erl new file mode 100644 index 0000000000..ba609aa0dc --- /dev/null +++ b/lib/ssl/test/ssl_ECC_openssl_SUITE.erl @@ -0,0 +1,185 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% + +-module(ssl_ECC_openssl_SUITE). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("public_key/include/public_key.hrl"). + +%%-------------------------------------------------------------------- +%% Common Test interface functions ----------------------------------- +%%-------------------------------------------------------------------- + +all() -> + [ + {group, 'tlsv1.2'}, + {group, 'tlsv1.1'}, + {group, 'tlsv1'}, + {group, 'dtlsv1.2'}, + {group, 'dtlsv1'} + ]. + +groups() -> + [ + {'tlsv1.2', [], test_cases()}, + {'tlsv1.1', [], test_cases()}, + {'tlsv1', [], test_cases()}, + {'dtlsv1.2', [], test_cases()}, + {'dtlsv1', [], test_cases()} + ]. + +test_cases()-> + %% cert_combinations(). + server_ecdh_rsa(). +cert_combinations() -> + lists:append(lists:filtermap(fun({Name, Suites}) -> + case ssl_test_lib:openssl_filter(Name) of + [] -> + false; + [_|_] -> + {true, Suites} + end + end, [{"ECDH-RSA", server_ecdh_rsa()}, + {"ECDHE-RSA", server_ecdhe_rsa()}, + {"ECDH-ECDSA", server_ecdh_ecdsa()}, + {"ECDHE-ECDSA", server_ecdhe_ecdsa()} + ])). +server_ecdh_rsa() -> + [client_ecdh_rsa_server_ecdh_rsa, + client_ecdhe_rsa_server_ecdh_rsa, + client_ecdhe_ecdsa_server_ecdh_rsa]. + +server_ecdhe_rsa() -> + [client_ecdh_rsa_server_ecdhe_rsa, + client_ecdhe_rsa_server_ecdhe_rsa, + client_ecdhe_ecdsa_server_ecdhe_rsa]. + +server_ecdh_ecdsa() -> + [client_ecdh_ecdsa_server_ecdh_ecdsa, + client_ecdhe_rsa_server_ecdh_ecdsa, + client_ecdhe_ecdsa_server_ecdh_ecdsa]. + +server_ecdhe_ecdsa() -> + [client_ecdh_rsa_server_ecdhe_ecdsa, + client_ecdh_ecdsa_server_ecdhe_ecdsa, + client_ecdhe_ecdsa_server_ecdhe_ecdsa]. + +%%-------------------------------------------------------------------- +init_per_suite(Config0) -> + end_per_suite(Config0), + try crypto:start() of + ok -> + case ssl_test_lib:sufficient_crypto_support(cipher_ec) of + true -> + Config0; + false -> + {skip, "Crypto does not support ECC"} + end + catch _:_ -> + {skip, "Crypto did not start"} + end. + +end_per_suite(_Config) -> + application:stop(ssl), + application:stop(crypto). + +%%-------------------------------------------------------------------- +init_per_group(GroupName, Config) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + case ssl_test_lib:check_sane_openssl_version(GroupName) of + true -> + [{tls_version, GroupName}, + {server_type, erlang}, + {client_type, openssl} | ssl_test_lib:init_tls_version(GroupName, Config)]; + false -> + {skip, openssl_does_not_support_version} + end; + _ -> + Config + end. + +end_per_group(GroupName, Config0) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + Config = ssl_test_lib:clean_tls_version(Config0), + proplists:delete(tls_version, Config); + false -> + Config0 + end. + +%%-------------------------------------------------------------------- + +init_per_testcase(TestCase, Config) -> + ssl_test_lib:ct_log_supported_protocol_versions(Config), + Version = proplists:get_value(tls_version, Config), + ct:log("Ciphers: ~p~n ", [ssl:cipher_suites(default, Version)]), + end_per_testcase(TestCase, Config), + ssl:start(), + ct:timetrap({seconds, 15}), + Config. + +end_per_testcase(_TestCase, Config) -> + application:stop(ssl), + Config. + +%%-------------------------------------------------------------------- +%% Test Cases -------------------------------------------------------- +%%-------------------------------------------------------------------- + +%% Test diffrent certificate chain types, note that it is the servers +%% chain that affect what cipher suit that will be choosen + +%% ECDH_RSA +client_ecdh_rsa_server_ecdh_rsa(Config) when is_list(Config) -> + ssl_ECC:client_ecdh_rsa_server_ecdh_rsa(Config). +client_ecdhe_rsa_server_ecdh_rsa(Config) when is_list(Config) -> + ssl_ECC:client_ecdhe_rsa_server_ecdh_rsa(Config). +client_ecdhe_ecdsa_server_ecdh_rsa(Config) when is_list(Config) -> + ssl_ECC:client_ecdhe_ecdsa_server_ecdh_rsa(Config). +%% ECDHE_RSA +client_ecdh_rsa_server_ecdhe_rsa(Config) when is_list(Config) -> + ssl_ECC:client_ecdh_rsa_server_ecdhe_rsa(Config). +client_ecdhe_rsa_server_ecdhe_rsa(Config) when is_list(Config) -> + ssl_ECC:client_ecdhe_rsa_server_ecdhe_rsa(Config). +client_ecdhe_ecdsa_server_ecdhe_rsa(Config) when is_list(Config) -> + ssl_ECC:client_ecdhe_ecdsa_server_ecdhe_rsa(Config). +%% ECDH_ECDSA +client_ecdh_ecdsa_server_ecdh_ecdsa(Config) when is_list(Config) -> + ssl_ECC:client_ecdh_ecdsa_server_ecdh_ecdsa(Config). +client_ecdhe_rsa_server_ecdh_ecdsa(Config) when is_list(Config) -> + ssl_ECC:client_ecdhe_rsa_server_ecdh_ecdsa(Config). +client_ecdhe_ecdsa_server_ecdh_ecdsa(Config) when is_list(Config) -> + ssl_ECC:client_ecdhe_ecdsa_server_ecdh_ecdsa(Config). +%% ECDHE_ECDSA +client_ecdh_rsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> + ssl_ECC:client_ecdh_rsa_server_ecdhe_ecdsa(Config). +client_ecdh_ecdsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> + ssl_ECC:client_ecdh_ecdsa_server_ecdhe_ecdsa(Config). +client_ecdhe_ecdsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> + ssl_ECC:client_ecdhe_ecdsa_server_ecdhe_ecdsa(Config). +%%-------------------------------------------------------------------- +%% Internal functions ------------------------------------------------ +%%-------------------------------------------------------------------- diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index a9901007db..fe4f02f100 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -164,7 +164,10 @@ api_tests() -> accept_pool, prf, socket_options, - cipher_suites + cipher_suites, + handshake_continue, + hello_client_cancel, + hello_server_cancel ]. api_tests_tls() -> @@ -291,6 +294,7 @@ init_per_group(GroupName, Config) when GroupName == basic_tls; -> ssl_test_lib:clean_tls_version(Config); init_per_group(GroupName, Config) -> + ssl_test_lib:clean_tls_version(Config), case ssl_test_lib:is_tls_version(GroupName) andalso ssl_test_lib:sufficient_crypto_support(GroupName) of true -> ssl_test_lib:init_tls_version(GroupName, Config); @@ -629,6 +633,84 @@ new_options_in_accept(Config) when is_list(Config) -> ssl_test_lib:close(Server), ssl_test_lib:close(Client). + +%%-------------------------------------------------------------------- +handshake_continue() -> + [{doc, "Test API function ssl:handshake_continue/3"}]. +handshake_continue(Config) when is_list(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ssl_test_lib:ssl_options([{reuseaddr, true}, {handshake, hello}], + Config)}, + {continue_options, proplists:delete(reuseaddr, ServerOpts)} + ]), + + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ssl_test_lib:ssl_options([{handshake, hello}], + Config)}, + {continue_options, proplists:delete(reuseaddr, ClientOpts)}]), + + ssl_test_lib:check_result(Server, ok, Client, ok), + + ssl_test_lib:close(Server), + ssl_test_lib:close(Client). + +%%-------------------------------------------------------------------- +hello_client_cancel() -> + [{doc, "Test API function ssl:handshake_cancel/1 on the client side"}]. +hello_client_cancel(Config) when is_list(Config) -> + ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {options, ssl_test_lib:ssl_options([{handshake, hello}], Config)}, + {continue_options, proplists:delete(reuseaddr, ServerOpts)}]), + + Port = ssl_test_lib:inet_port(Server), + + %% That is ssl:handshake_cancel returns ok + {connect_failed, ok} = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {options, ssl_test_lib:ssl_options([{handshake, hello}], Config)}, + {continue_options, cancel}]), + + ssl_test_lib:check_result(Server, {error, {tls_alert, "user canceled"}}). +%%-------------------------------------------------------------------- + +hello_server_cancel() -> + [{doc, "Test API function ssl:handshake_cancel/1 on the server side"}]. +hello_server_cancel(Config) when is_list(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {options, ssl_test_lib:ssl_options([{handshake, hello}], Config)}, + {continue_options, cancel}]), + + Port = ssl_test_lib:inet_port(Server), + + {connect_failed, _} = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {options, ssl_test_lib:ssl_options([{handshake, hello}], Config)}, + {continue_options, proplists:delete(reuseaddr, ClientOpts)}]), + + ssl_test_lib:check_result(Server, ok). + %%-------------------------------------------------------------------- prf() -> [{doc,"Test that ssl:prf/5 uses the negotiated PRF."}]. @@ -962,7 +1044,7 @@ controller_dies(Config) when is_list(Config) -> {mfa, {?MODULE, controller_dies_result, [self(), ClientMsg]}}, - {options, [{reuseaddr,true}|ClientOpts]}]), + {options, ClientOpts}]), ct:sleep(?SLEEP), %% so that they are connected exit(Server, killed), @@ -987,7 +1069,7 @@ tls_client_closes_socket(Config) when is_list(Config) -> Connect = fun() -> {ok, _Socket} = rpc:call(ClientNode, gen_tcp, connect, - [Hostname, Port, TcpOpts]), + [Hostname, Port, [binary]]), %% Make sure that ssl_accept is called before %% client process ends and closes socket. ct:sleep(?SLEEP) @@ -1811,7 +1893,7 @@ tls_send_close(Config) when is_list(Config) -> {options, [{active, false} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), {ok, TcpS} = rpc:call(ClientNode, gen_tcp, connect, - [Hostname,Port,[binary, {active, false}, {reuseaddr, true}]]), + [Hostname,Port,[binary, {active, false}]]), {ok, SslS} = rpc:call(ClientNode, ssl, connect, [TcpS,[{active, false}|ClientOpts]]), @@ -1955,7 +2037,7 @@ tls_upgrade(Config) when is_list(Config) -> {host, Hostname}, {from, self()}, {mfa, {?MODULE, upgrade_result, []}}, - {tcp_options, TcpOpts}, + {tcp_options, [binary]}, {ssl_options, ClientOpts}]), ct:log("Testcase ~p, Client ~p Server ~p ~n", diff --git a/lib/ssl/test/ssl_certificate_verify_SUITE.erl b/lib/ssl/test/ssl_certificate_verify_SUITE.erl index 0bc265fa10..1de4c89d7f 100644 --- a/lib/ssl/test/ssl_certificate_verify_SUITE.erl +++ b/lib/ssl/test/ssl_certificate_verify_SUITE.erl @@ -40,14 +40,22 @@ %%-------------------------------------------------------------------- all() -> [ - {group, tls}, - {group, dtls} + {group, 'tlsv1.2'}, + {group, 'tlsv1.1'}, + {group, 'tlsv1'}, + {group, 'sslv3'}, + {group, 'dtlsv1.2'}, + {group, 'dtlsv1'} ]. groups() -> [ - {tls, [], all_protocol_groups()}, - {dtls, [], all_protocol_groups()}, + {'tlsv1.2', [], all_protocol_groups()}, + {'tlsv1.1', [], all_protocol_groups()}, + {'tlsv1', [], all_protocol_groups()}, + {'sslv3', [], all_protocol_groups()}, + {'dtlsv1.2', [], all_protocol_groups()}, + {'dtlsv1', [], all_protocol_groups()}, {active, [], tests()}, {active_once, [], tests()}, {passive, [], tests()}, @@ -65,6 +73,7 @@ tests() -> verify_none, server_require_peer_cert_ok, server_require_peer_cert_fail, + server_require_peer_cert_empty_ok, server_require_peer_cert_partial_chain, server_require_peer_cert_allow_partial_chain, server_require_peer_cert_do_not_allow_partial_chain, @@ -104,24 +113,6 @@ end_per_suite(_Config) -> ssl:stop(), application:stop(crypto). -init_per_group(tls, Config0) -> - Version = tls_record:protocol_version(tls_record:highest_protocol_version([])), - ssl:stop(), - application:load(ssl), - application:set_env(ssl, protocol_version, Version), - ssl:start(), - Config = ssl_test_lib:init_tls_version(Version, Config0), - [{version, tls_record:protocol_version(Version)} | Config]; - -init_per_group(dtls, Config0) -> - Version = dtls_record:protocol_version(dtls_record:highest_protocol_version([])), - ssl:stop(), - application:load(ssl), - application:set_env(ssl, protocol_version, Version), - ssl:start(), - Config = ssl_test_lib:init_tls_version(Version, Config0), - [{version, dtls_record:protocol_version(Version)} | Config]; - init_per_group(active, Config) -> [{active, true}, {receive_function, send_recv_result_active} | Config]; init_per_group(active_once, Config) -> @@ -130,15 +121,24 @@ init_per_group(passive, Config) -> [{active, false}, {receive_function, send_recv_result} | Config]; init_per_group(error_handling, Config) -> [{active, false}, {receive_function, send_recv_result} | Config]; +init_per_group(GroupName, Config) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + case ssl_test_lib:sufficient_crypto_support(GroupName) of + true -> + [{version, GroupName} | ssl_test_lib:init_tls_version(GroupName, Config)]; + false -> + {skip, "Missing crypto support"} + end + end. -init_per_group(_, Config) -> - Config. - -end_per_group(GroupName, Config) when GroupName == tls; - GroupName == dtls -> - ssl_test_lib:clean_tls_version(Config); -end_per_group(_GroupName, Config) -> - Config. +end_per_group(GroupName, Config) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + ssl_test_lib:clean_tls_version(Config); + false -> + Config + end. init_per_testcase(_TestCase, Config) -> ssl:stop(), @@ -306,6 +306,35 @@ server_require_peer_cert_fail(Config) when is_list(Config) -> end. %%-------------------------------------------------------------------- +server_require_peer_cert_empty_ok() -> + [{doc,"Test server option fail_if_no_peer_cert when peer sends cert"}]. + +server_require_peer_cert_empty_ok(Config) when is_list(Config) -> + ServerOpts = [{verify, verify_peer}, {fail_if_no_peer_cert, false} + | ssl_test_lib:ssl_options(server_rsa_opts, Config)], + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), + Active = proplists:get_value(active, Config), + ReceiveFunction = proplists:get_value(receive_function, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + ClientOpts = proplists:delete(keyfile, proplists:delete(certfile, ClientOpts0)), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, ReceiveFunction, []}}, + {options, [{active, Active} | ServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {ssl_test_lib, ReceiveFunction, []}}, + {options, [{active, Active} | ClientOpts]}]), + + ssl_test_lib:check_result(Server, ok, Client, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close(Client). + +%%-------------------------------------------------------------------- server_require_peer_cert_partial_chain() -> [{doc, "Client sends an incompleate chain, by default not acceptable."}]. @@ -930,6 +959,7 @@ client_with_cert_cipher_suites_handshake(Config) when is_list(Config) -> Config, "_sign_only_extensions"), ClientOpts = ssl_test_lib:ssl_options(ClientOpts0, Config), ServerOpts = ssl_test_lib:ssl_options(ServerOpts0, Config), + TLSVersion = ssl_test_lib:protocol_version(Config, tuple), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, @@ -938,7 +968,7 @@ client_with_cert_cipher_suites_handshake(Config) when is_list(Config) -> send_recv_result_active, []}}, {options, [{active, true}, {ciphers, - ssl_test_lib:rsa_non_signed_suites(proplists:get_value(version, Config))} + ssl_test_lib:rsa_non_signed_suites(TLSVersion)} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index e531494f0b..3a7e844cf8 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -79,17 +79,21 @@ run_server(ListenSocket, Opts, N) -> Pid ! {accepter, N, Server}, run_server(ListenSocket, Opts, N-1). -do_run_server(_, {error, timeout} = Result, Opts) -> +do_run_server(_, {error, _} = Result, Opts) -> + ct:log("Server error result ~p~n", [Result]), + Pid = proplists:get_value(from, Opts), + Pid ! {self(), Result}; +do_run_server(_, ok = Result, Opts) -> + ct:log("Server cancel result ~p~n", [Result]), Pid = proplists:get_value(from, Opts), Pid ! {self(), Result}; - do_run_server(ListenSocket, AcceptSocket, Opts) -> Node = proplists:get_value(node, Opts), Pid = proplists:get_value(from, Opts), Transport = proplists:get_value(transport, Opts, ssl), {Module, Function, Args} = proplists:get_value(mfa, Opts), ct:log("~p:~p~nServer: apply(~p,~p,~p)~n", - [?MODULE,?LINE, Module, Function, [AcceptSocket | Args]]), + [?MODULE,?LINE, Module, Function, [AcceptSocket | Args]]), case rpc:call(Node, Module, Function, [AcceptSocket | Args]) of no_result_msg -> ok; @@ -117,7 +121,8 @@ connect(#sslsocket{} = ListenSocket, Opts) -> ReconnectTimes = proplists:get_value(reconnect_times, Opts, 0), Timeout = proplists:get_value(timeout, Opts, infinity), SslOpts = proplists:get_value(ssl_extra_opts, Opts, []), - AcceptSocket = connect(ListenSocket, Node, 1 + ReconnectTimes, dummy, Timeout, SslOpts), + ContOpts = proplists:get_value(continue_options, Opts, []), + AcceptSocket = connect(ListenSocket, Node, 1 + ReconnectTimes, dummy, Timeout, SslOpts, ContOpts), case ReconnectTimes of 0 -> AcceptSocket; @@ -132,10 +137,45 @@ connect(ListenSocket, Opts) -> [ListenSocket]), AcceptSocket. -connect(_, _, 0, AcceptSocket, _, _) -> +connect(_, _, 0, AcceptSocket, _, _, _) -> AcceptSocket; - -connect(ListenSocket, Node, N, _, Timeout, []) -> +connect(ListenSocket, Node, _N, _, Timeout, SslOpts, cancel) -> + ct:log("ssl:transport_accept(~p)~n", [ListenSocket]), + {ok, AcceptSocket} = rpc:call(Node, ssl, transport_accept, + [ListenSocket]), + ct:log("~p:~p~nssl:handshake(~p,~p,~p)~n", [?MODULE,?LINE, AcceptSocket, SslOpts,Timeout]), + + case rpc:call(Node, ssl, handshake, [AcceptSocket, SslOpts, Timeout]) of + {ok, Socket0, Ext} -> + ct:log("Ext ~p:~n", [Ext]), + ct:log("~p:~p~nssl:handshake_cancel(~p)~n", [?MODULE,?LINE, Socket0]), + rpc:call(Node, ssl, handshake_cancel, [Socket0]); + Result -> + ct:log("~p:~p~nssl:handshake@~p ret ~p",[?MODULE,?LINE, Node,Result]), + Result + end; +connect(ListenSocket, Node, N, _, Timeout, SslOpts, [_|_] =ContOpts) -> + ct:log("ssl:transport_accept(~p)~n", [ListenSocket]), + {ok, AcceptSocket} = rpc:call(Node, ssl, transport_accept, + [ListenSocket]), + ct:log("~p:~p~nssl:handshake(~p,~p,~p)~n", [?MODULE,?LINE, AcceptSocket, SslOpts,Timeout]), + + case rpc:call(Node, ssl, handshake, [AcceptSocket, SslOpts, Timeout]) of + {ok, Socket0, Ext} -> + ct:log("Ext ~p:~n", [Ext]), + ct:log("~p:~p~nssl:handshake_continue(~p,~p,~p)~n", [?MODULE,?LINE, Socket0, ContOpts,Timeout]), + case rpc:call(Node, ssl, handshake_continue, [Socket0, ContOpts, Timeout]) of + {ok, Socket} -> + connect(ListenSocket, Node, N-1, Socket, Timeout, SslOpts, ContOpts); + Error -> + ct:log("~p:~p~nssl:handshake_continue@~p ret ~p",[?MODULE,?LINE, Node,Error]), + Error + end; + Result -> + ct:log("~p:~p~nssl:handshake@~p ret ~p",[?MODULE,?LINE, Node,Result]), + Result + end; +connect(ListenSocket, Node, N, _, Timeout, [], ContOpts) -> ct:log("ssl:transport_accept(~p)~n", [ListenSocket]), {ok, AcceptSocket} = rpc:call(Node, ssl, transport_accept, [ListenSocket]), @@ -143,12 +183,12 @@ connect(ListenSocket, Node, N, _, Timeout, []) -> case rpc:call(Node, ssl, ssl_accept, [AcceptSocket, Timeout]) of ok -> - connect(ListenSocket, Node, N-1, AcceptSocket, Timeout, []); + connect(ListenSocket, Node, N-1, AcceptSocket, Timeout, [], ContOpts); Result -> ct:log("~p:~p~nssl:ssl_accept@~p ret ~p",[?MODULE,?LINE, Node,Result]), Result end; -connect(ListenSocket, Node, _, _, Timeout, Opts) -> +connect(ListenSocket, Node, _, _, Timeout, Opts, _) -> ct:log("ssl:transport_accept(~p)~n", [ListenSocket]), {ok, AcceptSocket} = rpc:call(Node, ssl, transport_accept, [ListenSocket]), @@ -187,8 +227,17 @@ run_client(Opts) -> Pid = proplists:get_value(from, Opts), Transport = proplists:get_value(transport, Opts, ssl), Options = proplists:get_value(options, Opts), + ContOpts = proplists:get_value(continue_options, Opts, []), ct:log("~p:~p~n~p:connect(~p, ~p)@~p~n", [?MODULE,?LINE, Transport, Host, Port, Node]), ct:log("SSLOpts: ~p", [Options]), + case ContOpts of + [] -> + client_loop(Node, Host, Port, Pid, Transport, Options, Opts); + _ -> + client_cont_loop(Node, Host, Port, Pid, Transport, Options, ContOpts, Opts) + end. + +client_loop(Node, Host, Port, Pid, Transport, Options, Opts) -> case rpc:call(Node, Transport, connect, [Host, Port, Options]) of {ok, Socket} -> Pid ! {connected, Socket}, @@ -245,6 +294,40 @@ run_client(Opts) -> Pid ! {connect_failed, {badrpc,BadRPC}} end. +client_cont_loop(Node, Host, Port, Pid, Transport, Options, cancel, _Opts) -> + case rpc:call(Node, Transport, connect, [Host, Port, Options]) of + {ok, Socket, _} -> + Result = rpc:call(Node, Transport, handshake_cancel, [Socket]), + ct:log("~p:~p~nClient: Cancel: ~p ~n", [?MODULE,?LINE, Result]), + Pid ! {connect_failed, Result}; + {error, Reason} -> + ct:log("~p:~p~nClient: connection failed: ~p ~n", [?MODULE,?LINE, Reason]), + Pid ! {connect_failed, Reason} + end; + +client_cont_loop(Node, Host, Port, Pid, Transport, Options, ContOpts, Opts) -> + case rpc:call(Node, Transport, connect, [Host, Port, Options]) of + {ok, Socket0, _} -> + ct:log("~p:~p~nClient: handshake_continue(~p, ~p, infinity) ~n", [?MODULE, ?LINE, Socket0, ContOpts]), + case rpc:call(Node, Transport, handshake_continue, [Socket0, ContOpts, infinity]) of + {ok, Socket} -> + Pid ! {connected, Socket}, + {Module, Function, Args} = proplists:get_value(mfa, Opts), + ct:log("~p:~p~nClient: apply(~p,~p,~p)~n", + [?MODULE,?LINE, Module, Function, [Socket | Args]]), + case rpc:call(Node, Module, Function, [Socket | Args]) of + no_result_msg -> + ok; + Msg -> + ct:log("~p:~p~nClient Msg: ~p ~n", [?MODULE,?LINE, Msg]), + Pid ! {self(), Msg} + end + end; + {error, Reason} -> + ct:log("~p:~p~nClient: connection failed: ~p ~n", [?MODULE,?LINE, Reason]), + Pid ! {connect_failed, Reason} + end. + close(Pid) -> ct:log("~p:~p~nClose ~p ~n", [?MODULE,?LINE, Pid]), Monitor = erlang:monitor(process, Pid), @@ -862,6 +945,163 @@ accepters(Acc, N) -> accepters([Server| Acc], N-1) end. + +basic_test(COpts, SOpts, Config) -> + SType = proplists:get_value(server_type, Config), + CType = proplists:get_value(client_type, Config), + {Server, Port} = start_server(SType, SOpts, Config), + Client = start_client(CType, Port, COpts, Config), + gen_check_result(Server, SType, Client, CType), + stop(Server, Client). + +ecc_test(Expect, COpts, SOpts, CECCOpts, SECCOpts, Config) -> + {Server, Port} = start_server_ecc(erlang, SOpts, Expect, SECCOpts, Config), + Client = start_client_ecc(erlang, Port, COpts, Expect, CECCOpts, Config), + check_result(Server, ok, Client, ok), + stop(Server, Client). + +ecc_test_error(COpts, SOpts, CECCOpts, SECCOpts, Config) -> + {Server, Port} = start_server_ecc_error(erlang, SOpts, SECCOpts, Config), + Client = start_client_ecc_error(erlang, Port, COpts, CECCOpts, Config), + Error = {error, {tls_alert, "insufficient security"}}, + check_result(Server, Error, Client, Error). + + +start_client(openssl, Port, ClientOpts, Config) -> + Cert = proplists:get_value(certfile, ClientOpts), + Key = proplists:get_value(keyfile, ClientOpts), + CA = proplists:get_value(cacertfile, ClientOpts), + Version = ssl_test_lib:protocol_version(Config), + Exe = "openssl", + Args = ["s_client", "-verify", "2", "-port", integer_to_list(Port), + ssl_test_lib:version_flag(Version), + "-cert", Cert, "-CAfile", CA, + "-key", Key, "-host","localhost", "-msg", "-debug"], + + OpenSslPort = ssl_test_lib:portable_open_port(Exe, Args), + true = port_command(OpenSslPort, "Hello world"), + OpenSslPort; + +start_client(erlang, Port, ClientOpts, Config) -> + {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), + KeyEx = proplists:get_value(check_keyex, Config, false), + ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {ssl_test_lib, check_key_exchange_send_active, [KeyEx]}}, + {options, [{verify, verify_peer} | ClientOpts]}]). + + +start_client_ecc(erlang, Port, ClientOpts, Expect, ECCOpts, Config) -> + {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), + ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, check_ecc, [client, Expect]}}, + {options, + ECCOpts ++ + [{verify, verify_peer} | ClientOpts]}]). + +start_client_ecc_error(erlang, Port, ClientOpts, ECCOpts, Config) -> + {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), + ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {options, + ECCOpts ++ + [{verify, verify_peer} | ClientOpts]}]). + + +start_server(openssl, ServerOpts, Config) -> + Cert = proplists:get_value(certfile, ServerOpts), + Key = proplists:get_value(keyfile, ServerOpts), + CA = proplists:get_value(cacertfile, ServerOpts), + Port = inet_port(node()), + Version = protocol_version(Config), + Exe = "openssl", + Args = ["s_server", "-accept", integer_to_list(Port), ssl_test_lib:version_flag(Version), + "-verify", "2", "-cert", Cert, "-CAfile", CA, + "-key", Key, "-msg", "-debug"], + OpenSslPort = portable_open_port(Exe, Args), + true = port_command(OpenSslPort, "Hello world"), + {OpenSslPort, Port}; +start_server(erlang, ServerOpts, Config) -> + {_, ServerNode, _} = ssl_test_lib:run_where(Config), + KeyEx = proplists:get_value(check_keyex, Config, false), + Server = start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, + check_key_exchange_send_active, + [KeyEx]}}, + {options, [{verify, verify_peer} | ServerOpts]}]), + {Server, inet_port(Server)}. + +start_server_with_raw_key(erlang, ServerOpts, Config) -> + {_, ServerNode, _} = ssl_test_lib:run_where(Config), + Server = start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, + send_recv_result_active, + []}}, + {options, + [{verify, verify_peer} | ServerOpts]}]), + {Server, inet_port(Server)}. + +start_server_ecc(erlang, ServerOpts, Expect, ECCOpts, Config) -> + {_, ServerNode, _} = run_where(Config), + Server = start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, check_ecc, [server, Expect]}}, + {options, + ECCOpts ++ + [{verify, verify_peer} | ServerOpts]}]), + {Server, inet_port(Server)}. + +start_server_ecc_error(erlang, ServerOpts, ECCOpts, Config) -> + {_, ServerNode, _} = run_where(Config), + Server = start_server_error([{node, ServerNode}, {port, 0}, + {from, self()}, + {options, + ECCOpts ++ + [{verify, verify_peer} | ServerOpts]}]), + {Server, inet_port(Server)}. + +gen_check_result(Server, erlang, Client, erlang) -> + check_result(Server, ok, Client, ok); +gen_check_result(Server, erlang, _, _) -> + check_result(Server, ok); +gen_check_result(_, _, Client, erlang) -> + check_result(Client, ok); +gen_check_result(_,openssl, _, openssl) -> + ok. + +stop(Port1, Port2) when is_port(Port1), is_port(Port2) -> + close_port(Port1), + close_port(Port2); +stop(Port, Pid) when is_port(Port) -> + close_port(Port), + close(Pid); +stop(Pid, Port) when is_port(Port) -> + close_port(Port), + close(Pid); +stop(Client, Server) -> + close(Server), + close(Client). + +supported_eccs(Opts) -> + ToCheck = proplists:get_value(eccs, Opts, []), + Supported = ssl:eccs(), + lists:all(fun(Curve) -> lists:member(Curve, Supported) end, ToCheck). + +check_ecc(SSL, Role, Expect) -> + {ok, Data} = ssl:connection_information(SSL), + case lists:keyfind(ecc, 1, Data) of + {ecc, {named_curve, Expect}} -> ok; + false when Expect == undefined -> ok; + false when Expect == secp256r1 andalso Role == client_no_ecc -> ok; + Other -> {error, Role, Expect, Other} + end. + inet_port(Pid) when is_pid(Pid)-> receive {Pid, {port, Port}} -> @@ -1185,10 +1425,7 @@ sufficient_crypto_support(Version) when Version == 'tlsv1.2'; Version == 'dtlsv1.2' -> CryptoSupport = crypto:supports(), proplists:get_bool(sha256, proplists:get_value(hashs, CryptoSupport)); -sufficient_crypto_support(Group) when Group == ciphers_ec; %% From ssl_basic_SUITE - Group == erlang_server; %% From ssl_ECC_SUITE - Group == erlang_client; %% From ssl_ECC_SUITE - Group == erlang -> %% From ssl_ECC_SUITE +sufficient_crypto_support(cipher_ec) -> CryptoSupport = crypto:supports(), proplists:get_bool(ecdh, proplists:get_value(public_keys, CryptoSupport)); sufficient_crypto_support(_) -> diff --git a/lib/stdlib/doc/src/Makefile b/lib/stdlib/doc/src/Makefile index af27efa6c1..508a4fa2de 100644 --- a/lib/stdlib/doc/src/Makefile +++ b/lib/stdlib/doc/src/Makefile @@ -163,7 +163,7 @@ clean clean_docs: rm -f errs core *~ $(SPECDIR)/specs_erl_id_trans.xml: - escript $(SPECS_EXTRACTOR) $(SPECS_FLAGS) \ + $(gen_verbose)escript $(SPECS_EXTRACTOR) $(SPECS_FLAGS) \ -o$(dir $@) -module erl_id_trans # ---------------------------------------------------- diff --git a/lib/stdlib/doc/src/calendar.xml b/lib/stdlib/doc/src/calendar.xml index 65b3edcdf6..8f2b6b747a 100644 --- a/lib/stdlib/doc/src/calendar.xml +++ b/lib/stdlib/doc/src/calendar.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2016</year> + <year>1996</year><year>2018</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -317,6 +317,30 @@ </func> <func> + <name name="rfc3339_to_system_time" arity="1"/> + <name name="rfc3339_to_system_time" arity="2"/> + <fsummary>Convert from RFC 3339 timestamp to system time.</fsummary> + <type name="rfc3339_string"/> + <type name="rfc3339_time_unit"/> + <desc> + <p>Converts an RFC 3339 timestamp into system time.</p> + <p>Valid option:</p> + <taglist> + <tag><c>{unit, Unit}</c></tag> + <item><p>The time unit of the return value. + The default is <c>second</c>.</p> + </item> + </taglist> + <pre> +1> <input>calendar:rfc3339_to_system_time("2018-02-01T16:17:58+01:00").</input> +1517498278 +2> <input>calendar:rfc3339_to_system_time("2018-02-01 15:18:02.088Z", + [{unit, nanosecond}]).</input> +1517498282088000000</pre> + </desc> + </func> + + <func> <name name="seconds_to_daystime" arity="1"/> <fsummary>Compute days and time from seconds.</fsummary> <desc> @@ -339,6 +363,68 @@ </func> <func> + <name name="system_time_to_local_time" arity="2"/> + <fsummary>Convert system time to local date and time.</fsummary> + <desc> + <p>Converts a specified system time into local date and time.</p> + </desc> + </func> + + <func> + <name name="system_time_to_rfc3339" arity="1"/> + <name name="system_time_to_rfc3339" arity="2"/> + <fsummary>Convert from system to RFC 3339 timestamp.</fsummary> + <type name="offset"/> + <type name="rfc3339_string"/> + <type name="rfc3339_time_unit"/> + <desc> + <p>Converts a system time into RFC 3339 timestamp.</p> + <p>Valid options:</p> + <taglist> + <tag><c>{offset, Offset}</c></tag> + <item><p>The offset, either a string or an integer, to be + included in the formatted string. + An empty string, which is the default, is interpreted + as local time. A non-empty string is included as is. + The time unit of the integer is the same as the one + of <c><anno>Time</anno></c>.</p> + </item> + <tag><c>{time_designator, Character}</c></tag> + <item><p>The character used as time designator, that is, + the date and time separator. The default is <c>$T</c>.</p> + </item> + <tag><c>{unit, Unit}</c></tag> + <item><p>The time unit of <c><anno>Time</anno></c>. The + default is <c>second</c>. If some other unit is given + (<c>millisecond</c>, <c>microsecond</c>, or + <c>nanosecond</c>), the formatted string includes a + fraction of a second.</p> + </item> + </taglist> + <pre> +1> <input>calendar:system_time_to_rfc3339(erlang:system_time(second)).</input> +"2018-04-23T14:56:28+02:00" +2> <input>calendar:system_time_to_rfc3339(erlang:system_time(second), + [{offset, "-02:00"}]).</input> +"2018-04-23T10:56:52-02:00" +3> <input>calendar:system_time_to_rfc3339(erlang:system_time(second), + [{offset, -7200}]).</input> +"2018-04-23T10:57:05-02:00" +4> <input>calendar:system_time_to_rfc3339(erlang:system_time(millisecond), + [{unit, millisecond}, {time_designator, $\s}, {offset, "Z"}]).</input> +"2018-04-23 12:57:20.482Z"</pre> + </desc> + </func> + + <func> + <name name="system_time_to_universal_time" arity="2"/> + <fsummary>Convert system time to universal date and time.</fsummary> + <desc> + <p>Converts a specified system time into universal date and time.</p> + </desc> + </func> + + <func> <name name="time_difference" arity="2"/> <fsummary>Compute the difference between two times (deprecated). </fsummary> diff --git a/lib/stdlib/doc/src/erl_tar.xml b/lib/stdlib/doc/src/erl_tar.xml index 14c543ee2b..68fa071090 100644 --- a/lib/stdlib/doc/src/erl_tar.xml +++ b/lib/stdlib/doc/src/erl_tar.xml @@ -90,8 +90,8 @@ <section> <title>Other Storage Media</title> - <p>The <seealso marker="inets:ftp"><c>ftp</c></seealso> - module (Inets) normally accesses the tar file on disk using + <p>The <seealso marker="ftp:ftp"><c>ftp</c></seealso> + module normally accesses the tar file on disk using the <seealso marker="kernel:file"><c>file</c></seealso> module. When other needs arise, you can define your own low-level Erlang functions to perform the writing and reading on the storage media; diff --git a/lib/stdlib/doc/src/gb_sets.xml b/lib/stdlib/doc/src/gb_sets.xml index 7bfe477a11..2a3785dc27 100644 --- a/lib/stdlib/doc/src/gb_sets.xml +++ b/lib/stdlib/doc/src/gb_sets.xml @@ -83,6 +83,8 @@ </item> <item><seealso marker="#is_element/2"><c>is_element/2</c></seealso> </item> + <item><seealso marker="#is_empty/1"><c>is_empty/1</c></seealso> + </item> <item><seealso marker="#is_set/1"><c>is_set/1</c></seealso> </item> <item><seealso marker="#is_subset/2"><c>is_subset/2</c></seealso> diff --git a/lib/stdlib/doc/src/gen_statem.xml b/lib/stdlib/doc/src/gen_statem.xml index be0d64feba..e918e83df7 100644 --- a/lib/stdlib/doc/src/gen_statem.xml +++ b/lib/stdlib/doc/src/gen_statem.xml @@ -32,7 +32,68 @@ <modulesummary>Generic state machine behavior.</modulesummary> <description> <p> - This behavior module provides a state machine. Two + <c>gen_statem</c> provides a generic state machine behaviour + and replaces its predecessor + <seealso marker="gen_fsm"><c>gen_fsm</c></seealso> + since Erlang/OTP 20.0. + </p> + <p> + This reference manual describes types generated from the types + in the <c>gen_statem</c> source code, so they are correct. + However, the generated descriptions also reflect the type hiearchy, + which makes them kind of hard to read. + </p> + <p> + To get an overview of the concepts and operation of <c>gen_statem</c>, + do read the + <seealso marker="doc/design_principles:statem"> + <c>gen_statem</c> Behaviour + </seealso> + in + <seealso marker="doc/design_principles:users_guide"> + OTP Design Principles + </seealso> + which frequently links back to this reference manual to avoid + containing detailed facts that may rot by age. + </p> + <note> + <p> + This behavior appeared in Erlang/OTP 19.0. + In OTP 19.1 a backwards incompatible change of + the return tuple from + <seealso marker="#Module:init/1"><c>Module:init/1</c></seealso> + was made and the mandatory callback function + <seealso marker="#Module:callback_mode/0"> + <c>Module:callback_mode/0</c> + </seealso> + was introduced. In OTP 20.0 the + <seealso marker="#type-generic_timeout"><c>generic timeouts</c></seealso> + were added. + </p> + </note> + <p> + <c>gen_statem</c> has got the same features that + <seealso marker="gen_fsm"><c>gen_fsm</c></seealso> + had and adds some really useful: + </p> + <list type="bulleted"> + <item>Co-located state code</item> + <item>Arbitrary term state</item> + <item>Event postponing</item> + <item>Self-generated events</item> + <item>State time-out</item> + <item>Multiple generic named time-outs</item> + <item>Absolute time-out time</item> + <item>Automatic state enter calls</item> + <item> + Reply from other state than the request, <c>sys</c> traceable + </item> + <item>Multiple <c>sys</c> traceable replies</item> + </list> + + + <p> + Two <seealso marker="#type-callback_mode"><em>callback modes</em></seealso> are supported: </p> @@ -50,34 +111,6 @@ </p> </item> </list> - <note> - <p> - This is a new behavior in Erlang/OTP 19.0. - It has been thoroughly reviewed, is stable enough - to be used by at least two heavy OTP applications, - and is here to stay. - Depending on user feedback, we do not expect - but can find it necessary to make minor - not backward compatible changes into Erlang/OTP 20.0. - </p> - </note> - <p> - The <c>gen_statem</c> behavior replaces - <seealso marker="gen_fsm"><c>gen_fsm</c> </seealso> in Erlang/OTP 20.0. - It has the same features and adds some really useful: - </p> - <list type="bulleted"> - <item>Gathered state code.</item> - <item>Arbitrary term state.</item> - <item>Event postponing.</item> - <item>Self-generated events.</item> - <item>State time-out.</item> - <item>Multiple generic named time-outs.</item> - <item>Absolute time-out time.</item> - <item>Automatic state enter calls.</item> - <item>Reply from other state than the request.</item> - <item>Multiple <c>sys</c> traceable replies.</item> - </list> <p> The callback model(s) for <c>gen_statem</c> differs from the one for <seealso marker="gen_fsm"><c>gen_fsm</c></seealso>, @@ -148,7 +181,7 @@ erlang:'!' -----> Module:StateName/3 is <c>state_functions</c>, the state must be an atom and is used as the state callback name; see <seealso marker="#Module:StateName/3"><c>Module:StateName/3</c></seealso>. - This gathers all code for a specific state + This co-locates all code for a specific state in one function as the <c>gen_statem</c> engine branches depending on state name. Note the fact that the callback function @@ -207,8 +240,10 @@ erlang:'!' -----> Module:StateName/3 whenever a new state is entered; see <seealso marker="#type-state_enter"><c>state_enter()</c></seealso>. This is for writing code common to all state entries. - Another way to do it is to insert events at state transitions, - but you have to do so everywhere it is needed. + Another way to do it is to insert an event at the state transition, + and/or to use a dedicated state transition function, + but that is something you will have to remember + at every state transition to the state(s) that need it. </p> <note> <p>If you in <c>gen_statem</c>, for example, postpone @@ -252,6 +287,16 @@ erlang:'!' -----> Module:StateName/3 to use after every event; see <seealso marker="erts:erlang#hibernate/3"><c>erlang:hibernate/3</c></seealso>. </p> + <p> + There is also a server start option + <seealso marker="#type-hibernate_after_opt"> + <c>{hibernate_after, Timeout}</c> + </seealso> + for + <seealso marker="#start/3"><c>start/3,4</c></seealso> or + <seealso marker="#start_link/3"><c>start_link/3,4</c></seealso> + that may be used to automatically hibernate the server. + </p> </description> <section> @@ -668,9 +713,9 @@ handle_event(_, _, State, Data) -> <p> If <seealso marker="#Module:code_change/4"><c>Module:code_change/4</c></seealso> - should transform the state to a state with a different - name it is still regarded as the same state so this - does not cause a state enter call. + should transform the state, + it is regarded as a state rename and not a state change, + which will not cause a state enter call. </p> <p> Note that a state enter call <em>will</em> be done @@ -688,12 +733,19 @@ handle_event(_, _, State, Data) -> <p> Transition options can be set by <seealso marker="#type-action">actions</seealso> - and they modify how the state transition is done: + and modify the state transition. + Here are the sequence of steps for a state transition: </p> <list type="ordered"> <item> <p> - If the state changes, is the initial state, + If + <seealso marker="#type-state_enter"> + <em>state enter calls</em> + </seealso> + are used, and either: + the state changes, it is the initial state, + or one of the callback results <seealso marker="#type-state_callback_result"> <c>repeat_state</c> </seealso> @@ -701,16 +753,21 @@ handle_event(_, _, State, Data) -> <seealso marker="#type-state_callback_result"> <c>repeat_state_and_data</c> </seealso> - is used, and also - <seealso marker="#type-state_enter"><em>state enter calls</em></seealso> - are used, the <c>gen_statem</c> calls + is used; the <c>gen_statem</c> calls the new state callback with arguments - <seealso marker="#type-state_enter">(enter, OldState, Data)</seealso>. + <seealso marker="#type-state_enter"><c>(enter, OldState, Data)</c></seealso>. + </p> + <p> Any - <seealso marker="#type-enter_action"><c>actions</c></seealso> + <seealso marker="#type-enter_action">actions</seealso> returned from this call are handled as if they were - appended to the actions - returned by the state callback that changed states. + appended to the actions + returned by the state callback that caused the state entry. + </p> + <p> + Should this state enter call return any of + the mentioned <c>repeat_*</c> callback results + it is repeated again, with the updated <c>Data</c>. </p> </item> <item> @@ -739,7 +796,7 @@ handle_event(_, _, State, Data) -> All events stored with <seealso marker="#type-action"><c>action()</c></seealso> <c>next_event</c> - are inserted to be processed before the other queued events. + are inserted to be processed before previously queued events. </p> </item> <item> @@ -753,7 +810,9 @@ handle_event(_, _, State, Data) -> delivered to the state machine before any external not yet received event so if there is such a time-out requested, the corresponding time-out zero event is enqueued as - the newest event. + the newest received event; + that is after already queued events + such as inserted and postponed events. </p> <p> Any event cancels an @@ -791,7 +850,7 @@ handle_event(_, _, State, Data) -> When a new message arrives the <seealso marker="#state callback">state callback</seealso> is called with the corresponding event, - and we start again from the top of this list. + and we start again from the top of this sequence. </p> </item> </list> @@ -816,13 +875,19 @@ handle_event(_, _, State, Data) -> <seealso marker="proc_lib#hibernate/3"><c>proc_lib:hibernate/3</c></seealso> before going into <c>receive</c> to wait for a new external event. - If there are enqueued events, - to prevent receiving any new event, an - <seealso marker="erts:erlang#garbage_collect/0"><c>erlang:garbage_collect/0</c></seealso> - is done instead to simulate - that the <c>gen_statem</c> entered hibernation - and immediately got awakened by the oldest enqueued event. </p> + <note> + <p> + If there are enqueued events to process + when hibrnation is requested, + this is optimized by not hibernating but instead calling + <seealso marker="erts:erlang#garbage_collect/0"> + <c>erlang:garbage_collect/0</c> + </seealso> + to simulate that the <c>gen_statem</c> entered hibernation + and immediately got awakened by an enqueued event. + </p> + </note> </desc> </datatype> <datatype> @@ -857,7 +922,7 @@ handle_event(_, _, State, Data) -> no timer is actually started, instead the the time-out event is enqueued to ensure that it gets processed before any not yet - received external event. + received external event, but after already queued events. </p> <p> Note that it is not possible nor needed to cancel this time-out, @@ -943,7 +1008,9 @@ handle_event(_, _, State, Data) -> If <c>Abs</c> is <c>true</c> an absolute timer is started, and if it is <c>false</c> a relative, which is the default. See - <seealso marker="erts:erlang#start_timer/4"><c>erlang:start_timer/4</c></seealso> + <seealso marker="erts:erlang#start_timer/4"> + <c>erlang:start_timer/4</c> + </seealso> for details. </p> <p> @@ -969,7 +1036,9 @@ handle_event(_, _, State, Data) -> </p> <p> Actions that set - <seealso marker="#type-transition_option">transition options</seealso> + <seealso marker="#type-transition_option"> + transition options + </seealso> override any previous of the same type, so the last in the containing list wins. For example, the last @@ -981,7 +1050,9 @@ handle_event(_, _, State, Data) -> <item> <p> Sets the - <seealso marker="#type-transition_option"><c>transition_option()</c></seealso> + <seealso marker="#type-transition_option"> + <c>transition_option()</c> + </seealso> <seealso marker="#type-postpone"><c>postpone()</c></seealso> for this state transition. This action is ignored when returned from @@ -994,7 +1065,11 @@ handle_event(_, _, State, Data) -> <tag><c>next_event</c></tag> <item> <p> - Stores the specified <c><anno>EventType</anno></c> + This action does not set any + <seealso marker="#type-transition_option"> + <c>transition_option()</c> + </seealso> + but instead stores the specified <c><anno>EventType</anno></c> and <c><anno>EventContent</anno></c> for insertion after all actions have been executed. </p> @@ -1066,15 +1141,15 @@ handle_event(_, _, State, Data) -> <seealso marker="#type-transition_option">transition options</seealso>. </p> <taglist> - <tag><c>Timeout</c></tag> + <tag><c>Time</c></tag> <item> <p> - Short for <c>{timeout,Timeout,Timeout}</c>, that is, + Short for <c>{timeout,Time,Time}</c>, that is, the time-out message is the time-out time. This form exists to make the <seealso marker="#state callback">state callback</seealso> - return value <c>{next_state,NextState,NewData,Timeout}</c> - allowed like for <c>gen_fsm</c>'s + return value <c>{next_state,NextState,NewData,Time}</c> + allowed like for <c>gen_fsm</c>. </p> </item> <tag><c>timeout</c></tag> @@ -1126,7 +1201,11 @@ handle_event(_, _, State, Data) -> <seealso marker="#enter_loop/5"><c>enter_loop/5,6</c></seealso>. </p> <p> - It replies to a caller waiting for a reply in + It does not set any + <seealso marker="#type-transition_option"> + <c>transition_option()</c> + </seealso> + but instead replies to a caller waiting for a reply in <seealso marker="#call/2"><c>call/2</c></seealso>. <c><anno>From</anno></c> must be the term from argument <seealso marker="#type-event_type"><c>{call,<anno>From</anno>}</c></seealso> @@ -2109,16 +2188,20 @@ init(Args) -> erlang:error(not_implemented, [Args]).</pre> You may also not change states from this call. Should you return <c>{next_state,NextState, ...}</c> with <c>NextState =/= State</c> the <c>gen_statem</c> crashes. - It is possible to use <c>{repeat_state, ...}</c>, - <c>{repeat_state_and_data,_}</c> or - <c>repeat_state_and_data</c> but all of them makes little + Note that it is actually allowed to use + <c>{repeat_state, NewData, ...}</c> although it makes little sense since you immediately will be called again with a new <em>state enter call</em> making this just a weird way of looping, and there are better ways to loop in Erlang. + If you do not update <c>NewData</c> and have some + loop termination condition, or if you use + <c>{repeat_state_and_data, _}</c> or + <c>repeat_state_and_data</c> you have an infinite loop! You are advised to use <c>{keep_state,...}</c>, <c>{keep_state_and_data,_}</c> or - <c>keep_state_and_data</c> since you can not change states - from a <em>state enter call</em> anyway. + <c>keep_state_and_data</c> + since changing states from a <em>state enter call</em> + is not possible anyway. </p> <p> Note the fact that you can use diff --git a/lib/stdlib/doc/src/io_lib.xml b/lib/stdlib/doc/src/io_lib.xml index bc1d77ac83..4a2b425e8e 100644 --- a/lib/stdlib/doc/src/io_lib.xml +++ b/lib/stdlib/doc/src/io_lib.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2017</year> + <year>1996</year><year>2018</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -52,6 +52,9 @@ </desc> </datatype> <datatype> + <name name="chars_limit"/> + </datatype> + <datatype> <name name="depth"/> </datatype> <datatype> @@ -153,6 +156,27 @@ </func> <func> + <name name="format" arity="3"/> + <name name="fwrite" arity="3"/> + <fsummary>Write formatted output.</fsummary> + <desc> + <p>Returns a character list that represents <c><anno>Data</anno></c> + formatted in accordance with <c><anno>Format</anno></c> in + the same way as + <seealso marker="#fwrite/2"><c>fwrite/2</c></seealso> and + <seealso marker="#format/2"><c>format/2</c></seealso>, + but takes an extra argument, a list of options.</p> + <p>Available options:</p> + <taglist> + <tag><c><anno>CharsLimit</anno></c></tag> + <item> + <p>A soft limit on the number of characters returned.</p> + </item> + </taglist> + </desc> + </func> + + <func> <name name="fread" arity="2"/> <fsummary>Read formatted input.</fsummary> <desc> @@ -361,17 +385,24 @@ <fsummary>Write a term.</fsummary> <desc> <p>Returns a character list that represents <c><anno>Term</anno></c>. - Argument <c><anno>Depth</anno></c> controls the depth of the + Option <c><anno>Depth</anno></c> controls the depth of the structures written. When the specified depth is reached, everything below this level is replaced by "<c>...</c>". <c><anno>Depth</anno></c> defaults to -1, which means - no limitation.</p> + no limitation. Option <c><anno>CharsLimit</anno></c> puts a + soft limit on the number of characters returned. When the + number of characters is reached, remaining structures are + replaced by "<c>...</c>". <c><anno>CharsLimit</anno></c> + defaults to -1, which means no limit on the number of + characters returned.</p> <p><em>Example:</em></p> <pre> 1> <input>lists:flatten(io_lib:write({1,[2],[3],[4,5],6,7,8,9})).</input> "{1,[2],[3],[4,5],6,7,8,9}" 2> <input>lists:flatten(io_lib:write({1,[2],[3],[4,5],6,7,8,9}, 5)).</input> -"{1,[2],[3],[...],...}"</pre> +"{1,[2],[3],[...],...}" +3> <input>lists:flatten(io_lib:write({[1,2,3],[4,5],6,7,8,9}, [{chars_limit,20}])).</input> +"{[1,2|...],[4|...],...}"</pre> </desc> </func> diff --git a/lib/stdlib/doc/src/lists.xml b/lib/stdlib/doc/src/lists.xml index 7efafedc82..c3d5d7e07a 100644 --- a/lib/stdlib/doc/src/lists.xml +++ b/lib/stdlib/doc/src/lists.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2017</year> + <year>1996</year><year>2018</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -771,6 +771,18 @@ length(lists:seq(From, To, Incr)) =:= (To - From + Incr) div Incr</code> </func> <func> + <name name="search" arity="2"/> + <fsummary>Find the first element that satisfies a predicate.</fsummary> + <desc> + <p>If there is a <c><anno>Value</anno></c> in <c><anno>List</anno></c> + such that <c><anno>Pred</anno>(<anno>Value</anno>)</c> returns + <c>true</c>, returns <c>{value, <anno>Value</anno>}</c> + for the first such <c><anno>Value</anno></c>, + otherwise returns <c>false</c>.</p> + </desc> + </func> + + <func> <name name="splitwith" arity="2"/> <fsummary>Split a list into two lists based on a predicate.</fsummary> <desc> diff --git a/lib/stdlib/doc/src/notes.xml b/lib/stdlib/doc/src/notes.xml index bf6b06859e..e26c4aba74 100644 --- a/lib/stdlib/doc/src/notes.xml +++ b/lib/stdlib/doc/src/notes.xml @@ -31,6 +31,25 @@ </header> <p>This document describes the changes made to the STDLIB application.</p> +<section><title>STDLIB 3.4.5</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The <c>Module:init/1</c> function in <c>gen_statem</c> + may return an actions list containing any action, but an + erroneous check only allowed state enter actions so e.g + <c>{next_event,internal,event}</c> caused a server crash. + This bug has been fixed.</p> + <p> + Own Id: OTP-13995</p> + </item> + </list> + </section> + +</section> + <section><title>STDLIB 3.4.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/stdlib/doc/src/ordsets.xml b/lib/stdlib/doc/src/ordsets.xml index 7b590932e4..2d891d7a5a 100644 --- a/lib/stdlib/doc/src/ordsets.xml +++ b/lib/stdlib/doc/src/ordsets.xml @@ -142,6 +142,15 @@ </func> <func> + <name name="is_empty" arity="1"/> + <fsummary>Test for empty set.</fsummary> + <desc> + <p>Returns <c>true</c> if <c><anno>Ordset</anno></c> is an empty set, + otherwise <c>false</c>.</p> + </desc> + </func> + + <func> <name name="is_set" arity="1"/> <fsummary>Test for an <c>Ordset</c>.</fsummary> <desc> diff --git a/lib/stdlib/doc/src/sets.xml b/lib/stdlib/doc/src/sets.xml index 4934bed365..1ed96ddc3f 100644 --- a/lib/stdlib/doc/src/sets.xml +++ b/lib/stdlib/doc/src/sets.xml @@ -140,6 +140,15 @@ </func> <func> + <name name="is_empty" arity="1"/> + <fsummary>Test for empty set.</fsummary> + <desc> + <p>Returns <c>true</c> if <c><anno>Set</anno></c> is an empty set, + otherwise <c>false</c>.</p> + </desc> + </func> + + <func> <name name="is_set" arity="1"/> <fsummary>Test for a <c>Set</c>.</fsummary> <desc> diff --git a/lib/stdlib/doc/src/string.xml b/lib/stdlib/doc/src/string.xml index 130fc74a28..c7772d63a3 100644 --- a/lib/stdlib/doc/src/string.xml +++ b/lib/stdlib/doc/src/string.xml @@ -109,8 +109,10 @@ <p>This module has been reworked in Erlang/OTP 20 to handle <seealso marker="unicode#type-chardata"> <c>unicode:chardata()</c></seealso> and operate on grapheme - clusters. The <c>old functions</c> that only work on Latin-1 lists as input - are kept for backwards compatibility reasons but should not be used. + clusters. The <seealso marker="#oldapi"> <c>old + functions</c></seealso> that only work on Latin-1 lists as input + are still available but should not be used, they will be + deprecated in a future release. </p> </description> @@ -629,5 +631,393 @@ ÖÄÅ</pre> </func> </funcs> + + <section> + <marker id="oldapi"/> + <title>Obsolete API functions</title> + <p>Here follows the function of the old API. + These functions only work on a list of Latin-1 characters. + </p> + <note><p> + The functions are kept for backward compatibility, but are + not recommended. + They will be deprecated in Erlang/OTP 21. + </p> + <p>Any undocumented functions in <c>string</c> are not to be used.</p> + </note> + </section> + + <funcs> + <func> + <name name="centre" arity="2"/> + <name name="centre" arity="3"/> + <fsummary>Center a string.</fsummary> + <desc> + <p>Returns a string, where <c><anno>String</anno></c> is centered in the + string and surrounded by blanks or <c><anno>Character</anno></c>. + The resulting string has length <c><anno>Number</anno></c>.</p> + <p>This function is <seealso marker="#oldapi">obsolete</seealso>. + Use + <seealso marker="#pad/3"><c>pad/3</c></seealso>. + </p> + </desc> + </func> + + <func> + <name name="chars" arity="2"/> + <name name="chars" arity="3"/> + <fsummary>Return a string consisting of numbers of characters.</fsummary> + <desc> + <p>Returns a string consisting of <c><anno>Number</anno></c> characters + <c><anno>Character</anno></c>. Optionally, the string can end with + string <c><anno>Tail</anno></c>.</p> + <p>This function is <seealso marker="#oldapi">obsolete</seealso>. + Use + <seealso marker="lists#duplicate/2"><c>lists:duplicate/2</c></seealso>.</p> + </desc> + </func> + + <func> + <name name="chr" arity="2"/> + <fsummary>Return the index of the first occurrence of + a character in a string.</fsummary> + <desc> + <p>Returns the index of the first occurrence of + <c><anno>Character</anno></c> in <c><anno>String</anno></c>. Returns + <c>0</c> if <c><anno>Character</anno></c> does not occur.</p> + <p>This function is <seealso marker="#oldapi">obsolete</seealso>. + Use + <seealso marker="#find/2"><c>find/2</c></seealso>.</p> + </desc> + </func> + + <func> + <name name="concat" arity="2"/> + <fsummary>Concatenate two strings.</fsummary> + <desc> + <p>Concatenates <c><anno>String1</anno></c> and + <c><anno>String2</anno></c> to form a new string + <c><anno>String3</anno></c>, which is returned.</p> + <p> + This function is <seealso marker="#oldapi">obsolete</seealso>. + Use <c>[<anno>String1</anno>, <anno>String2</anno>]</c> as + <c>Data</c> argument, and call + <seealso marker="unicode#characters_to_list/2"> + <c>unicode:characters_to_list/2</c></seealso> or + <seealso marker="unicode#characters_to_binary/2"> + <c>unicode:characters_to_binary/2</c></seealso> + to flatten the output. + </p> + </desc> + </func> + + <func> + <name name="copies" arity="2"/> + <fsummary>Copy a string.</fsummary> + <desc> + <p>Returns a string containing <c><anno>String</anno></c> repeated + <c><anno>Number</anno></c> times.</p> + <p>This function is <seealso marker="#oldapi">obsolete</seealso>. + Use + <seealso marker="lists#duplicate/2"><c>lists:duplicate/2</c></seealso>.</p> + </desc> + </func> + + <func> + <name name="cspan" arity="2"/> + <fsummary>Span characters at start of a string.</fsummary> + <desc> + <p>Returns the length of the maximum initial segment of + <c><anno>String</anno></c>, which consists entirely of characters + not from <c><anno>Chars</anno></c>.</p> + <p>This function is <seealso marker="#oldapi">obsolete</seealso>. + Use + <seealso marker="#take/3"><c>take/3</c></seealso>.</p> + <p><em>Example:</em></p> + <code type="none"> +> string:cspan("\t abcdef", " \t"). +0</code> + </desc> + </func> + + <func> + <name name="join" arity="2"/> + <fsummary>Join a list of strings with separator.</fsummary> + <desc> + <p>Returns a string with the elements of <c><anno>StringList</anno></c> + separated by the string in <c><anno>Separator</anno></c>.</p> + <p>This function is <seealso marker="#oldapi">obsolete</seealso>. + Use + <seealso marker="lists#join/2"><c>lists:join/2</c></seealso>.</p> + <p><em>Example:</em></p> + <code type="none"> +> join(["one", "two", "three"], ", "). +"one, two, three"</code> + </desc> + </func> + + <func> + <name name="left" arity="2"/> + <name name="left" arity="3"/> + <fsummary>Adjust left end of a string.</fsummary> + <desc> + <p>Returns <c><anno>String</anno></c> with the length adjusted in + accordance with <c><anno>Number</anno></c>. The left margin is + fixed. If <c>length(<anno>String</anno>)</c> < + <c><anno>Number</anno></c>, then <c><anno>String</anno></c> is padded + with blanks or <c><anno>Character</anno></c>s.</p> + <p>This function is <seealso marker="#oldapi">obsolete</seealso>. + Use + <seealso marker="#pad/2"><c>pad/2</c></seealso> or + <seealso marker="#pad/3"><c>pad/3</c></seealso>.</p> + <p><em>Example:</em></p> + <code type="none"> +> string:left("Hello",10,$.). +"Hello....."</code> + </desc> + </func> + + <func> + <name name="len" arity="1"/> + <fsummary>Return the length of a string.</fsummary> + <desc> + <p>Returns the number of characters in <c><anno>String</anno></c>.</p> + <p>This function is <seealso marker="#oldapi">obsolete</seealso>. + Use + <seealso marker="#length/1"><c>length/1</c></seealso>.</p> + </desc> + </func> + + <func> + <name name="rchr" arity="2"/> + <fsummary>Return the index of the last occurrence of + a character in a string.</fsummary> + <desc> + <p>Returns the index of the last occurrence of + <c><anno>Character</anno></c> in <c><anno>String</anno></c>. Returns + <c>0</c> if <c><anno>Character</anno></c> does not occur.</p> + <p>This function is <seealso marker="#oldapi">obsolete</seealso>. + Use + <seealso marker="#find/3"><c>find/3</c></seealso>.</p> + </desc> + </func> + + <func> + <name name="right" arity="2"/> + <name name="right" arity="3"/> + <fsummary>Adjust right end of a string.</fsummary> + <desc> + <p>Returns <c><anno>String</anno></c> with the length adjusted in + accordance with <c><anno>Number</anno></c>. The right margin is + fixed. If the length of <c>(<anno>String</anno>)</c> < + <c><anno>Number</anno></c>, then <c><anno>String</anno></c> is padded + with blanks or <c><anno>Character</anno></c>s.</p> + <p>This function is <seealso marker="#oldapi">obsolete</seealso>. + Use + <seealso marker="#pad/3"><c>pad/3</c></seealso>.</p> + <p><em>Example:</em></p> + <code type="none"> +> string:right("Hello", 10, $.). +".....Hello"</code> + </desc> + </func> + + <func> + <name name="rstr" arity="2"/> + <fsummary>Find the index of a substring.</fsummary> + <desc> + <p>Returns the position where the last occurrence of + <c><anno>SubString</anno></c> begins in <c><anno>String</anno></c>. + Returns <c>0</c> if <c><anno>SubString</anno></c> + does not exist in <c><anno>String</anno></c>.</p> + <p>This function is <seealso marker="#oldapi">obsolete</seealso>. + Use + <seealso marker="#find/3"><c>find/3</c></seealso>.</p> + <p><em>Example:</em></p> + <code type="none"> +> string:rstr(" Hello Hello World World ", "Hello World"). +8</code> + </desc> + </func> + + <func> + <name name="span" arity="2"/> + <fsummary>Span characters at start of a string.</fsummary> + <desc> + <p>Returns the length of the maximum initial segment of + <c><anno>String</anno></c>, which consists entirely of characters + from <c><anno>Chars</anno></c>.</p> + <p>This function is <seealso marker="#oldapi">obsolete</seealso>. + Use + <seealso marker="#take/2"><c>take/2</c></seealso>.</p> + <p><em>Example:</em></p> + <code type="none"> +> string:span("\t abcdef", " \t"). +5</code> + </desc> + </func> + + <func> + <name name="str" arity="2"/> + <fsummary>Find the index of a substring.</fsummary> + <desc> + <p>Returns the position where the first occurrence of + <c><anno>SubString</anno></c> begins in <c><anno>String</anno></c>. + Returns <c>0</c> if <c><anno>SubString</anno></c> + does not exist in <c><anno>String</anno></c>.</p> + <p>This function is <seealso marker="#oldapi">obsolete</seealso>. + Use + <seealso marker="#find/2"><c>find/2</c></seealso>.</p> + <p><em>Example:</em></p> + <code type="none"> +> string:str(" Hello Hello World World ", "Hello World"). +8</code> + </desc> + </func> + + <func> + <name name="strip" arity="1"/> + <name name="strip" arity="2"/> + <name name="strip" arity="3"/> + <fsummary>Strip leading or trailing characters.</fsummary> + <desc> + <p>Returns a string, where leading or trailing, or both, blanks or a + number of <c><anno>Character</anno></c> have been removed. + <c><anno>Direction</anno></c>, which can be <c>left</c>, <c>right</c>, + or <c>both</c>, indicates from which direction blanks are to be + removed. <c>strip/1</c> is equivalent to + <c>strip(String, both)</c>.</p> + <p>This function is <seealso marker="#oldapi">obsolete</seealso>. + Use + <seealso marker="#trim/3"><c>trim/3</c></seealso>.</p> + <p><em>Example:</em></p> + <code type="none"> +> string:strip("...Hello.....", both, $.). +"Hello"</code> + </desc> + </func> + + <func> + <name name="sub_string" arity="2"/> + <name name="sub_string" arity="3"/> + <fsummary>Extract a substring.</fsummary> + <desc> + <p>Returns a substring of <c><anno>String</anno></c>, starting at + position <c><anno>Start</anno></c> to the end of the string, or to + and including position <c><anno>Stop</anno></c>.</p> + <p>This function is <seealso marker="#oldapi">obsolete</seealso>. + Use + <seealso marker="#slice/3"><c>slice/3</c></seealso>.</p> + <p><em>Example:</em></p> + <code type="none"> +sub_string("Hello World", 4, 8). +"lo Wo"</code> + </desc> + </func> + + <func> + <name name="substr" arity="2"/> + <name name="substr" arity="3"/> + <fsummary>Return a substring of a string.</fsummary> + <desc> + <p>Returns a substring of <c><anno>String</anno></c>, starting at + position <c><anno>Start</anno></c>, and ending at the end of the + string or at length <c><anno>Length</anno></c>.</p> + <p>This function is <seealso marker="#oldapi">obsolete</seealso>. + Use + <seealso marker="#slice/3"><c>slice/3</c></seealso>.</p> + <p><em>Example:</em></p> + <code type="none"> +> substr("Hello World", 4, 5). +"lo Wo"</code> + </desc> + </func> + + <func> + <name name="sub_word" arity="2"/> + <name name="sub_word" arity="3"/> + <fsummary>Extract subword.</fsummary> + <desc> + <p>Returns the word in position <c><anno>Number</anno></c> of + <c><anno>String</anno></c>. Words are separated by blanks or + <c><anno>Character</anno></c>s.</p> + <p>This function is <seealso marker="#oldapi">obsolete</seealso>. + Use + <seealso marker="#nth_lexeme/3"><c>nth_lexeme/3</c></seealso>.</p> + <p><em>Example:</em></p> + <code type="none"> +> string:sub_word(" Hello old boy !",3,$o). +"ld b"</code> + </desc> + </func> + + <func> + <name name="to_lower" arity="1" clause_i="1"/> + <name name="to_lower" arity="1" clause_i="2"/> + <name name="to_upper" arity="1" clause_i="1"/> + <name name="to_upper" arity="1" clause_i="2"/> + <fsummary>Convert case of string (ISO/IEC 8859-1).</fsummary> + <type variable="String" name_i="1"/> + <type variable="Result" name_i="1"/> + <type variable="Char"/> + <type variable="CharResult"/> + <desc> + <p>The specified string or character is case-converted. Notice that + the supported character set is ISO/IEC 8859-1 (also called Latin 1); + all values outside this set are unchanged</p> + <p>This function is <seealso marker="#oldapi">obsolete</seealso> use + <seealso marker="#lowercase/1"><c>lowercase/1</c></seealso>, + <seealso marker="#uppercase/1"><c>uppercase/1</c></seealso>, + <seealso marker="#titlecase/1"><c>titlecase/1</c></seealso> or + <seealso marker="#casefold/1"><c>casefold/1</c></seealso>.</p> + </desc> + </func> + + <func> + <name name="tokens" arity="2"/> + <fsummary>Split string into tokens.</fsummary> + <desc> + <p>Returns a list of tokens in <c><anno>String</anno></c>, separated + by the characters in <c><anno>SeparatorList</anno></c>.</p> + <p><em>Example:</em></p> + <code type="none"> +> tokens("abc defxxghix jkl", "x "). +["abc", "def", "ghi", "jkl"]</code> + <p>Notice that, as shown in this example, two or more + adjacent separator characters in <c><anno>String</anno></c> + are treated as one. That is, there are no empty + strings in the resulting list of tokens.</p> + <p>This function is <seealso marker="#oldapi">obsolete</seealso>. + Use + <seealso marker="#lexemes/2"><c>lexemes/2</c></seealso>.</p> + </desc> + </func> + + <func> + <name name="words" arity="1"/> + <name name="words" arity="2"/> + <fsummary>Count blank separated words.</fsummary> + <desc> + <p>Returns the number of words in <c><anno>String</anno></c>, separated + by blanks or <c><anno>Character</anno></c>.</p> + <p>This function is <seealso marker="#oldapi">obsolete</seealso>. + Use + <seealso marker="#lexemes/2"><c>lexemes/2</c></seealso>.</p> + <p><em>Example:</em></p> + <code type="none"> +> words(" Hello old boy!", $o). +4</code> + </desc> + </func> + </funcs> + + <section> + <title>Notes</title> + <p>Some of the general string functions can seem to overlap each + other. The reason is that this string package is the + combination of two earlier packages and all functions of + both packages have been retained.</p> + </section> + </erlref> diff --git a/lib/stdlib/doc/src/sys.xml b/lib/stdlib/doc/src/sys.xml index 64d8789016..59e5bb6cb5 100644 --- a/lib/stdlib/doc/src/sys.xml +++ b/lib/stdlib/doc/src/sys.xml @@ -102,7 +102,7 @@ then treated in the debug function. For example, <c>trace</c> formats the system events to the terminal. </p> - <p>Three predefined system events are used when a + <p>Four predefined system events are used when a process receives or sends a message. The process can also define its own system events. It is always up to the process itself to format these events.</p> @@ -276,7 +276,9 @@ <p><c><anno>Func</anno></c> is called whenever a system event is generated. This function is to return <c>done</c>, or a new <c>Func</c> state. In the first case, the function is removed. It is - also removed if the function fails.</p> + also removed if the function fails. If one debug function should be + installed more times, a unique <c><anno>FuncId</anno></c> must be + specified for each installation.</p> </desc> </func> @@ -330,8 +332,8 @@ <fsummary>Remove a debug function from the process.</fsummary> <desc> <p>Removes an installed debug function from the - process. <c><anno>Func</anno></c> must be the same as previously - installed.</p> + process. <c><anno>Func</anno></c> or <c><anno>FuncId</anno></c> must be + the same as previously installed.</p> </desc> </func> diff --git a/lib/stdlib/doc/src/timer.xml b/lib/stdlib/doc/src/timer.xml index 350847bf7d..53107ade2c 100644 --- a/lib/stdlib/doc/src/timer.xml +++ b/lib/stdlib/doc/src/timer.xml @@ -270,8 +270,8 @@ <item> <p>Evaluates <c>apply(<anno>Module</anno>, <anno>Function</anno>, <anno>Arguments</anno>)</c> and measures the elapsed real time as - reported by <seealso marker="kernel:os#timestamp/0"> - <c>os:timestamp/0</c></seealso>.</p> + reported by <seealso marker="erts:erlang#monotonic_time/0"> + <c>erlang:monotonic_time/0</c></seealso>.</p> <p>Returns <c>{<anno>Time</anno>, <anno>Value</anno>}</c>, where <c><anno>Time</anno></c> is the elapsed real time in <em>microseconds</em>, and <c><anno>Value</anno></c> is what is diff --git a/lib/stdlib/src/Makefile b/lib/stdlib/src/Makefile index 8b156929d7..dc3735055a 100644 --- a/lib/stdlib/src/Makefile +++ b/lib/stdlib/src/Makefile @@ -238,6 +238,13 @@ $(EBIN)/erl_tar.beam: ../../kernel/include/file.hrl erl_tar.hrl $(EBIN)/file_sorter.beam: ../../kernel/include/file.hrl $(EBIN)/filelib.beam: ../../kernel/include/file.hrl $(EBIN)/filename.beam: ../../kernel/include/file.hrl +$(EBIN)/gen_event.beam: ../../kernel/include/logger.hrl +$(EBIN)/gen_fsm.beam: ../../kernel/include/logger.hrl +$(EBIN)/gen_server.beam: ../../kernel/include/logger.hrl +$(EBIN)/gen_statem.beam: ../../kernel/include/logger.hrl +$(EBIN)/proc_lib.beam: ../../kernel/include/logger.hrl $(EBIN)/qlc_pt.beam: ../include/ms_transform.hrl $(EBIN)/shell.beam: ../../kernel/include/file.hrl +$(EBIN)/supervisor.beam: ../../kernel/include/logger.hrl +$(EBIN)/supervisor_bridge.beam: ../../kernel/include/logger.hrl $(EBIN)/zip.beam: ../include/zip.hrl ../../kernel/include/file.hrl diff --git a/lib/stdlib/src/calendar.erl b/lib/stdlib/src/calendar.erl index 55a0cfc9a1..9a600c1972 100644 --- a/lib/stdlib/src/calendar.erl +++ b/lib/stdlib/src/calendar.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -39,8 +39,14 @@ now_to_datetime/1, % = now_to_universal_time/1 now_to_local_time/1, now_to_universal_time/1, + rfc3339_to_system_time/1, + rfc3339_to_system_time/2, seconds_to_daystime/1, seconds_to_time/1, + system_time_to_local_time/2, + system_time_to_universal_time/2, + system_time_to_rfc3339/1, + system_time_to_rfc3339/2, time_difference/2, time_to_seconds/1, universal_time/0, @@ -55,10 +61,13 @@ -define(SECONDS_PER_DAY, 86400). -define(DAYS_PER_YEAR, 365). -define(DAYS_PER_LEAP_YEAR, 366). --define(DAYS_PER_4YEARS, 1461). --define(DAYS_PER_100YEARS, 36524). --define(DAYS_PER_400YEARS, 146097). +%% -define(DAYS_PER_4YEARS, 1461). +%% -define(DAYS_PER_100YEARS, 36524). +%% -define(DAYS_PER_400YEARS, 146097). -define(DAYS_FROM_0_TO_1970, 719528). +-define(DAYS_FROM_0_TO_10000, 2932897). +-define(SECONDS_FROM_0_TO_1970, (?DAYS_FROM_0_TO_1970*?SECONDS_PER_DAY)). +-define(SECONDS_FROM_0_TO_10000, (?DAYS_FROM_0_TO_10000*?SECONDS_PER_DAY)). %%---------------------------------------------------------------------- %% Types @@ -83,6 +92,13 @@ -type datetime1970() :: {{year1970(),month(),day()},time()}. -type yearweeknum() :: {year(),weeknum()}. +-type rfc3339_string() :: [byte(), ...]. +%% By design 'native' is not supported: +-type rfc3339_time_unit() :: 'microsecond' + | 'millisecond' + | 'nanosecond' + | 'second'. + %%---------------------------------------------------------------------- %% All dates are according the the Gregorian calendar. In this module @@ -309,8 +325,7 @@ local_time_to_universal_time_dst(DateTime) -> -spec now_to_datetime(Now) -> datetime1970() when Now :: erlang:timestamp(). now_to_datetime({MSec, Sec, _uSec}) -> - Sec0 = MSec*1000000 + Sec + ?DAYS_FROM_0_TO_1970*?SECONDS_PER_DAY, - gregorian_seconds_to_datetime(Sec0). + system_time_to_datetime(MSec*1000000 + Sec). -spec now_to_universal_time(Now) -> datetime1970() when Now :: erlang:timestamp(). @@ -328,6 +343,33 @@ now_to_local_time({MSec, Sec, _uSec}) -> erlang:universaltime_to_localtime( now_to_universal_time({MSec, Sec, _uSec})). +-spec rfc3339_to_system_time(DateTimeString) -> integer() when + DateTimeString :: rfc3339_string(). + +rfc3339_to_system_time(DateTimeString) -> + rfc3339_to_system_time(DateTimeString, []). + +-spec rfc3339_to_system_time(DateTimeString, Options) -> integer() when + DateTimeString :: rfc3339_string(), + Options :: [Option], + Option :: {'unit', rfc3339_time_unit()}. + +rfc3339_to_system_time(DateTimeString, Options) -> + Unit = proplists:get_value(unit, Options, second), + %% _T is the character separating the date and the time: + {DateStr, [_T|TimeStr]} = lists:split(10, DateTimeString), + {TimeStr2, TimeStr3} = lists:split(8, TimeStr), + {ok, [Hour, Min, Sec], []} = io_lib:fread("~d:~d:~d", TimeStr2), + {ok, [Year, Month, Day], []} = io_lib:fread("~d-~d-~d", DateStr), + DateTime = {{Year, Month, Day}, {Hour, Min, Sec}}, + IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end, + {FractionStr, UtcOffset} = lists:splitwith(IsFractionChar, TimeStr3), + Time = datetime_to_system_time(DateTime), + Secs = Time - offset_adjustment(Time, second, UtcOffset), + check(DateTimeString, Options, Secs), + ScaledEpoch = erlang:convert_time_unit(Secs, second, Unit), + ScaledEpoch + copy_sign(fraction(Unit, FractionStr), ScaledEpoch). + %% seconds_to_daystime(Secs) = {Days, {Hour, Minute, Second}} @@ -363,6 +405,55 @@ seconds_to_time(Secs) when Secs >= 0, Secs < ?SECONDS_PER_DAY -> Second = Secs1 rem ?SECONDS_PER_MINUTE, {Hour, Minute, Second}. +-spec system_time_to_local_time(Time, TimeUnit) -> datetime() when + Time :: integer(), + TimeUnit :: erlang:time_unit(). + +system_time_to_local_time(Time, TimeUnit) -> + UniversalDate = system_time_to_universal_time(Time, TimeUnit), + erlang:universaltime_to_localtime(UniversalDate). + +-spec system_time_to_universal_time(Time, TimeUnit) -> datetime() when + Time :: integer(), + TimeUnit :: erlang:time_unit(). + +system_time_to_universal_time(Time, TimeUnit) -> + Secs = erlang:convert_time_unit(Time, TimeUnit, second), + system_time_to_datetime(Secs). + +-spec system_time_to_rfc3339(Time) -> DateTimeString when + Time :: integer(), + DateTimeString :: rfc3339_string(). + +system_time_to_rfc3339(Time) -> + system_time_to_rfc3339(Time, []). + +-type offset() :: [byte()] | (Time :: integer()). +-spec system_time_to_rfc3339(Time, Options) -> DateTimeString when + Time :: integer(), % Since Epoch + Options :: [Option], + Option :: {'offset', offset()} + | {'time_designator', byte()} + | {'unit', rfc3339_time_unit()}, + DateTimeString :: rfc3339_string(). + +system_time_to_rfc3339(Time, Options) -> + Unit = proplists:get_value(unit, Options, second), + OffsetOption = proplists:get_value(offset, Options, ""), + T = proplists:get_value(time_designator, Options, $T), + AdjustmentSecs = offset_adjustment(Time, Unit, OffsetOption), + Offset = offset(OffsetOption, AdjustmentSecs), + Adjustment = erlang:convert_time_unit(AdjustmentSecs, second, Unit), + AdjustedTime = Time + Adjustment, + Factor = factor(Unit), + Secs = AdjustedTime div Factor, + check(Time, Options, Secs), + DateTime = system_time_to_datetime(Secs), + {{Year, Month, Day}, {Hour, Min, Sec}} = DateTime, + FractionStr = fraction_str(Factor, AdjustedTime), + flat_fwrite("~4.10.0B-~2.10.0B-~2.10.0B~c~2.10.0B:~2.10.0B:~2.10.0B~s~s", + [Year, Month, Day, T, Hour, Min, Sec, FractionStr, Offset]). + %% time_difference(T1, T2) = Tdiff %% %% Returns the difference between two {Date, Time} structures. @@ -550,3 +641,85 @@ df(Year, _) -> true -> 1; false -> 0 end. + +check(_Arg, _Options, Secs) when Secs >= - ?SECONDS_FROM_0_TO_1970, + Secs < ?SECONDS_FROM_0_TO_10000 -> + ok; +check(Arg, Options, _Secs) -> + erlang:error({badarg, [Arg, Options]}). + +datetime_to_system_time(DateTime) -> + datetime_to_gregorian_seconds(DateTime) - ?SECONDS_FROM_0_TO_1970. + +system_time_to_datetime(Seconds) -> + gregorian_seconds_to_datetime(Seconds + ?SECONDS_FROM_0_TO_1970). + +offset(OffsetOption, Secs0) when OffsetOption =:= ""; + is_integer(OffsetOption) -> + Sign = case Secs0 < 0 of + true -> $-; + false -> $+ + end, + Secs = abs(Secs0), + Hour = Secs div 3600, + Min = (Secs rem 3600) div 60, + io_lib:fwrite("~c~2.10.0B:~2.10.0B", [Sign, Hour, Min]); +offset(OffsetOption, _Secs) -> + OffsetOption. + +offset_adjustment(Time, Unit, OffsetString) when is_list(OffsetString) -> + offset_string_adjustment(Time, Unit, OffsetString); +offset_adjustment(_Time, Unit, Offset) when is_integer(Offset) -> + erlang:convert_time_unit(Offset, Unit, second). + +offset_string_adjustment(Time, Unit, "") -> + local_offset(Time, Unit); +offset_string_adjustment(_Time, _Unit, "Z") -> + 0; +offset_string_adjustment(_Time, _Unit, "z") -> + 0; +offset_string_adjustment(_Time, _Unit, [Sign|Tz]) -> + {ok, [Hour, Min], []} = io_lib:fread("~d:~d", Tz), + Adjustment = 3600 * Hour + 60 * Min, + case Sign of + $- -> -Adjustment; + $+ -> Adjustment + end. + +local_offset(SystemTime, Unit) -> + LocalTime = system_time_to_local_time(SystemTime, Unit), + UniversalTime = system_time_to_universal_time(SystemTime, Unit), + LocalSecs = datetime_to_gregorian_seconds(LocalTime), + UniversalSecs = datetime_to_gregorian_seconds(UniversalTime), + LocalSecs - UniversalSecs. + +fraction_str(Factor, Time) -> + case Time rem Factor of + 0 -> + ""; + Fraction -> + FS = io_lib:fwrite(".~*..0B", [log10(Factor), abs(Fraction)]), + string:trim(FS, trailing, "0") + end. + +fraction(second, _) -> + 0; +fraction(_, "") -> + 0; +fraction(Unit, FractionStr) -> + round(factor(Unit) * list_to_float([$0|FractionStr])). + +copy_sign(N1, N2) when N2 < 0 -> -N1; +copy_sign(N1, _N2) -> N1. + +factor(second) -> 1; +factor(millisecond) -> 1000; +factor(microsecond) -> 1000000; +factor(nanosecond) -> 1000000000. + +log10(1000) -> 3; +log10(1000000) -> 6; +log10(1000000000) -> 9. + +flat_fwrite(F, S) -> + lists:flatten(io_lib:fwrite(F, S)). diff --git a/lib/stdlib/src/erl_internal.erl b/lib/stdlib/src/erl_internal.erl index 89b97b901e..6d3d5baa23 100644 --- a/lib/stdlib/src/erl_internal.erl +++ b/lib/stdlib/src/erl_internal.erl @@ -76,6 +76,7 @@ guard_bif(floor, 1) -> true; guard_bif(hd, 1) -> true; guard_bif(length, 1) -> true; guard_bif(map_size, 1) -> true; +guard_bif(map_get, 2) -> true; guard_bif(node, 0) -> true; guard_bif(node, 1) -> true; guard_bif(round, 1) -> true; @@ -337,6 +338,7 @@ bif(list_to_tuple, 1) -> true; bif(load_module, 2) -> true; bif(make_ref, 0) -> true; bif(map_size,1) -> true; +bif(map_get,2) -> true; bif(max,2) -> true; bif(min,2) -> true; bif(module_loaded, 1) -> true; diff --git a/lib/stdlib/src/erl_lint.erl b/lib/stdlib/src/erl_lint.erl index 9a62d21d34..e9ac2fcdff 100644 --- a/lib/stdlib/src/erl_lint.erl +++ b/lib/stdlib/src/erl_lint.erl @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2017. All Rights Reserved. +%% Copyright Ericsson AB 1996-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -93,13 +93,6 @@ value_option(Flag, Default, On, OnVal, Off, OffVal, Opts) -> }). -%% Are we outside or inside a catch or try/catch? --type catch_scope() :: 'none' - | 'after_old_catch' - | 'after_try' - | 'wrong_part_of_try' - | 'try_catch'. - %% Define the lint state record. %% 'called' and 'exports' contain {Line, {Function, Arity}}, %% the other function collections contain {Function, Arity}. @@ -144,9 +137,7 @@ value_option(Flag, Default, On, OnVal, Off, OffVal, Opts) -> :: dict:dict(ta(), #typeinfo{}), exp_types=gb_sets:empty() %Exported types :: gb_sets:set(ta()), - in_try_head=false :: boolean(), %In a try head. - catch_scope = none %Inside/outside try or catch - :: catch_scope() + in_try_head=false :: boolean() %In a try head. }). -type lint_state() :: #lint{}. @@ -233,15 +224,6 @@ format_error({redefine_old_bif_import,{F,A}}) -> format_error({redefine_bif_import,{F,A}}) -> io_lib:format("import directive overrides auto-imported BIF ~w/~w~n" " - use \"-compile({no_auto_import,[~w/~w]}).\" to resolve name clash", [F,A,F,A]); -format_error({get_stacktrace,wrong_part_of_try}) -> - "erlang:get_stacktrace/0 used in the wrong part of 'try' expression. " - "(Use it in the block between 'catch' and 'end'.)"; -format_error({get_stacktrace,after_old_catch}) -> - "erlang:get_stacktrace/0 used following an old-style 'catch' " - "may stop working in a future release. (Use it inside 'try'.)"; -format_error({get_stacktrace,after_try}) -> - "erlang:get_stacktrace/0 used following a 'try' expression " - "may stop working in a future release. (Use it inside 'try'.)"; format_error({deprecated, MFA, ReplacementMFA, Rel}) -> io_lib:format("~s is deprecated and will be removed in ~s; use ~s", [format_mfa(MFA), Rel, format_mfa(ReplacementMFA)]); @@ -591,10 +573,7 @@ start(File, Opts) -> false, Opts)}, {missing_spec_all, bool_option(warn_missing_spec_all, nowarn_missing_spec_all, - false, Opts)}, - {get_stacktrace, - bool_option(warn_get_stacktrace, nowarn_get_stacktrace, - true, Opts)} + false, Opts)} ], Enabled1 = [Category || {Category,true} <- Enabled0], Enabled = ordsets:from_list(Enabled1), @@ -1426,7 +1405,7 @@ call_function(Line, F, A, #lint{usage=Usage0,called=Cd,func=Func,file=File}=St) %% function(Line, Name, Arity, Clauses, State) -> State. function(Line, Name, Arity, Cs, St0) -> - St1 = St0#lint{func={Name,Arity},catch_scope=none}, + St1 = St0#lint{func={Name,Arity}}, St2 = define_function(Line, Name, Arity, St1), clauses(Cs, St2). @@ -2116,6 +2095,10 @@ is_gexpr({cons,_L,H,T}, Info) -> is_gexpr_list([H,T], Info); is_gexpr({tuple,_L,Es}, Info) -> is_gexpr_list(Es, Info); %%is_gexpr({struct,_L,_Tag,Es}, Info) -> %% is_gexpr_list(Es, Info); +is_gexpr({map,_L,Es}, Info) -> + is_map_fields(Es, Info); +is_gexpr({map,_L,Src,Es}, Info) -> + is_gexpr(Src, Info) andalso is_map_fields(Es, Info); is_gexpr({record_index,_L,_Name,Field}, Info) -> is_gexpr(Field, Info); is_gexpr({record_field,_L,Rec,_Name,Field}, Info) -> @@ -2158,6 +2141,14 @@ is_gexpr_op(Op, A) -> is_gexpr_list(Es, Info) -> all(fun (E) -> is_gexpr(E, Info) end, Es). +is_map_fields([{Tag,_,K,V}|Fs], Info) when Tag =:= map_field_assoc; + Tag =:= map_field_exact -> + is_gexpr(K, Info) andalso + is_gexpr(V, Info) andalso + is_map_fields(Fs, Info); +is_map_fields([], _Info) -> true; +is_map_fields(_T, _Info) -> false. + is_gexpr_fields(Fs, L, Name, {RDs,_}=Info) -> IFs = case dict:find(Name, RDs) of {ok,{_Line,Fields}} -> Fs ++ init_fields(Fs, L, Fields); @@ -2367,7 +2358,7 @@ expr({call,Line,F,As}, Vt, St0) -> expr({'try',Line,Es,Scs,Ccs,As}, Vt, St0) -> %% Currently, we don't allow any exports because later %% passes cannot handle exports in combination with 'after'. - {Evt0,St1} = exprs(Es, Vt, St0#lint{catch_scope=wrong_part_of_try}), + {Evt0,St1} = exprs(Es, Vt, St0), TryLine = {'try',Line}, Uvt = vtunsafe(TryLine, Evt0, Vt), Evt1 = vtupdate(Uvt, Evt0), @@ -2379,12 +2370,11 @@ expr({'try',Line,Es,Scs,Ccs,As}, Vt, St0) -> {Avt0,St} = exprs(As, vtupdate(Evt2, Vt), St2), Avt1 = vtupdate(vtunsafe(TryLine, Avt0, Vt), Avt0), Avt = vtmerge(Evt2, Avt1), - {Avt,St#lint{catch_scope=after_try}}; + {Avt,St}; expr({'catch',Line,E}, Vt, St0) -> %% No new variables added, flag new variables as unsafe. {Evt,St} = expr(E, Vt, St0), - {vtupdate(vtunsafe({'catch',Line}, Evt, Vt), Evt), - St#lint{catch_scope=after_old_catch}}; + {vtupdate(vtunsafe({'catch',Line}, Evt, Vt), Evt),St}; expr({match,_Line,P,E}, Vt, St0) -> {Evt,St1} = expr(E, Vt, St0), {Pvt,Bvt,St2} = pattern(P, vtupdate(Evt, Vt), St1), @@ -3223,7 +3213,7 @@ is_module_dialyzer_option(Option) -> try_clauses(Scs, Ccs, In, Vt, St0) -> {Csvt0,St1} = icrt_clauses(Scs, Vt, St0), - St2 = St1#lint{catch_scope=try_catch,in_try_head=true}, + St2 = St1#lint{in_try_head=true}, {Csvt1,St3} = icrt_clauses(Ccs, Vt, St2), Csvt = Csvt0 ++ Csvt1, UpdVt = icrt_export(Csvt, Vt, In, St3), @@ -3243,7 +3233,7 @@ icrt_clauses(Cs, In, Vt, St0) -> icrt_clauses(Cs, Vt, St) -> mapfoldl(fun (C, St0) -> icrt_clause(C, Vt, St0) end, St, Cs). -icrt_clause({clause,_Line,H,G,B}, Vt0, #lint{catch_scope=Scope}=St0) -> +icrt_clause({clause,_Line,H,G,B}, Vt0, St0) -> Vt1 = taint_stack_var(Vt0, H, St0), {Hvt,Binvt,St1} = head(H, Vt1, St0), Vt2 = vtupdate(Hvt, Binvt), @@ -3251,7 +3241,7 @@ icrt_clause({clause,_Line,H,G,B}, Vt0, #lint{catch_scope=Scope}=St0) -> {Gvt,St2} = guard(G, vtupdate(Vt3, Vt0), St1#lint{in_try_head=false}), Vt4 = vtupdate(Gvt, Vt2), {Bvt,St3} = exprs(B, vtupdate(Vt4, Vt0), St2), - {vtupdate(Bvt, Vt4),St3#lint{catch_scope=Scope}}. + {vtupdate(Bvt, Vt4),St3}. taint_stack_var(Vt, Pat, #lint{in_try_head=true}) -> [{tuple,_,[_,_,{var,_,Stk}]}] = Pat, @@ -3736,8 +3726,7 @@ has_wildcard_field([]) -> false. check_remote_function(Line, M, F, As, St0) -> St1 = deprecated_function(Line, M, F, As, St0), St2 = check_qlc_hrl(Line, M, F, As, St1), - St3 = check_get_stacktrace(Line, M, F, As, St2), - format_function(Line, M, F, As, St3). + format_function(Line, M, F, As, St2). %% check_qlc_hrl(Line, ModName, FuncName, [Arg], State) -> State %% Add warning if qlc:q/1,2 has been called but qlc.hrl has not @@ -3786,23 +3775,6 @@ deprecated_function(Line, M, F, As, St) -> St end. -check_get_stacktrace(Line, erlang, get_stacktrace, [], St) -> - case St of - #lint{catch_scope=none} -> - St; - #lint{catch_scope=try_catch} -> - St; - #lint{catch_scope=Scope} -> - case is_warn_enabled(get_stacktrace, St) of - false -> - St; - true -> - add_warning(Line, {get_stacktrace,Scope}, St) - end - end; -check_get_stacktrace(_, _, _, _, St) -> - St. - -dialyzer({no_match, deprecated_type/5}). deprecated_type(L, M, N, As, St) -> diff --git a/lib/stdlib/src/erl_parse.yrl b/lib/stdlib/src/erl_parse.yrl index 14ca24362e..0c338b5952 100644 --- a/lib/stdlib/src/erl_parse.yrl +++ b/lib/stdlib/src/erl_parse.yrl @@ -1377,6 +1377,8 @@ normalise({map,_,Pairs}=M) -> ({map_field_assoc,_,K,V}) -> {normalise(K),normalise(V)}; (_) -> erlang:error({badarg,M}) end, Pairs)); +normalise({'fun',_,{function,{atom,_,M},{atom,_,F},{integer,_,A}}}) -> + fun M:F/A; %% Special case for unary +/-. normalise({op,_,'+',{char,_,I}}) -> I; normalise({op,_,'+',{integer,_,I}}) -> I; diff --git a/lib/stdlib/src/erl_posix_msg.erl b/lib/stdlib/src/erl_posix_msg.erl index bfafca1ff7..8959fea498 100644 --- a/lib/stdlib/src/erl_posix_msg.erl +++ b/lib/stdlib/src/erl_posix_msg.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2016. All Rights Reserved. +%% Copyright Ericsson AB 1997-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -64,6 +64,7 @@ message_1(eduppkg) -> <<"duplicate package name">>; message_1(eexist) -> <<"file already exists">>; message_1(efault) -> <<"bad address in system call argument">>; message_1(efbig) -> <<"file too large">>; +message_1(eftype) -> <<"EFTYPE">>; message_1(ehostdown) -> <<"host is down">>; message_1(ehostunreach) -> <<"host is unreachable">>; message_1(eidrm) -> <<"identifier removed">>; @@ -115,6 +116,7 @@ message_1(enopkg) -> <<"package not installed">>; message_1(enoprotoopt) -> <<"bad proocol option">>; message_1(enospc) -> <<"no space left on device">>; message_1(enosr) -> <<"out of stream resources or not a stream device">>; +message_1(enostr) -> <<"not a stream">>; message_1(enosym) -> <<"unresolved symbol name">>; message_1(enosys) -> <<"function not implemented">>; message_1(enotblk) -> <<"block device required">>; @@ -128,6 +130,7 @@ message_1(enotty) -> <<"inappropriate device for ioctl">>; message_1(enotuniq) -> <<"name not unique on network">>; message_1(enxio) -> <<"no such device or address">>; message_1(eopnotsupp) -> <<"operation not supported on socket">>; +message_1(eoverflow) -> <<"offset too large for file system">>; message_1(eperm) -> <<"not owner">>; message_1(epfnosupport) -> <<"protocol family not supported">>; message_1(epipe) -> <<"broken pipe">>; @@ -167,4 +170,6 @@ message_1(ewouldblock) -> <<"operation would block">>; message_1(exdev) -> <<"cross-domain link">>; message_1(exfull) -> <<"message tables full">>; message_1(nxdomain) -> <<"non-existing domain">>; +message_1(exbadport) -> <<"inet_drv bad port state">>; +message_1(exbadseq) -> <<"inet_drv bad request sequence">>; message_1(_) -> <<"unknown POSIX error">>. diff --git a/lib/stdlib/src/gen_event.erl b/lib/stdlib/src/gen_event.erl index 73e4457bd0..53042251cc 100644 --- a/lib/stdlib/src/gen_event.erl +++ b/lib/stdlib/src/gen_event.erl @@ -47,16 +47,19 @@ system_replace_state/2, format_status/2]). +%% logger callback +-export([format_log/1]). + -export_type([handler/0, handler_args/0, add_handler_ret/0, del_handler_ret/0]). --import(error_logger, [error_msg/2]). - -record(handler, {module :: atom(), id = false, state, supervised = false :: 'false' | pid()}). +-include("logger.hrl"). + %%%========================================================================= %%% API %%%========================================================================= @@ -583,9 +586,13 @@ server_update(Handler1, Func, Event, SName) -> remove, SName, normal), no; {'EXIT', {undef, [{Mod1, handle_info, [_,_], _}|_]}} -> - error_logger:warning_msg("** Undefined handle_info in ~tp~n" - "** Unhandled message: ~tp~n", [Mod1, Event]), - {ok, Handler1}; + ?LOG_WARNING(#{label=>{gen_event,no_handle_info}, + module=>Mod1, + message=>Event}, + #{domain=>[beam,erlang,otp], + report_cb=>fun gen_event:format_log/1, + error_logger=>#{tag=>warning_msg}}), % warningmap?? + {ok, Handler1}; Other -> do_terminate(Mod1, Handler1, {error, Other}, State, Event, SName, crash), @@ -737,6 +744,23 @@ report_error(_Handler, normal, _, _, _) -> ok; report_error(_Handler, shutdown, _, _, _) -> ok; report_error(_Handler, {swapped,_,_}, _, _, _) -> ok; report_error(Handler, Reason, State, LastIn, SName) -> + ?LOG_ERROR(#{label=>{gen_event,terminate}, + handler=>handler(Handler), + name=>SName, + last_message=>LastIn, + state=>format_status(terminate,Handler#handler.module, + get(),State), + reason=>Reason}, + #{domain=>[beam,erlang,otp], + report_cb=>fun gen_event:format_log/1, + error_logger=>#{tag=>error}}). + +format_log(#{label:={gen_event,terminate}, + handler:=Handler, + name:=SName, + last_message:=LastIn, + state:=State, + reason:=Reason}) -> Reason1 = case Reason of {'EXIT',{undef,[{M,F,A,L}|MFAs]}} -> @@ -756,23 +780,18 @@ report_error(Handler, Reason, State, LastIn, SName) -> _ -> Reason end, - Mod = Handler#handler.module, - FmtState = case erlang:function_exported(Mod, format_status, 2) of - true -> - Args = [get(), State], - case catch Mod:format_status(terminate, Args) of - {'EXIT', _} -> State; - Else -> Else - end; - _ -> - State - end, - error_msg("** gen_event handler ~p crashed.~n" - "** Was installed in ~tp~n" - "** Last event was: ~tp~n" - "** When handler state == ~tp~n" - "** Reason == ~tp~n", - [handler(Handler),SName,LastIn,FmtState,Reason1]). + {"** gen_event handler ~p crashed.~n" + "** Was installed in ~tp~n" + "** Last event was: ~tp~n" + "** When handler state == ~tp~n" + "** Reason == ~tp~n", + [Handler,SName,LastIn,State,Reason1]}; +format_log(#{label:={gen_event,no_handle_info}, + module:=Mod, + message:=Msg}) -> + {"** Undefined handle_info in ~tp~n" + "** Unhandled message: ~tp~n", + [Mod, Msg]}. handler(Handler) when not Handler#handler.id -> Handler#handler.module; @@ -805,17 +824,21 @@ format_status(Opt, StatusData) -> [PDict, SysState, Parent, _Debug, [ServerName, MSL, _HibernateAfterTimeout, _Hib]] = StatusData, Header = gen:format_status_header("Status for event handler", ServerName), - FmtMSL = [case erlang:function_exported(Mod, format_status, 2) of - true -> - Args = [PDict, State], - case catch Mod:format_status(Opt, Args) of - {'EXIT', _} -> MSL; - Else -> MS#handler{state = Else} - end; - _ -> - MS - end || #handler{module = Mod, state = State} = MS <- MSL], + FmtMSL = [MS#handler{state=format_status(Opt, Mod, PDict, State)} + || #handler{module = Mod, state = State} = MS <- MSL], [{header, Header}, {data, [{"Status", SysState}, {"Parent", Parent}]}, {items, {"Installed handlers", FmtMSL}}]. + +format_status(Opt, Mod, PDict, State) -> + case erlang:function_exported(Mod, format_status, 2) of + true -> + Args = [PDict, State], + case catch Mod:format_status(Opt, Args) of + {'EXIT', _} -> State; + Else -> Else + end; + false -> + State + end. diff --git a/lib/stdlib/src/gen_fsm.erl b/lib/stdlib/src/gen_fsm.erl index 8c7db65563..77826c3dc6 100644 --- a/lib/stdlib/src/gen_fsm.erl +++ b/lib/stdlib/src/gen_fsm.erl @@ -105,6 +105,8 @@ %%% %%% --------------------------------------------------- +-include("logger.hrl"). + -export([start/3, start/4, start_link/3, start_link/4, stop/1, stop/3, @@ -124,6 +126,9 @@ system_replace_state/2, format_status/2]). +%% logger callback +-export([format_log/1]). + -deprecated({start, 3, next_major_release}). -deprecated({start, 4, next_major_release}). -deprecated({start_link, 3, next_major_release}). @@ -144,8 +149,6 @@ -deprecated({enter_loop, 5, next_major_release}). -deprecated({enter_loop, 6, next_major_release}). --import(error_logger, [format/2]). - %%% --------------------------------------------------- %%% Interface functions. %%% --------------------------------------------------- @@ -499,8 +502,12 @@ handle_msg(Msg, Parent, Name, StateName, StateData, Mod, _Time, HibernateAfterTi reply(From, Reply), exit(R); {'EXIT', {undef, [{Mod, handle_info, [_,_,_], _}|_]}} -> - error_logger:warning_msg("** Undefined handle_info in ~p~n" - "** Unhandled message: ~tp~n", [Mod, Msg]), + ?LOG_WARNING(#{label=>{gen_fsm,no_handle_info}, + module=>Mod, + message=>Msg}, + #{domain=>[beam,erlang,otp], + report_cb=>fun gen_fsm:format_log/1, + error_logger=>#{tag=>warning_msg}}), loop(Parent, Name, StateName, StateData, Mod, infinity, HibernateAfterTimeout, []); {'EXIT', What} -> terminate(What, Name, Msg, Mod, StateName, StateData, []); @@ -603,6 +610,24 @@ terminate(Reason, Name, Msg, Mod, StateName, StateData, Debug) -> end. error_info(Reason, Name, Msg, StateName, StateData, Debug) -> + ?LOG_ERROR(#{label=>{gen_fsm,terminate}, + name=>Name, + last_message=>Msg, + state_name=>StateName, + state_data=>StateData, + reason=>Reason}, + #{domain=>[beam,erlang,otp], + report_cb=>fun gen_fsm:format_log/1, + error_logger=>#{tag=>error}}), + sys:print_log(Debug), + ok. + +format_log(#{label:={gen_fsm,terminate}, + name:=Name, + last_message:=Msg, + state_name:=StateName, + state_data:=StateData, + reason:=Reason}) -> Reason1 = case Reason of {undef,[{M,F,A,L}|MFAs]} -> @@ -620,14 +645,18 @@ error_info(Reason, Name, Msg, StateName, StateData, Debug) -> _ -> Reason end, - Str = "** State machine ~tp terminating \n" ++ - get_msg_str(Msg) ++ - "** When State == ~tp~n" - "** Data == ~tp~n" - "** Reason for termination = ~n** ~tp~n", - format(Str, [Name, get_msg(Msg), StateName, StateData, Reason1]), - sys:print_log(Debug), - ok. + {"** State machine ~tp terminating \n" ++ + get_msg_str(Msg) ++ + "** When State == ~tp~n" + "** Data == ~tp~n" + "** Reason for termination = ~n** ~tp~n", + [Name, get_msg(Msg), StateName, StateData, Reason1]}; +format_log(#{label:={gen_fsm,no_handle_info}, + module:=Mod, + message:=Msg}) -> + {"** Undefined handle_info in ~p~n" + "** Unhandled message: ~tp~n", + [Mod, Msg]}. get_msg_str({'$gen_event', _Event}) -> "** Last event in was ~tp~n"; diff --git a/lib/stdlib/src/gen_server.erl b/lib/stdlib/src/gen_server.erl index f29314d0a2..f65ef78636 100644 --- a/lib/stdlib/src/gen_server.erl +++ b/lib/stdlib/src/gen_server.erl @@ -104,9 +104,14 @@ system_replace_state/2, format_status/2]). +%% logger callback +-export([format_log/1]). + %% Internal exports -export([init_it/6]). +-include("logger.hrl"). + -define( STACKTRACE(), element(2, erlang:process_info(self(), current_stacktrace))). @@ -636,9 +641,13 @@ try_dispatch(Mod, Func, Msg, State) -> error:undef = R:Stacktrace when Func == handle_info -> case erlang:function_exported(Mod, handle_info, 2) of false -> - error_logger:warning_msg("** Undefined handle_info in ~p~n" - "** Unhandled message: ~tp~n", - [Mod, Msg]), + ?LOG_WARNING( + #{label=>{gen_server,no_handle_info}, + module=>Mod, + message=>Msg}, + #{domain=>[beam,erlang,otp], + report_cb=>fun gen_server:format_log/1, + error_logger=>#{tag=>warning_msg}}), {ok, {noreply, State}}; true -> {'EXIT', error, R, Stacktrace} @@ -849,8 +858,7 @@ terminate(Class, Reason, Stacktrace, ReportReason, Name, From, Msg, Mod, State, Reply = try_terminate(Mod, terminate_reason(Class, Reason, Stacktrace), State), case Reply of {'EXIT', C, R, S} -> - FmtState = format_status(terminate, Mod, get(), State), - error_info({R, S}, Name, From, Msg, FmtState, Debug), + error_info({R, S}, Name, From, Msg, Mod, State, Debug), erlang:raise(C, R, S); _ -> case {Class, Reason} of @@ -858,8 +866,7 @@ terminate(Class, Reason, Stacktrace, ReportReason, Name, From, Msg, Mod, State, {exit, shutdown} -> ok; {exit, {shutdown,_}} -> ok; _ -> - FmtState = format_status(terminate, Mod, get(), State), - error_info(ReportReason, Name, From, Msg, FmtState, Debug) + error_info(ReportReason, Name, From, Msg, Mod, State, Debug) end end, case Stacktrace of @@ -872,12 +879,46 @@ terminate(Class, Reason, Stacktrace, ReportReason, Name, From, Msg, Mod, State, terminate_reason(error, Reason, Stacktrace) -> {Reason, Stacktrace}; terminate_reason(exit, Reason, _Stacktrace) -> Reason. -error_info(_Reason, application_controller, _From, _Msg, _State, _Debug) -> +error_info(_Reason, application_controller, _From, _Msg, _Mod, _State, _Debug) -> %% OTP-5811 Don't send an error report if it's the system process %% application_controller which is terminating - let init take care %% of it instead ok; -error_info(Reason, Name, From, Msg, State, Debug) -> +error_info(Reason, Name, From, Msg, Mod, State, Debug) -> + ?LOG_ERROR(#{label=>{gen_server,terminate}, + name=>Name, + last_message=>Msg, + state=>format_status(terminate, Mod, get(), State), + reason=>Reason, + client_info=>client_stacktrace(From)}, + #{domain=>[beam,erlang,otp], + report_cb=>fun gen_server:format_log/1, + error_logger=>#{tag=>error}}), + sys:print_log(Debug), + ok. + +client_stacktrace(undefined) -> + undefined; +client_stacktrace({From,_Tag}) -> + client_stacktrace(From); +client_stacktrace(From) when is_pid(From), node(From) =:= node() -> + case process_info(From, [current_stacktrace, registered_name]) of + undefined -> + {From,dead}; + [{current_stacktrace, Stacktrace}, {registered_name, []}] -> + {From,{From,Stacktrace}}; + [{current_stacktrace, Stacktrace}, {registered_name, Name}] -> + {From,{Name,Stacktrace}} + end; +client_stacktrace(From) when is_pid(From) -> + {From,remote}. + +format_log(#{label:={gen_server,terminate}, + name:=Name, + last_message:=Msg, + state:=State, + reason:=Reason, + client_info:=Client}) -> Reason1 = case Reason of {undef,[{M,F,A,L}|MFAs]} -> @@ -893,36 +934,31 @@ error_info(Reason, Name, From, Msg, State, Debug) -> end end; _ -> - error_logger:limit_term(Reason) + logger:limit_term(Reason) end, - {ClientFmt, ClientArgs} = client_stacktrace(From), - LimitedState = error_logger:limit_term(State), - error_logger:format("** Generic server ~tp terminating \n" - "** Last message in was ~tp~n" - "** When Server state == ~tp~n" - "** Reason for termination == ~n** ~tp~n" ++ ClientFmt, - [Name, Msg, LimitedState, Reason1] ++ ClientArgs), - sys:print_log(Debug), - ok. -client_stacktrace(undefined) -> + {ClientFmt,ClientArgs} = format_client_log(Client), + {"** Generic server ~tp terminating \n" + "** Last message in was ~tp~n" + "** When Server state == ~tp~n" + "** Reason for termination == ~n** ~tp~n" ++ ClientFmt, + [Name, Msg, logger:limit_term(State), Reason1] ++ ClientArgs}; +format_log(#{label:={gen_server,no_handle_info}, + module:=Mod, + message:=Msg}) -> + {"** Undefined handle_info in ~p~n" + "** Unhandled message: ~tp~n", + [Mod, Msg]}. + +format_client_log(undefined) -> {"", []}; -client_stacktrace({From, _Tag}) -> - client_stacktrace(From); -client_stacktrace(From) when is_pid(From), node(From) =:= node() -> - case process_info(From, [current_stacktrace, registered_name]) of - undefined -> - {"** Client ~p is dead~n", [From]}; - [{current_stacktrace, Stacktrace}, {registered_name, []}] -> - {"** Client ~p stacktrace~n" - "** ~tp~n", - [From, Stacktrace]}; - [{current_stacktrace, Stacktrace}, {registered_name, Name}] -> - {"** Client ~tp stacktrace~n" - "** ~tp~n", - [Name, Stacktrace]} - end; -client_stacktrace(From) when is_pid(From) -> - {"** Client ~p is remote on node ~p~n", [From, node(From)]}. +format_client_log({From,dead}) -> + {"** Client ~p is dead~n", [From]}; +format_client_log({From,remote}) -> + {"** Client ~p is remote on node ~p~n", [From, node(From)]}; +format_client_log({_From,{Name,Stacktrace}}) -> + {"** Client ~tp stacktrace~n" + "** ~tp~n", + [Name, Stacktrace]}. %%----------------------------------------------------------------- %% Status information diff --git a/lib/stdlib/src/gen_statem.erl b/lib/stdlib/src/gen_statem.erl index 9dc360a289..f558f0d33e 100644 --- a/lib/stdlib/src/gen_statem.erl +++ b/lib/stdlib/src/gen_statem.erl @@ -19,6 +19,8 @@ %% -module(gen_statem). +-include("logger.hrl"). + %% API -export( [start/3,start/4,start_link/3,start_link/4, @@ -44,6 +46,9 @@ -export( [wakeup_from_hibernate/3]). +%% logger callback +-export([format_log/1]). + %% Type exports for templates and callback modules -export_type( [event_type/0, @@ -143,7 +148,7 @@ timeout_action() | reply_action(). -type timeout_action() :: - (Timeout :: event_timeout()) | % {timeout,Timeout} + (Time :: event_timeout()) | % {timeout,Time,Time} {'timeout', % Set the event_timeout option Time :: event_timeout(), EventContent :: term()} | {'timeout', % Set the event_timeout option @@ -327,7 +332,8 @@ %% Type validation functions -compile( {inline, - [callback_mode/1, state_enter/1, from/1, event_type/1]}). + [callback_mode/1, state_enter/1, + event_type/1, from/1, timeout_event_type/1]}). %% callback_mode(CallbackMode) -> case CallbackMode of @@ -344,23 +350,26 @@ state_enter(StateEnter) -> false end. %% -from({Pid,_}) when is_pid(Pid) -> true; -from(_) -> false. -%% -event_type({call,From}) -> - from(From); event_type(Type) -> case Type of {call,From} -> from(From); + %% cast -> true; info -> true; - timeout -> true; - state_timeout -> true; internal -> true; - {timeout,_} -> true; - _ -> false + _ -> timeout_event_type(Type) + end. +%% +from({Pid,_}) when is_pid(Pid) -> true; +from(_) -> false. +%% +timeout_event_type(Type) -> + case Type of + timeout -> true; + state_timeout -> true; + {timeout,_Name} -> true; + _ -> false end. - -define( @@ -669,9 +678,9 @@ enter(Module, Opts, State, Data, Server, Actions, Parent) -> NewDebug = ?sys_debug(Debug, {Name,State}, {enter,Event,State}), case call_callback_mode(S) of #state{} = NewS -> - loop_event_actions( + loop_event_actions_list( Parent, NewDebug, NewS, - Events, Event, State, Data, #trans_opts{}, + Events, Event, State, Data, false, NewActions, CallEnter); [Class,Reason,Stacktrace] -> terminate( @@ -698,7 +707,7 @@ init_it(Starter, Parent, ServerRef, Module, Args, Opts) -> error_info( Class, Reason, Stacktrace, #state{name = Name}, - [], undefined), + []), erlang:raise(Class, Reason, Stacktrace) end. @@ -729,7 +738,7 @@ init_result(Starter, Parent, ServerRef, Module, Result, Opts) -> error_info( error, Error, ?STACKTRACE(), #state{name = Name}, - [], undefined), + []), exit(Error) end. @@ -1056,6 +1065,15 @@ loop_event_result( Parent, Debug, S, Events, Event, NextState, NewData, TransOpts, [], true); + {next_state,_NextState,_NewData} -> + terminate( + error, + {bad_state_enter_return_from_state_function,Result}, + ?STACKTRACE(), Debug, + S#state{ + state = State, data = Data, + hibernate = hibernate_in_trans_opts(TransOpts)}, + [Event|Events]); {next_state,State,NewData,Actions} -> loop_event_actions( Parent, Debug, S, @@ -1067,6 +1085,15 @@ loop_event_result( Parent, Debug, S, Events, Event, NextState, NewData, TransOpts, Actions, true); + {next_state,_NextState,_NewData,_Actions} -> + terminate( + error, + {bad_state_enter_return_from_state_function,Result}, + ?STACKTRACE(), Debug, + S#state{ + state = State, data = Data, + hibernate = hibernate_in_trans_opts(TransOpts)}, + [Event|Events]); %% {keep_state,NewData} -> loop_event_actions( @@ -1160,12 +1187,6 @@ loop_event_result( [Event|Events]) end. --compile({inline, [hibernate_in_trans_opts/1]}). -hibernate_in_trans_opts(false) -> - (#trans_opts{})#trans_opts.hibernate; -hibernate_in_trans_opts(#trans_opts{hibernate = Hibernate}) -> - Hibernate. - %% Ensure that Actions are a list loop_event_actions( Parent, Debug, S, @@ -1198,10 +1219,16 @@ loop_event_actions_list( S#state{ state = NextState, data = NewerData, - hibernate = TransOpts#trans_opts.hibernate}, + hibernate = hibernate_in_trans_opts(TransOpts)}, [Event|Events]) end. +-compile({inline, [hibernate_in_trans_opts/1]}). +hibernate_in_trans_opts(false) -> + (#trans_opts{})#trans_opts.hibernate; +hibernate_in_trans_opts(#trans_opts{hibernate = Hibernate}) -> + Hibernate. + parse_actions(false, Debug, S, Actions) -> parse_actions(true, Debug, S, Actions, #trans_opts{}); parse_actions(TransOpts, Debug, S, Actions) -> @@ -1234,6 +1261,11 @@ parse_actions(StateCall, Debug, S, [Action|Actions], TransOpts) -> parse_actions( StateCall, Debug, S, Actions, TransOpts#trans_opts{postpone = true}); + postpone -> + [error, + {bad_state_enter_action_from_state_function,Action}, + ?STACKTRACE(), + Debug]; %% {next_event,Type,Content} -> parse_actions_next_event( @@ -1286,7 +1318,8 @@ parse_actions_next_event( next_events_r = [{Type,Content}|NextEventsR]}); _ -> [error, - {bad_action_from_state_function,{next_events,Type,Content}}, + {bad_state_enter_action_from_state_function, + {next_event,Type,Content}}, ?STACKTRACE(), ?not_sys_debug] end; @@ -1303,22 +1336,23 @@ parse_actions_next_event( next_events_r = [{Type,Content}|NextEventsR]}); _ -> [error, - {bad_action_from_state_function,{next_events,Type,Content}}, + {bad_state_enter_action_from_state_function, + {next_event,Type,Content}}, ?STACKTRACE(), Debug] end. parse_actions_timeout( StateCall, Debug, S, Actions, TransOpts, - {TimerType,Time,TimerMsg,TimerOpts} = AbsoluteTimeout) -> + {TimeoutType,Time,TimerMsg,TimerOpts} = AbsoluteTimeout) -> %% - case classify_timer(Time, listify(TimerOpts)) of + case classify_timeout(TimeoutType, Time, listify(TimerOpts)) of absolute -> parse_actions_timeout_add( StateCall, Debug, S, Actions, TransOpts, AbsoluteTimeout); relative -> - RelativeTimeout = {TimerType,Time,TimerMsg}, + RelativeTimeout = {TimeoutType,Time,TimerMsg}, parse_actions_timeout_add( StateCall, Debug, S, Actions, TransOpts, RelativeTimeout); @@ -1330,8 +1364,8 @@ parse_actions_timeout( end; parse_actions_timeout( StateCall, Debug, S, Actions, TransOpts, - {_,Time,_} = RelativeTimeout) -> - case classify_timer(Time, []) of + {TimeoutType,Time,_} = RelativeTimeout) -> + case classify_timeout(TimeoutType, Time, []) of relative -> parse_actions_timeout_add( StateCall, Debug, S, Actions, @@ -1344,14 +1378,16 @@ parse_actions_timeout( end; parse_actions_timeout( StateCall, Debug, S, Actions, TransOpts, - Timeout) -> - case classify_timer(Timeout, []) of + Time) -> + case classify_timeout(timeout, Time, []) of relative -> + RelativeTimeout = {timeout,Time,Time}, parse_actions_timeout_add( - StateCall, Debug, S, Actions, TransOpts, Timeout); + StateCall, Debug, S, Actions, + TransOpts, RelativeTimeout); badarg -> [error, - {bad_action_from_state_function,Timeout}, + {bad_action_from_state_function,Time}, ?STACKTRACE(), Debug] end. @@ -1637,10 +1673,15 @@ call_state_function( %% -> absolute | relative | badarg -classify_timer(Time, Opts) -> - classify_timer(Time, Opts, false). -%% -classify_timer(Time, [], Abs) -> +classify_timeout(TimeoutType, Time, Opts) -> + case timeout_event_type(TimeoutType) of + true -> + classify_time(false, Time, Opts); + false -> + badarg + end. + +classify_time(Abs, Time, []) -> case Abs of true when is_integer(Time); @@ -1653,9 +1694,9 @@ classify_timer(Time, [], Abs) -> _ -> badarg end; -classify_timer(Time, [{abs,Abs}|Opts], _) when is_boolean(Abs) -> - classify_timer(Time, Opts, Abs); -classify_timer(_, Opts, _) when is_list(Opts) -> +classify_time(_, Time, [{abs,Abs}|Opts]) when is_boolean(Abs) -> + classify_time(Abs, Time, Opts); +classify_time(_, _, Opts) when is_list(Opts) -> badarg. %% Stop and start timers as well as create timeout zero events @@ -1686,15 +1727,7 @@ parse_timers( {TimerType,Time,TimerMsg} -> parse_timers( TimerRefs, Timers, TimeoutsR, Seen, TimeoutEvents, - TimerType, Time, TimerMsg, []); - 0 -> - parse_timers( - TimerRefs, Timers, TimeoutsR, Seen, TimeoutEvents, - timeout, zero, 0, []); - Time -> - parse_timers( - TimerRefs, Timers, TimeoutsR, Seen, TimeoutEvents, - timeout, Time, Time, []) + TimerType, Time, TimerMsg, []) end. parse_timers( @@ -1821,9 +1854,7 @@ terminate( catch _ -> ok; C:R:ST -> - error_info( - C, R, ST, S, Q, - format_status(terminate, get(), S)), + error_info(C, R, ST, S, Q), sys:print_log(Debug), erlang:raise(C, R, ST) end; @@ -1839,9 +1870,7 @@ terminate( {shutdown,_} -> terminate_sys_debug(Debug, S, State, Reason); _ -> - error_info( - Class, Reason, Stacktrace, S, Q, - format_status(terminate, get(), S)), + error_info(Class, Reason, Stacktrace, S, Q), sys:print_log(Debug) end, case Stacktrace of @@ -1861,8 +1890,28 @@ error_info( name = Name, callback_mode = CallbackMode, state_enter = StateEnter, - postponed = P}, - Q, FmtData) -> + postponed = P} = S, + Q) -> + ?LOG_ERROR(#{label=>{gen_statem,terminate}, + name=>Name, + queue=>Q, + postponed=>P, + callback_mode=>CallbackMode, + state_enter=>StateEnter, + state=>format_status(terminate, get(), S), + reason=>{Class,Reason,Stacktrace}}, + #{domain=>[beam,erlang,otp], + report_cb=>fun gen_statem:format_log/1, + error_logger=>#{tag=>error}}). + +format_log(#{label:={gen_statem,terminate}, + name:=Name, + queue:=Q, + postponed:=P, + callback_mode:=CallbackMode, + state_enter:=StateEnter, + state:=FmtData, + reason:={Class,Reason,Stacktrace}}) -> {FixedReason,FixedStacktrace} = case Stacktrace of [{M,F,Args,_}|ST] @@ -1889,7 +1938,7 @@ error_info( _ -> {Reason,Stacktrace} end, [LimitedP, LimitedFmtData, LimitedFixedReason] = - [error_logger:limit_term(D) || D <- [P, FmtData, FixedReason]], + [logger:limit_term(D) || D <- [P, FmtData, FixedReason]], CBMode = case StateEnter of true -> @@ -1897,48 +1946,46 @@ error_info( false -> CallbackMode end, - error_logger:format( - "** State machine ~tp terminating~n" ++ - case Q of - [] -> ""; - _ -> "** Last event = ~tp~n" - end ++ - "** When server state = ~tp~n" ++ - "** Reason for termination = ~w:~tp~n" ++ - "** Callback mode = ~p~n" ++ - case Q of - [_,_|_] -> "** Queued = ~tp~n"; - _ -> "" - end ++ - case P of - [] -> ""; - _ -> "** Postponed = ~tp~n" - end ++ - case FixedStacktrace of - [] -> ""; - _ -> "** Stacktrace =~n** ~tp~n" - end, - [Name | - case Q of - [] -> []; - [Event|_] -> [Event] - end] ++ - [LimitedFmtData, - Class,LimitedFixedReason, - CBMode] ++ - case Q of - [_|[_|_] = Events] -> [Events]; - _ -> [] - end ++ - case P of - [] -> []; - _ -> [LimitedP] - end ++ - case FixedStacktrace of - [] -> []; - _ -> [FixedStacktrace] - end). - + {"** State machine ~tp terminating~n" ++ + case Q of + [] -> ""; + _ -> "** Last event = ~tp~n" + end ++ + "** When server state = ~tp~n" ++ + "** Reason for termination = ~w:~tp~n" ++ + "** Callback mode = ~p~n" ++ + case Q of + [_,_|_] -> "** Queued = ~tp~n"; + _ -> "" + end ++ + case P of + [] -> ""; + _ -> "** Postponed = ~tp~n" + end ++ + case FixedStacktrace of + [] -> ""; + _ -> "** Stacktrace =~n** ~tp~n" + end, + [Name | + case Q of + [] -> []; + [Event|_] -> [Event] + end] ++ + [LimitedFmtData, + Class,LimitedFixedReason, + CBMode] ++ + case Q of + [_|[_|_] = Events] -> [Events]; + _ -> [] + end ++ + case P of + [] -> []; + _ -> [LimitedP] + end ++ + case FixedStacktrace of + [] -> []; + _ -> [FixedStacktrace] + end}. %% Call Module:format_status/2 or return a default value format_status( diff --git a/lib/stdlib/src/io_lib.erl b/lib/stdlib/src/io_lib.erl index e37c13093b..3a5aba60b4 100644 --- a/lib/stdlib/src/io_lib.erl +++ b/lib/stdlib/src/io_lib.erl @@ -60,11 +60,12 @@ -module(io_lib). --export([fwrite/2,fread/2,fread/3,format/2]). --export([scan_format/2,unscan_format/1,build_text/1]). +-export([fwrite/2,fwrite/3,fread/2,fread/3,format/2,format/3]). +-export([scan_format/2,unscan_format/1,build_text/1,build_text/2]). -export([print/1,print/4,indentation/2]). -export([write/1,write/2,write/3,nl/0,format_prompt/1,format_prompt/2]). +-export([write_binary/3]). -export([write_atom/1,write_string/1,write_string/2,write_latin1_string/1, write_latin1_string/2, write_char/1, write_latin1_char/1]). @@ -87,7 +88,7 @@ -export([limit_term/2]). -export_type([chars/0, latin1_string/0, continuation/0, - fread_error/0, fread_item/0, format_spec/0]). + fread_error/0, fread_item/0, format_spec/0, chars_limit/0]). %%---------------------------------------------------------------------- @@ -135,6 +136,18 @@ fwrite(Format, Args) -> format(Format, Args). +-type chars_limit() :: integer(). + +-spec fwrite(Format, Data, Options) -> chars() when + Format :: io:format(), + Data :: [term()], + Options :: [Option], + Option :: {'chars_limit', CharsLimit}, + CharsLimit :: chars_limit(). + +fwrite(Format, Args, Options) -> + format(Format, Args, Options). + -spec fread(Format, String) -> Result when Format :: string(), String :: string(), @@ -172,6 +185,21 @@ format(Format, Args) -> Other end. +-spec format(Format, Data, Options) -> chars() when + Format :: io:format(), + Data :: [term()], + Options :: [Option], + Option :: {'chars_limit', CharsLimit}, + CharsLimit :: chars_limit(). + +format(Format, Args, Options) -> + case catch io_lib_format:fwrite(Format, Args, Options) of + {'EXIT',_} -> + erlang:error(badarg, [Format, Args, Options]); + Other -> + Other + end. + -spec scan_format(Format, Data) -> FormatList when Format :: io:format(), Data :: [term()], @@ -197,6 +225,15 @@ unscan_format(FormatList) -> build_text(FormatList) -> io_lib_format:build(FormatList). +-spec build_text(FormatList, Options) -> chars() when + FormatList :: [char() | format_spec()], + Options :: [Option], + Option :: {'chars_limit', CharsLimit}, + CharsLimit :: chars_limit(). + +build_text(FormatList, Options) -> + io_lib_format:build(FormatList, Options). + -spec print(Term) -> chars() when Term :: term(). @@ -240,7 +277,7 @@ format_prompt(Prompt, Encoding) -> do_format_prompt(add_modifier(Encoding, "p"), [Prompt]). do_format_prompt(Format, Args) -> - case catch io_lib:format(Format, Args) of + case catch format(Format, Args) of {'EXIT',_} -> "???"; List -> List end. @@ -259,7 +296,8 @@ add_modifier(_, C) -> -spec write(Term) -> chars() when Term :: term(). -write(Term) -> write(Term, -1). +write(Term) -> + write1(Term, -1, latin1). -spec write(term(), depth(), boolean()) -> chars(). @@ -274,16 +312,29 @@ write(Term, D, false) -> (Term, Options) -> chars() when Term :: term(), Options :: [Option], - Option :: {'depth', Depth} + Option :: {'chars_limit', CharsLimit} + | {'depth', Depth} | {'encoding', 'latin1' | 'utf8' | 'unicode'}, + CharsLimit :: chars_limit(), Depth :: depth(). write(Term, Options) when is_list(Options) -> Depth = get_option(depth, Options, -1), Encoding = get_option(encoding, Options, epp:default_encoding()), - write1(Term, Depth, Encoding); + CharsLimit = get_option(chars_limit, Options, -1), + if + Depth =:= 0; CharsLimit =:= 0 -> + "..."; + CharsLimit < 0 -> + write1(Term, Depth, Encoding); + CharsLimit > 0 -> + RecDefFun = fun(_, _) -> no end, + If = io_lib_pretty:intermediate + (Term, Depth, CharsLimit, RecDefFun, Encoding, _Str=false), + io_lib_pretty:write(If) + end; write(Term, Depth) -> - write1(Term, Depth, latin1). + write(Term, [{depth, Depth}, {encoding, latin1}]). write1(_Term, 0, _E) -> "..."; write1(Term, _D, _E) when is_integer(Term) -> integer_to_list(Term); @@ -300,7 +351,7 @@ write1([H|T], D, E) -> if D =:= 1 -> "[...]"; true -> - [$[,[write1(H, D-1, E)|write_tail(T, D-1, E, $|)],$]] + [$[,[write1(H, D-1, E)|write_tail(T, D-1, E)],$]] end; write1(F, _D, _E) when is_function(F) -> erlang:fun_to_list(F); @@ -311,20 +362,24 @@ write1(T, D, E) when is_tuple(T) -> D =:= 1 -> "{...}"; true -> [${, - [write1(element(1, T), D-1, E)| - write_tail(tl(tuple_to_list(T)), D-1, E, $,)], + [write1(element(1, T), D-1, E)|write_tuple(T, 2, D-1, E)], $}] end. -%% write_tail(List, Depth, CharacterBeforeDots) +%% write_tail(List, Depth, Encoding) %% Test the terminating case first as this looks better with depth. -write_tail([], _D, _E, _S) -> ""; -write_tail(_, 1, _E, S) -> [S | "..."]; -write_tail([H|T], D, E, S) -> - [$,,write1(H, D-1, E)|write_tail(T, D-1, E, S)]; -write_tail(Other, D, E, S) -> - [S,write1(Other, D-1, E)]. +write_tail([], _D, _E) -> ""; +write_tail(_, 1, _E) -> [$| | "..."]; +write_tail([H|T], D, E) -> + [$,,write1(H, D-1, E)|write_tail(T, D-1, E)]; +write_tail(Other, D, E) -> + [$|,write1(Other, D-1, E)]. + +write_tuple(T, I, _D, _E) when I > tuple_size(T) -> ""; +write_tuple(_, _I, 1, _E) -> [$, | "..."]; +write_tuple(T, I, D, E) -> + [$,,write1(element(I, T), D-1, E)|write_tuple(T, I+1, D-1, E)]. write_port(Port) -> erlang:port_to_list(Port). @@ -333,32 +388,43 @@ write_ref(Ref) -> erlang:ref_to_list(Ref). write_map(Map, D, E) when is_integer(D) -> - [$#,${,write_map_body(maps:to_list(Map), D, E),$}]. + [$#,${,write_map_body(maps:to_list(Map), D, D - 1, E),$}]. -write_map_body(_, 0, _E) -> "..."; -write_map_body([], _, _E) -> []; -write_map_body([{K,V}], D, E) -> write_map_assoc(K, V, D, E); -write_map_body([{K,V}|KVs], D, E) -> - [write_map_assoc(K, V, D, E),$, | write_map_body(KVs, D-1, E)]. +write_map_body(_, 1, _D0, _E) -> "..."; +write_map_body([], _, _D0, _E) -> []; +write_map_body([{K,V}], _D, D0, E) -> write_map_assoc(K, V, D0, E); +write_map_body([{K,V}|KVs], D, D0, E) -> + [write_map_assoc(K, V, D0, E),$, | write_map_body(KVs, D - 1, D0, E)]. write_map_assoc(K, V, D, E) -> - [write1(K, D - 1, E),"=>",write1(V, D-1, E)]. + [write1(K, D, E)," => ",write1(V, D, E)]. write_binary(B, D) when is_integer(D) -> - [$<,$<,write_binary_body(B, D),$>,$>]. - -write_binary_body(<<>>, _D) -> - ""; -write_binary_body(_B, 1) -> - "..."; -write_binary_body(<<X:8>>, _D) -> - [integer_to_list(X)]; -write_binary_body(<<X:8,Rest/bitstring>>, D) -> - [integer_to_list(X),$,|write_binary_body(Rest, D-1)]; -write_binary_body(B, _D) -> + {S, _} = write_binary(B, D, -1), + S. + +write_binary(B, D, T) -> + {S, Rest} = write_binary_body(B, D, tsub(T, 4), []), + {[$<,$<,lists:reverse(S),$>,$>], Rest}. + +write_binary_body(<<>> = B, _D, _T, Acc) -> + {Acc, B}; +write_binary_body(B, D, T, Acc) when D =:= 1; T =:= 0-> + {["..."|Acc], B}; +write_binary_body(<<X:8>>, _D, _T, Acc) -> + {[integer_to_list(X)|Acc], <<>>}; +write_binary_body(<<X:8,Rest/bitstring>>, D, T, Acc) -> + S = integer_to_list(X), + write_binary_body(Rest, D-1, tsub(T, length(S) + 1), [$,,S|Acc]); +write_binary_body(B, _D, _T, Acc) -> L = bit_size(B), <<X:L>> = B, - [integer_to_list(X),$:,integer_to_list(L)]. + {[integer_to_list(L),$:,integer_to_list(X)|Acc], <<>>}. + +%% Make sure T does not change sign. +tsub(T, _) when T < 0 -> T; +tsub(T, E) when T >= E -> T - E; +tsub(_, _) -> 0. get_option(Key, TupleList, Default) -> case lists:keyfind(Key, 1, TupleList) of @@ -947,7 +1013,7 @@ limit(T, D) when is_tuple(T) -> D =:= 1 -> {'...'}; true -> list_to_tuple([limit(element(1, T), D-1)| - limit_tail(tl(tuple_to_list(T)), D-1)]) + limit_tuple(T, 2, D-1)]) end; limit(<<_/bitstring>>=Term, D) -> limit_bitstring(Term, D); limit(Term, _D) -> Term. @@ -959,6 +1025,11 @@ limit_tail([H|T], D) -> limit_tail(Other, D) -> limit(Other, D-1). +limit_tuple(T, I, _D) when I > tuple_size(T) -> []; +limit_tuple(_, _I, 1) -> ['...']; +limit_tuple(T, I, D) -> + [limit(element(I, T), D-1)|limit_tuple(T, I+1, D-1)]. + %% Cannot limit maps properly since there is no guarantee that %% maps:from_list() creates a map with the same internal ordering of %% the selected associations as in Map. Instead of subtracting one diff --git a/lib/stdlib/src/io_lib_format.erl b/lib/stdlib/src/io_lib_format.erl index 64edbf1824..c814ab50d4 100644 --- a/lib/stdlib/src/io_lib_format.erl +++ b/lib/stdlib/src/io_lib_format.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2017. All Rights Reserved. +%% Copyright Ericsson AB 1996-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -21,7 +21,8 @@ %% Formatting functions of io library. --export([fwrite/2,fwrite_g/1,indentation/2,scan/2,unscan/1,build/1]). +-export([fwrite/2,fwrite/3,fwrite_g/1,indentation/2,scan/2,unscan/1, + build/1, build/2]). %% Format the arguments in Args after string Format. Just generate %% an error if there is an error in the arguments. @@ -45,14 +46,42 @@ fwrite(Format, Args) -> build(scan(Format, Args)). +-spec fwrite(Format, Data, Options) -> FormatList when + Format :: io:format(), + Data :: [term()], + FormatList :: [char() | io_lib:format_spec()], + Options :: [Option], + Option :: {'chars_limit', CharsLimit}, + CharsLimit :: io_lib:chars_limit(). + +fwrite(Format, Args, Options) -> + build(scan(Format, Args), Options). + %% Build the output text for a pre-parsed format list. -spec build(FormatList) -> io_lib:chars() when FormatList :: [char() | io_lib:format_spec()]. build(Cs) -> - Pc = pcount(Cs), - build(Cs, Pc, 0). + build(Cs, []). + +-spec build(FormatList, Options) -> io_lib:chars() when + FormatList :: [char() | io_lib:format_spec()], + Options :: [Option], + Option :: {'chars_limit', CharsLimit}, + CharsLimit :: io_lib:chars_limit(). + +build(Cs, Options) -> + CharsLimit = get_option(chars_limit, Options, -1), + Res1 = build_small(Cs), + {P, S, W, Other} = count_small(Res1), + case P + S + W of + 0 -> + Res1; + NumOfLimited -> + RemainingChars = sub(CharsLimit, Other), + build_limited(Res1, P, NumOfLimited, RemainingChars, 0) + end. %% Parse all control sequences in the format string. @@ -202,40 +231,77 @@ collect_cc([$~|Fmt], Args) when is_list(Args) -> {$~,[],Fmt,Args}; collect_cc([$n|Fmt], Args) when is_list(Args) -> {$n,[],Fmt,Args}; collect_cc([$i|Fmt], [A|Args]) -> {$i,[A],Fmt,Args}. -%% pcount([ControlC]) -> Count. -%% Count the number of print requests. - -pcount(Cs) -> pcount(Cs, 0). - -pcount([#{control_char := $p}|Cs], Acc) -> pcount(Cs, Acc+1); -pcount([#{control_char := $P}|Cs], Acc) -> pcount(Cs, Acc+1); -pcount([_|Cs], Acc) -> pcount(Cs, Acc); -pcount([], Acc) -> Acc. - -%% build([Control], Pc, Indentation) -> io_lib:chars(). +%% count_small([ControlC]) -> Count. +%% Count the number of big (pPwWsS) print requests and +%% number of characters of other print (small) requests. + +count_small(Cs) -> + count_small(Cs, #{p => 0, s => 0, w => 0, other => 0}). + +count_small([#{control_char := $p}|Cs], #{p := P} = Cnts) -> + count_small(Cs, Cnts#{p := P + 1}); +count_small([#{control_char := $P}|Cs], #{p := P} = Cnts) -> + count_small(Cs, Cnts#{p := P + 1}); +count_small([#{control_char := $w}|Cs], #{w := W} = Cnts) -> + count_small(Cs, Cnts#{w := W + 1}); +count_small([#{control_char := $W}|Cs], #{w := W} = Cnts) -> + count_small(Cs, Cnts#{w := W + 1}); +count_small([#{control_char := $s}|Cs], #{w := W} = Cnts) -> + count_small(Cs, Cnts#{w := W + 1}); +count_small([S|Cs], #{other := Other} = Cnts) when is_list(S) -> + count_small(Cs, Cnts#{other := Other + string:length(S)}); +count_small([C|Cs], #{other := Other} = Cnts) when is_integer(C) -> + count_small(Cs, Cnts#{other := Other + 1}); +count_small([], #{p := P, s := S, w := W, other := Other}) -> + {P, S, W, Other}. + +%% build_small([Control]) -> io_lib:chars(). +%% Interpret the control structures, but only the small ones. +%% The big ones are saved for later. +%% build_limited([Control], NumberOfPps, NumberOfLimited, +%% CharsLimit, Indentation) %% Interpret the control structures. Count the number of print %% remaining and only calculate indentation when necessary. Must also %% be smart when calculating indentation for characters in format. -build([#{control_char := C, args := As, width := F, adjust := Ad, - precision := P, pad_char := Pad, encoding := Enc, - strings := Str} | Cs], Pc0, I) -> - S = control(C, As, F, Ad, P, Pad, Enc, Str, I), - Pc1 = decr_pc(C, Pc0), +build_small([#{control_char := C, args := As, width := F, adjust := Ad, + precision := P, pad_char := Pad, encoding := Enc}=CC | Cs]) -> + case control_small(C, As, F, Ad, P, Pad, Enc) of + not_small -> [CC | build_small(Cs)]; + S -> lists:flatten(S) ++ build_small(Cs) + end; +build_small([C|Cs]) -> [C|build_small(Cs)]; +build_small([]) -> []. + +build_limited([#{control_char := C, args := As, width := F, adjust := Ad, + precision := P, pad_char := Pad, encoding := Enc, + strings := Str} | Cs], NumOfPs0, Count0, MaxLen0, I) -> + MaxChars = if + MaxLen0 < 0 -> MaxLen0; + true -> MaxLen0 div Count0 + end, + S = control_limited(C, As, F, Ad, P, Pad, Enc, Str, MaxChars, I), + Len = string:length(S), + NumOfPs = decr_pc(C, NumOfPs0), + Count = Count0 - 1, + MaxLen = sub(MaxLen0, Len), if - Pc1 > 0 -> [S|build(Cs, Pc1, indentation(S, I))]; - true -> [S|build(Cs, Pc1, I)] + NumOfPs > 0 -> [S|build_limited(Cs, NumOfPs, Count, + MaxLen, indentation(S, I))]; + true -> [S|build_limited(Cs, NumOfPs, Count, MaxLen, I)] end; -build([$\n|Cs], Pc, _I) -> [$\n|build(Cs, Pc, 0)]; -build([$\t|Cs], Pc, I) -> [$\t|build(Cs, Pc, ((I + 8) div 8) * 8)]; -build([C|Cs], Pc, I) -> [C|build(Cs, Pc, I+1)]; -build([], _Pc, _I) -> []. +build_limited([$\n|Cs], NumOfPs, Count, MaxLen, _I) -> + [$\n|build_limited(Cs, NumOfPs, Count, MaxLen, 0)]; +build_limited([$\t|Cs], NumOfPs, Count, MaxLen, I) -> + [$\t|build_limited(Cs, NumOfPs, Count, MaxLen, ((I + 8) div 8) * 8)]; +build_limited([C|Cs], NumOfPs, Count, MaxLen, I) -> + [C|build_limited(Cs, NumOfPs, Count, MaxLen, I+1)]; +build_limited([], _, _, _, _) -> []. decr_pc($p, Pc) -> Pc - 1; decr_pc($P, Pc) -> Pc - 1; decr_pc(_, Pc) -> Pc. - %% Calculate the indentation of the end of a string given its start %% indentation. We assume tabs at 8 cols. @@ -251,67 +317,74 @@ indentation([C|Cs], I) -> indentation(Cs, indentation(C, I)); indentation([], I) -> I. -%% control(FormatChar, [Argument], FieldWidth, Adjust, Precision, PadChar, -%% Encoding, Indentation) -> String -%% This is the main dispatch function for the various formatting commands. -%% Field widths and precisions have already been calculated. - -control($w, [A], F, Adj, P, Pad, Enc, _Str, _I) -> - term(io_lib:write(A, [{depth,-1}, {encoding, Enc}]), F, Adj, P, Pad); -control($p, [A], F, Adj, P, Pad, Enc, Str, I) -> - print(A, -1, F, Adj, P, Pad, Enc, Str, I); -control($W, [A,Depth], F, Adj, P, Pad, Enc, _Str, _I) when is_integer(Depth) -> - term(io_lib:write(A, [{depth,Depth}, {encoding, Enc}]), F, Adj, P, Pad); -control($P, [A,Depth], F, Adj, P, Pad, Enc, Str, I) when is_integer(Depth) -> - print(A, Depth, F, Adj, P, Pad, Enc, Str, I); -control($s, [A], F, Adj, P, Pad, latin1, _Str, _I) when is_atom(A) -> +%% control_small(FormatChar, [Argument], FieldWidth, Adjust, Precision, +%% PadChar, Encoding) -> String +%% control_limited(FormatChar, [Argument], FieldWidth, Adjust, Precision, +%% PadChar, Encoding, StringP, ChrsLim, Indentation) -> String +%% These are the dispatch functions for the various formatting controls. + +control_small($s, [A], F, Adj, P, Pad, latin1) when is_atom(A) -> L = iolist_to_chars(atom_to_list(A)), string(L, F, Adj, P, Pad); -control($s, [A], F, Adj, P, Pad, unicode, _Str, _I) when is_atom(A) -> +control_small($s, [A], F, Adj, P, Pad, unicode) when is_atom(A) -> string(atom_to_list(A), F, Adj, P, Pad); -control($s, [L0], F, Adj, P, Pad, latin1, _Str, _I) -> - L = iolist_to_chars(L0), - string(L, F, Adj, P, Pad); -control($s, [L0], F, Adj, P, Pad, unicode, _Str, _I) -> - L = cdata_to_chars(L0), - uniconv(string(L, F, Adj, P, Pad)); -control($e, [A], F, Adj, P, Pad, _Enc, _Str, _I) when is_float(A) -> +control_small($e, [A], F, Adj, P, Pad, _Enc) when is_float(A) -> fwrite_e(A, F, Adj, P, Pad); -control($f, [A], F, Adj, P, Pad, _Enc, _Str, _I) when is_float(A) -> +control_small($f, [A], F, Adj, P, Pad, _Enc) when is_float(A) -> fwrite_f(A, F, Adj, P, Pad); -control($g, [A], F, Adj, P, Pad, _Enc, _Str, _I) when is_float(A) -> +control_small($g, [A], F, Adj, P, Pad, _Enc) when is_float(A) -> fwrite_g(A, F, Adj, P, Pad); -control($b, [A], F, Adj, P, Pad, _Enc, _Str, _I) when is_integer(A) -> +control_small($b, [A], F, Adj, P, Pad, _Enc) when is_integer(A) -> unprefixed_integer(A, F, Adj, base(P), Pad, true); -control($B, [A], F, Adj, P, Pad, _Enc, _Str, _I) when is_integer(A) -> +control_small($B, [A], F, Adj, P, Pad, _Enc) when is_integer(A) -> unprefixed_integer(A, F, Adj, base(P), Pad, false); -control($x, [A,Prefix], F, Adj, P, Pad, _Enc, _Str, _I) when is_integer(A), - is_atom(Prefix) -> +control_small($x, [A,Prefix], F, Adj, P, Pad, _Enc) when is_integer(A), + is_atom(Prefix) -> prefixed_integer(A, F, Adj, base(P), Pad, atom_to_list(Prefix), true); -control($x, [A,Prefix], F, Adj, P, Pad, _Enc, _Str, _I) when is_integer(A) -> +control_small($x, [A,Prefix], F, Adj, P, Pad, _Enc) when is_integer(A) -> true = io_lib:deep_char_list(Prefix), %Check if Prefix a character list prefixed_integer(A, F, Adj, base(P), Pad, Prefix, true); -control($X, [A,Prefix], F, Adj, P, Pad, _Enc, _Str, _I) when is_integer(A), - is_atom(Prefix) -> +control_small($X, [A,Prefix], F, Adj, P, Pad, _Enc) when is_integer(A), + is_atom(Prefix) -> prefixed_integer(A, F, Adj, base(P), Pad, atom_to_list(Prefix), false); -control($X, [A,Prefix], F, Adj, P, Pad, _Enc, _Str, _I) when is_integer(A) -> +control_small($X, [A,Prefix], F, Adj, P, Pad, _Enc) when is_integer(A) -> true = io_lib:deep_char_list(Prefix), %Check if Prefix a character list prefixed_integer(A, F, Adj, base(P), Pad, Prefix, false); -control($+, [A], F, Adj, P, Pad, _Enc, _Str, _I) when is_integer(A) -> +control_small($+, [A], F, Adj, P, Pad, _Enc) when is_integer(A) -> Base = base(P), Prefix = [integer_to_list(Base), $#], prefixed_integer(A, F, Adj, Base, Pad, Prefix, true); -control($#, [A], F, Adj, P, Pad, _Enc, _Str, _I) when is_integer(A) -> +control_small($#, [A], F, Adj, P, Pad, _Enc) when is_integer(A) -> Base = base(P), Prefix = [integer_to_list(Base), $#], prefixed_integer(A, F, Adj, Base, Pad, Prefix, false); -control($c, [A], F, Adj, P, Pad, unicode, _Str, _I) when is_integer(A) -> +control_small($c, [A], F, Adj, P, Pad, unicode) when is_integer(A) -> char(A, F, Adj, P, Pad); -control($c, [A], F, Adj, P, Pad, _Enc, _Str, _I) when is_integer(A) -> +control_small($c, [A], F, Adj, P, Pad, _Enc) when is_integer(A) -> char(A band 255, F, Adj, P, Pad); -control($~, [], F, Adj, P, Pad, _Enc, _Str, _I) -> char($~, F, Adj, P, Pad); -control($n, [], F, Adj, P, Pad, _Enc, _Str, _I) -> newline(F, Adj, P, Pad); -control($i, [_A], _F, _Adj, _P, _Pad, _Enc, _Str, _I) -> []. +control_small($~, [], F, Adj, P, Pad, _Enc) -> char($~, F, Adj, P, Pad); +control_small($n, [], F, Adj, P, Pad, _Enc) -> newline(F, Adj, P, Pad); +control_small($i, [_A], _F, _Adj, _P, _Pad, _Enc) -> []; +control_small(_C, _As, _F, _Adj, _P, _Pad, _Enc) -> not_small. + +control_limited($s, [L0], F, Adj, P, Pad, latin1, _Str, CL, _I) -> + L = iolist_to_chars(L0), + string(limit_string(L, F, CL), limit_field(F, CL), Adj, P, Pad); +control_limited($s, [L0], F, Adj, P, Pad, unicode, _Str, CL, _I) -> + L = cdata_to_chars(L0), + uniconv(string(limit_string(L, F, CL), limit_field(F, CL), Adj, P, Pad)); +control_limited($w, [A], F, Adj, P, Pad, Enc, _Str, CL, _I) -> + Chars = io_lib:write(A, [{depth, -1}, {encoding, Enc}, {chars_limit, CL}]), + term(Chars, F, Adj, P, Pad); +control_limited($p, [A], F, Adj, P, Pad, Enc, Str, CL, I) -> + print(A, -1, F, Adj, P, Pad, Enc, Str, CL, I); +control_limited($W, [A,Depth], F, Adj, P, Pad, Enc, _Str, CL, _I) + when is_integer(Depth) -> + Chars = io_lib:write(A, [{depth, Depth}, {encoding, Enc}, {chars_limit, CL}]), + term(Chars, F, Adj, P, Pad); +control_limited($P, [A,Depth], F, Adj, P, Pad, Enc, Str, CL, I) + when is_integer(Depth) -> + print(A, Depth, F, Adj, P, Pad, Enc, Str, CL, I). -ifdef(UNICODE_AS_BINARIES). uniconv(C) -> @@ -348,12 +421,13 @@ term(T, F, Adj, P0, Pad) -> %% Print a term. Field width sets maximum line length, Precision sets %% initial indentation. -print(T, D, none, Adj, P, Pad, E, Str, I) -> - print(T, D, 80, Adj, P, Pad, E, Str, I); -print(T, D, F, Adj, none, Pad, E, Str, I) -> - print(T, D, F, Adj, I+1, Pad, E, Str, I); -print(T, D, F, right, P, _Pad, Enc, Str, _I) -> - Options = [{column, P}, +print(T, D, none, Adj, P, Pad, E, Str, ChLim, I) -> + print(T, D, 80, Adj, P, Pad, E, Str, ChLim, I); +print(T, D, F, Adj, none, Pad, E, Str, ChLim, I) -> + print(T, D, F, Adj, I+1, Pad, E, Str, ChLim, I); +print(T, D, F, right, P, _Pad, Enc, Str, ChLim, _I) -> + Options = [{chars_limit, ChLim}, + {column, P}, {line_length, F}, {depth, D}, {encoding, Enc}, @@ -670,6 +744,18 @@ cdata_to_chars(B) when is_binary(B) -> _ -> binary_to_list(B) end. +limit_string(S, F, CharsLimit) when CharsLimit < 0; CharsLimit >= F -> S; +limit_string(S, _F, CharsLimit) -> + case string:length(S) =< CharsLimit of + true -> S; + false -> [string:slice(S, 0, sub(CharsLimit, 3)), "..."] + end. + +limit_field(F, CharsLimit) when CharsLimit < 0; F =:= none -> + F; +limit_field(F, CharsLimit) -> + max(3, min(F, CharsLimit)). + %% string(String, Field, Adjust, Precision, PadChar) string(S, none, _Adj, none, _Pad) -> S; @@ -783,3 +869,15 @@ lowercase([H|T]) -> [H|lowercase(T)]; lowercase([]) -> []. + +%% Make sure T does change sign. +sub(T, _) when T < 0 -> T; +sub(T, E) when T >= E -> T - E; +sub(_, _) -> 0. + +get_option(Key, TupleList, Default) -> + case lists:keyfind(Key, 1, TupleList) of + false -> Default; + {Key, Value} -> Value; + _ -> Default + end. diff --git a/lib/stdlib/src/io_lib_pretty.erl b/lib/stdlib/src/io_lib_pretty.erl index 89e1931d2d..3d5a979b3e 100644 --- a/lib/stdlib/src/io_lib_pretty.erl +++ b/lib/stdlib/src/io_lib_pretty.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2017. All Rights Reserved. +%% Copyright Ericsson AB 1996-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -26,6 +26,9 @@ -export([print/1,print/2,print/3,print/4,print/5,print/6]). +%% To be used by io_lib only. +-export([intermediate/6, write/1]). + %%% %%% Exported functions %%% @@ -45,20 +48,23 @@ print(Term) -> %% Used by the shell for printing records and for Unicode. -type rec_print_fun() :: fun((Tag :: atom(), NFields :: non_neg_integer()) -> - no | [FieldName :: atom()]). + 'no' | [FieldName :: atom()]). -type column() :: integer(). +-type encoding() :: epp:source_encoding() | 'unicode'. -type line_length() :: pos_integer(). -type depth() :: integer(). --type max_chars() :: integer(). +-type line_max_chars() :: integer(). +-type chars_limit() :: integer(). -type chars() :: io_lib:chars(). --type option() :: {column, column()} - | {line_length, line_length()} - | {depth, depth()} - | {max_chars, max_chars()} - | {record_print_fun, rec_print_fun()} - | {strings, boolean()} - | {encoding, latin1 | utf8 | unicode}. +-type option() :: {'chars_limit', chars_limit()} + | {'column', column()} + | {'depth', depth()} + | {'encoding', encoding()} + | {'line_length', line_length()} + | {'line_max_chars', line_max_chars()} + | {'record_print_fun', rec_print_fun()} + | {'strings', boolean()}. -type options() :: [option()]. -spec print(term(), rec_print_fun()) -> chars(); @@ -68,11 +74,12 @@ print(Term, Options) when is_list(Options) -> Col = get_option(column, Options, 1), Ll = get_option(line_length, Options, 80), D = get_option(depth, Options, -1), - M = get_option(max_chars, Options, -1), + M = get_option(line_max_chars, Options, -1), + T = get_option(chars_limit, Options, -1), RecDefFun = get_option(record_print_fun, Options, no_fun), Encoding = get_option(encoding, Options, epp:default_encoding()), Strings = get_option(strings, Options, true), - print(Term, Col, Ll, D, M, RecDefFun, Encoding, Strings); + print(Term, Col, Ll, D, M, T, RecDefFun, Encoding, Strings); print(Term, RecDefFun) -> print(Term, -1, RecDefFun). @@ -84,35 +91,43 @@ print(Term, Depth, RecDefFun) -> -spec print(term(), column(), line_length(), depth()) -> chars(). print(Term, Col, Ll, D) -> - print(Term, Col, Ll, D, _M=-1, no_fun, latin1, true). + print(Term, Col, Ll, D, _M=-1, _T=-1, no_fun, latin1, true). -spec print(term(), column(), line_length(), depth(), rec_print_fun()) -> chars(). print(Term, Col, Ll, D, RecDefFun) -> print(Term, Col, Ll, D, _M=-1, RecDefFun). --spec print(term(), column(), line_length(), depth(), max_chars(), +-spec print(term(), column(), line_length(), depth(), line_max_chars(), rec_print_fun()) -> chars(). print(Term, Col, Ll, D, M, RecDefFun) -> - print(Term, Col, Ll, D, M, RecDefFun, latin1, true). + print(Term, Col, Ll, D, M, _T=-1, RecDefFun, latin1, true). %% D = Depth, default -1 (infinite), or LINEMAX=30 when printing from shell +%% T = chars_limit, that is, maximal number of characters, default -1 +%% Used together with D to limit the output. It is possible that +%% more than T characters are returned. %% Col = current column, default 1 %% Ll = line length/~p field width, default 80 %% M = CHAR_MAX (-1 if no max, 60 when printing from shell) -print(_, _, _, 0, _M, _RF, _Enc, _Str) -> "..."; -print(Term, Col, Ll, D, M, RecDefFun, Enc, Str) when Col =< 0 -> +print(_, _, _, 0, _M, _T, _RF, _Enc, _Str) -> "..."; +print(_, _, _, _D, _M, 0, _RF, _Enc, _Str) -> "..."; +print(Term, Col, Ll, D, M, T, RecDefFun, Enc, Str) when Col =< 0 -> %% ensure Col is at least 1 - print(Term, 1, Ll, D, M, RecDefFun, Enc, Str); -print(Atom, _Col, _Ll, _D, _M, _RF, Enc, _Str) when is_atom(Atom) -> + print(Term, 1, Ll, D, M, T, RecDefFun, Enc, Str); +print(Atom, _Col, _Ll, _D, _M, _T, _RF, Enc, _Str) when is_atom(Atom) -> write_atom(Atom, Enc); -print(Term, Col, Ll, D, M0, RecDefFun, Enc, Str) when is_tuple(Term); - is_list(Term); - is_map(Term); - is_bitstring(Term) -> +print(Term, Col, Ll, D, M0, T, RecDefFun, Enc, Str) when is_tuple(Term); + is_list(Term); + is_map(Term); + is_bitstring(Term) -> %% preprocess and compute total number of chars - If = {_S, Len} = print_length(Term, D, RecDefFun, Enc, Str), + {_, Len, _Dots, _} = If = + case T < 0 of + true -> print_length(Term, D, T, RecDefFun, Enc, Str); + false -> intermediate(Term, D, T, RecDefFun, Enc, Str) + end, %% use Len as CHAR_MAX if M0 = -1 M = max_cs(M0, Len), if @@ -126,7 +141,7 @@ print(Term, Col, Ll, D, M0, RecDefFun, Enc, Str) when is_tuple(Term); 1), pp(If, Col, Ll, M, TInd, indent(Col), 0, 0) end; -print(Term, _Col, _Ll, _D, _M, _RF, _Enc, _Str) -> +print(Term, _Col, _Ll, _D, _M, _T, _RF, _Enc, _Str) -> %% atomic data types (bignums, atoms, ...) are never truncated io_lib:write(Term). @@ -147,28 +162,28 @@ max_cs(M, _Len) -> ?ATM(element(3, element(1, Pair)))). % Value -define(ATM_FLD(Field), ?ATM(element(4, element(1, Field)))). -pp({_S, Len} = If, Col, Ll, M, _TInd, _Ind, LD, W) +pp({_S,Len,_,_} = If, Col, Ll, M, _TInd, _Ind, LD, W) when Len < Ll - Col - LD, Len + W + LD =< M -> write(If); -pp({{list,L}, _Len}, Col, Ll, M, TInd, Ind, LD, W) -> +pp({{list,L}, _Len, _, _}, Col, Ll, M, TInd, Ind, LD, W) -> [$[, pp_list(L, Col + 1, Ll, M, TInd, indent(1, Ind), LD, $|, W + 1), $]]; -pp({{tuple,true,L}, _Len}, Col, Ll, M, TInd, Ind, LD, W) -> +pp({{tuple,true,L}, _Len, _, _}, Col, Ll, M, TInd, Ind, LD, W) -> [${, pp_tag_tuple(L, Col, Ll, M, TInd, Ind, LD, W + 1), $}]; -pp({{tuple,false,L}, _Len}, Col, Ll, M, TInd, Ind, LD, W) -> +pp({{tuple,false,L}, _Len, _, _}, Col, Ll, M, TInd, Ind, LD, W) -> [${, pp_list(L, Col + 1, Ll, M, TInd, indent(1, Ind), LD, $,, W + 1), $}]; -pp({{map,Pairs},_Len}, Col, Ll, M, TInd, Ind, LD, W) -> +pp({{map,Pairs}, _Len, _, _}, Col, Ll, M, TInd, Ind, LD, W) -> [$#, ${, pp_map(Pairs, Col + 2, Ll, M, TInd, indent(2, Ind), LD, W + 1), $}]; -pp({{record,[{Name,NLen} | L]}, _Len}, Col, Ll, M, TInd, Ind, LD, W) -> +pp({{record,[{Name,NLen} | L]}, _Len, _, _}, Col, Ll, M, TInd, Ind, LD, W) -> [Name, ${, pp_record(L, NLen, Col, Ll, M, TInd, Ind, LD, W + NLen+1), $}]; -pp({{bin,S}, _Len}, Col, Ll, M, _TInd, Ind, LD, W) -> +pp({{bin,S}, _Len, _, _}, Col, Ll, M, _TInd, Ind, LD, W) -> pp_binary(S, Col + 2, Ll, M, indent(2, Ind), LD, W); -pp({S, _Len}, _Col, _Ll, _M, _TInd, _Ind, _LD, _W) -> +pp({S,_Len,_,_}, _Col, _Ll, _M, _TInd, _Ind, _LD, _W) -> S. %% Print a tagged tuple by indenting the rest of the elements %% differently to the tag. Tuple has size >= 2. -pp_tag_tuple([{Tag,Tlen} | L], Col, Ll, M, TInd, Ind, LD, W) -> +pp_tag_tuple([{Tag,Tlen,_,_} | L], Col, Ll, M, TInd, Ind, LD, W) -> %% this uses TInd TagInd = Tlen + 2, Tcol = Col + TagInd, @@ -184,18 +199,18 @@ pp_tag_tuple([{Tag,Tlen} | L], Col, Ll, M, TInd, Ind, LD, W) -> end. pp_map([], _Col, _Ll, _M, _TInd, _Ind, _LD, _W) -> - ""; -pp_map({dots, _}, _Col, _Ll, _M, _TInd, _Ind, _LD, _W) -> - "..."; + ""; % cannot happen +pp_map({dots, _, _, _}, _Col, _Ll, _M, _TInd, _Ind, _LD, _W) -> + "..."; % cannot happen pp_map([P | Ps], Col, Ll, M, TInd, Ind, LD, W) -> {PS, PW} = pp_pair(P, Col, Ll, M, TInd, Ind, last_depth(Ps, LD), W), [PS | pp_pairs_tail(Ps, Col, Col + PW, Ll, M, TInd, Ind, LD, PW)]. pp_pairs_tail([], _Col0, _Col, _Ll, _M, _TInd, _Ind, _LD, _W) -> ""; -pp_pairs_tail({dots, _}, _Col0, _Col, _M, _Ll, _TInd, _Ind, _LD, _W) -> +pp_pairs_tail({dots, _, _, _}, _Col0, _Col, _M, _Ll, _TInd, _Ind, _LD, _W) -> ",..."; -pp_pairs_tail([{_, Len}=P | Ps], Col0, Col, Ll, M, TInd, Ind, LD, W) -> +pp_pairs_tail([{_, Len, _, _}=P | Ps], Col0, Col, Ll, M, TInd, Ind, LD, W) -> LD1 = last_depth(Ps, LD), ELen = 1 + Len, if @@ -209,7 +224,7 @@ pp_pairs_tail([{_, Len}=P | Ps], Col0, Col, Ll, M, TInd, Ind, LD, W) -> pp_pairs_tail(Ps, Col0, Col0 + PW, Ll, M, TInd, Ind, LD, PW)] end. -pp_pair({_, Len}=Pair, Col, Ll, M, _TInd, _Ind, LD, W) +pp_pair({_, Len, _, _}=Pair, Col, Ll, M, _TInd, _Ind, LD, W) when Len < Ll - Col - LD, Len + W + LD =< M -> {write_pair(Pair), if ?ATM_PAIR(Pair) -> @@ -217,7 +232,7 @@ pp_pair({_, Len}=Pair, Col, Ll, M, _TInd, _Ind, LD, W) true -> Ll % force nl end}; -pp_pair({{map_pair, K, V}, _Len}, Col0, Ll, M, TInd, Ind0, LD, W) -> +pp_pair({{map_pair, K, V}, _Len, _, _}, Col0, Ll, M, TInd, Ind0, LD, W) -> I = map_value_indent(TInd), Ind = indent(I, Ind0), {[pp(K, Col0, Ll, M, TInd, Ind0, LD, W), " =>\n", @@ -225,7 +240,7 @@ pp_pair({{map_pair, K, V}, _Len}, Col0, Ll, M, TInd, Ind0, LD, W) -> pp_record([], _Nlen, _Col, _Ll, _M, _TInd, _Ind, _LD, _W) -> ""; -pp_record({dots, _}, _Nlen, _Col, _Ll, _M, _TInd, _Ind, _LD, _W) -> +pp_record({dots, _, _, _}, _Nlen, _Col, _Ll, _M, _TInd, _Ind, _LD, _W) -> "..."; pp_record([F | Fs], Nlen, Col0, Ll, M, TInd, Ind0, LD, W0) -> Nind = Nlen + 1, @@ -235,9 +250,9 @@ pp_record([F | Fs], Nlen, Col0, Ll, M, TInd, Ind0, LD, W0) -> pp_fields_tail([], _Col0, _Col, _Ll, _M, _TInd, _Ind, _LD, _W) -> ""; -pp_fields_tail({dots, _}, _Col0, _Col, _M, _Ll, _TInd, _Ind, _LD, _W) -> +pp_fields_tail({dots, _, _ ,_}, _Col0, _Col, _M, _Ll, _TInd, _Ind, _LD, _W) -> ",..."; -pp_fields_tail([{_, Len}=F | Fs], Col0, Col, Ll, M, TInd, Ind, LD, W) -> +pp_fields_tail([{_, Len, _, _}=F | Fs], Col0, Col, Ll, M, TInd, Ind, LD, W) -> LD1 = last_depth(Fs, LD), ELen = 1 + Len, if @@ -251,7 +266,7 @@ pp_fields_tail([{_, Len}=F | Fs], Col0, Col, Ll, M, TInd, Ind, LD, W) -> pp_fields_tail(Fs, Col0, Col0 + FW, Ll, M, TInd, Ind, LD, FW)] end. -pp_field({_, Len}=Fl, Col, Ll, M, _TInd, _Ind, LD, W) +pp_field({_, Len, _, _}=Fl, Col, Ll, M, _TInd, _Ind, LD, W) when Len < Ll - Col - LD, Len + W + LD =< M -> {write_field(Fl), if ?ATM_FLD(Fl) -> @@ -259,7 +274,7 @@ pp_field({_, Len}=Fl, Col, Ll, M, _TInd, _Ind, LD, W) true -> Ll % force nl end}; -pp_field({{field, Name, NameL, F}, _Len}, Col0, Ll, M, TInd, Ind0, LD, W0) -> +pp_field({{field, Name, NameL, F},_,_, _}, Col0, Ll, M, TInd, Ind0, LD, W0) -> {Col, Ind, S, W} = rec_indent(NameL, TInd, Col0, Ind0, W0 + NameL), Sep = case S of [$\n | _] -> " ="; @@ -286,15 +301,15 @@ rec_indent(RInd, TInd, Col0, Ind0, W0) -> end, {Col, Ind, S, W}. -pp_list({dots, _}, _Col0, _Ll, _M, _TInd, _Ind, _LD, _S, _W) -> +pp_list({dots, _, _, _}, _Col0, _Ll, _M, _TInd, _Ind, _LD, _S, _W) -> "..."; pp_list([E | Es], Col0, Ll, M, TInd, Ind, LD, S, W) -> {ES, WE} = pp_element(E, Col0, Ll, M, TInd, Ind, last_depth(Es, LD), W), [ES | pp_tail(Es, Col0, Col0 + WE, Ll, M, TInd, Ind, LD, S, W + WE)]. pp_tail([], _Col0, _Col, _Ll, _M, _TInd, _Ind, _LD, _S, _W) -> - ""; -pp_tail([{_, Len}=E | Es], Col0, Col, Ll, M, TInd, Ind, LD, S, W) -> + []; +pp_tail([{_, Len, _, _}=E | Es], Col0, Col, Ll, M, TInd, Ind, LD, S, W) -> LD1 = last_depth(Es, LD), ELen = 1 + Len, if @@ -307,9 +322,9 @@ pp_tail([{_, Len}=E | Es], Col0, Col, Ll, M, TInd, Ind, LD, S, W) -> [$,, $\n, Ind, ES | pp_tail(Es, Col0, Col0 + WE, Ll, M, TInd, Ind, LD, S, WE)] end; -pp_tail({dots, _}, _Col0, _Col, _Ll, _M, _TInd, _Ind, _LD, S, _W) -> +pp_tail({dots, _, _, _}, _Col0, _Col, _Ll, _M, _TInd, _Ind, _LD, S, _W) -> [S | "..."]; -pp_tail({_, Len}=E, _Col0, Col, Ll, M, _TInd, _Ind, LD, S, W) +pp_tail({_, Len, _, _}=E, _Col0, Col, Ll, M, _TInd, _Ind, LD, S, W) when Len + 1 < Ll - Col - (LD + 1), Len + 1 + W + (LD + 1) =< M, ?ATM(E) -> @@ -317,7 +332,7 @@ pp_tail({_, Len}=E, _Col0, Col, Ll, M, _TInd, _Ind, LD, S, W) pp_tail(E, Col0, _Col, Ll, M, TInd, Ind, LD, S, _W) -> [S, $\n, Ind | pp(E, Col0, Ll, M, TInd, Ind, LD + 1, 0)]. -pp_element({_, Len}=E, Col, Ll, M, _TInd, _Ind, LD, W) +pp_element({_, Len, _, _}=E, Col, Ll, M, _TInd, _Ind, LD, W) when Len < Ll - Col - LD, Len + W + LD =< M, ?ATM(E) -> {write(E), Len}; pp_element(E, Col, Ll, M, TInd, Ind, LD, W) -> @@ -348,42 +363,42 @@ pp_binary(S, N, _N0, Ind) -> end. %% write the whole thing on a single line -write({{tuple, _IsTagged, L}, _}) -> +write({{tuple, _IsTagged, L}, _, _, _}) -> [${, write_list(L, $,), $}]; -write({{list, L}, _}) -> +write({{list, L}, _, _, _}) -> [$[, write_list(L, $|), $]]; -write({{map, Pairs}, _}) -> +write({{map, Pairs}, _, _, _}) -> [$#,${, write_list(Pairs, $,), $}]; -write({{map_pair, _K, _V}, _}=Pair) -> +write({{map_pair, _K, _V}, _, _, _}=Pair) -> write_pair(Pair); -write({{record, [{Name,_} | L]}, _}) -> +write({{record, [{Name,_} | L]}, _, _, _}) -> [Name, ${, write_fields(L), $}]; -write({{bin, S}, _}) -> +write({{bin, S}, _, _, _}) -> S; -write({S, _}) -> +write({S, _, _, _}) -> S. -write_pair({{map_pair, K, V}, _}) -> +write_pair({{map_pair, K, V}, _, _, _}) -> [write(K), " => ", write(V)]. write_fields([]) -> ""; -write_fields({dots, _}) -> +write_fields({dots, _, _, _}) -> "..."; write_fields([F | Fs]) -> [write_field(F) | write_fields_tail(Fs)]. write_fields_tail([]) -> ""; -write_fields_tail({dots, _}) -> +write_fields_tail({dots, _, _, _}) -> ",..."; write_fields_tail([F | Fs]) -> [$,, write_field(F) | write_fields_tail(Fs)]. -write_field({{field, Name, _NameL, F}, _}) -> +write_field({{field, Name, _NameL, F}, _, _, _}) -> [Name, " = " | write(F)]. -write_list({dots, _}, _S) -> +write_list({dots, _, _, _}, _S) -> "..."; write_list([E | Es], S) -> [write(E) | write_tail(Es, S)]. @@ -392,192 +407,359 @@ write_tail([], _S) -> []; write_tail([E | Es], S) -> [$,, write(E) | write_tail(Es, S)]; -write_tail({dots, _}, S) -> +write_tail({dots, _, _, _}, S) -> [S | "..."]; write_tail(E, S) -> [S | write(E)]. +-type more() :: fun((chars_limit(), DeltaDepth :: non_neg_integer()) -> + intermediate_format()). + +-type if_list() :: maybe_improper_list(intermediate_format(), + {'dots', non_neg_integer(), + non_neg_integer(), more()}). + +-type intermediate_format() :: + {chars() + | {'bin', chars()} + | 'dots' + | {'field', Name :: chars(), NameLen :: non_neg_integer(), + intermediate_format()} + | {'list', if_list()} + | {'map', if_list()} + | {'map_pair', K :: intermediate_format(), + V :: intermediate_format()} + | {'record', [{Name :: chars(), NameLen :: non_neg_integer()} + | if_list()]} + | {'tuple', IsTagged :: boolean(), if_list()}, + Len :: non_neg_integer(), + NumOfDots :: non_neg_integer(), + More :: more() | 'no_more' + }. + +-spec intermediate(term(), depth(), pos_integer(), rec_print_fun(), + encoding(), boolean()) -> intermediate_format(). + +intermediate(Term, D, T, RF, Enc, Str) when T > 0 -> + D0 = 1, + If = print_length(Term, D0, T, RF, Enc, Str), + case If of + {_, Len, Dots, _} when Dots =:= 0; Len > T; D =:= 1 -> + If; + _ -> + find_upper(If, Term, T, D0, 2, D, RF, Enc, Str) + end. + +find_upper(Lower, Term, T, Dl, Dd, D, RF, Enc, Str) -> + Dd2 = Dd * 2, + D1 = case D < 0 of + true -> Dl + Dd2; + false -> min(Dl + Dd2, D) + end, + If = expand(Lower, T, D1 - Dl), + case If of + {_, _, _Dots=0, _} -> % even if Len > T + If; + {_, Len, _, _} when Len =< T, D1 < D orelse D < 0 -> + find_upper(If, Term, T, D1, Dd2, D, RF, Enc, Str); + _ -> + search_depth(Lower, If, Term, T, Dl, D1, RF, Enc, Str) + end. + +%% Lower has NumOfDots > 0 and Len =< T. +%% Upper has NumOfDots > 0 and Len > T. +search_depth(Lower, Upper, _Term, T, Dl, Du, _RF, _Enc, _Str) + when Du - Dl =:= 1 -> + %% The returned intermediate format has Len >= T. + case Lower of + {_, T, _, _} -> + Lower; + _ -> + Upper + end; +search_depth(Lower, Upper, Term, T, Dl, Du, RF, Enc, Str) -> + D1 = (Dl + Du) div 2, + If = expand(Lower, T, D1 - Dl), + case If of + {_, Len, _, _} when Len > T -> + %% Len can be greater than Upper's length. + %% This is a bit expensive since the work to + %% crate Upper is wasted. It is the price + %% to pay to get a more balanced output. + search_depth(Lower, If, Term, T, Dl, D1, RF, Enc, Str); + _ -> + search_depth(If, Upper, Term, T, D1, Du, RF, Enc, Str) + end. + %% The depth (D) is used for extracting and counting the characters to %% print. The structure is kept so that the returned intermediate %% format can be formatted. The separators (list, tuple, record, map) are %% counted but need to be added later. %% D =/= 0 -print_length([], _D, _RF, _Enc, _Str) -> - {"[]", 2}; -print_length({}, _D, _RF, _Enc, _Str) -> - {"{}", 2}; -print_length(#{}=M, _D, _RF, _Enc, _Str) when map_size(M) =:= 0 -> - {"#{}", 3}; -print_length(Atom, _D, _RF, Enc, _Str) when is_atom(Atom) -> +print_length([], _D, _T, _RF, _Enc, _Str) -> + {"[]", 2, 0, no_more}; +print_length({}, _D, _T, _RF, _Enc, _Str) -> + {"{}", 2, 0, no_more}; +print_length(#{}=M, _D, _T, _RF, _Enc, _Str) when map_size(M) =:= 0 -> + {"#{}", 3, 0, no_more}; +print_length(Atom, _D, _T, _RF, Enc, _Str) when is_atom(Atom) -> S = write_atom(Atom, Enc), - {S, lists:flatlength(S)}; -print_length(List, D, RF, Enc, Str) when is_list(List) -> + {S, string:length(S), 0, no_more}; +print_length(List, D, T, RF, Enc, Str) when is_list(List) -> %% only flat lists are "printable" - case Str andalso printable_list(List, D, Enc) of + case Str andalso printable_list(List, D, T, Enc) of true -> %% print as string, escaping double-quotes in the list S = write_string(List, Enc), - {S, length(S)}; - %% Truncated lists could break some existing code. - % {true, Prefix} -> - % S = write_string(Prefix, Enc), - % {[S | "..."], 3 + length(S)}; + {S, string:length(S), 0, no_more}; + {true, Prefix} -> + %% Truncated lists when T < 0 could break some existing code. + S = write_string(Prefix, Enc), + %% NumOfDots = 0 to avoid looping--increasing the depth + %% does not make Prefix longer. + {[S | "..."], 3 + string:length(S), 0, no_more}; false -> - print_length_list(List, D, RF, Enc, Str) + case print_length_list(List, D, T, RF, Enc, Str) of + {What, Len, Dots, _More} when Dots > 0 -> + More = fun(T1, Dd) -> + ?FUNCTION_NAME(List, D+Dd, T1, RF, Enc, Str) + end, + {What, Len, Dots, More}; + If -> + If + end end; -print_length(Fun, _D, _RF, _Enc, _Str) when is_function(Fun) -> +print_length(Fun, _D, _T, _RF, _Enc, _Str) when is_function(Fun) -> S = io_lib:write(Fun), - {S, iolist_size(S)}; -print_length(R, D, RF, Enc, Str) when is_atom(element(1, R)), - is_function(RF) -> + {S, iolist_size(S), 0, no_more}; +print_length(R, D, T, RF, Enc, Str) when is_atom(element(1, R)), + is_function(RF) -> case RF(element(1, R), tuple_size(R) - 1) of no -> - print_length_tuple(R, D, RF, Enc, Str); + print_length_tuple(R, D, T, RF, Enc, Str); RDefs -> - print_length_record(R, D, RF, RDefs, Enc, Str) + print_length_record(R, D, T, RF, RDefs, Enc, Str) end; -print_length(Tuple, D, RF, Enc, Str) when is_tuple(Tuple) -> - print_length_tuple(Tuple, D, RF, Enc, Str); -print_length(Map, D, RF, Enc, Str) when is_map(Map) -> - print_length_map(Map, D, RF, Enc, Str); -print_length(<<>>, _D, _RF, _Enc, _Str) -> - {"<<>>", 4}; -print_length(<<_/bitstring>>, 1, _RF, _Enc, _Str) -> - {"<<...>>", 7}; -print_length(<<_/bitstring>>=Bin, D, _RF, Enc, Str) -> - case bit_size(Bin) rem 8 of - 0 -> - D1 = D - 1, - case Str andalso printable_bin(Bin, D1, Enc) of - {true, List} when is_list(List) -> - S = io_lib:write_string(List, $"), %" - {[$<,$<,S,$>,$>], 4 + length(S)}; - {false, List} when is_list(List) -> - S = io_lib:write_string(List, $"), %" - {[$<,$<,S,"/utf8>>"], 9 + length(S)}; - {true, true, Prefix} -> - S = io_lib:write_string(Prefix, $"), %" - {[$<,$<, S | "...>>"], 7 + length(S)}; - {false, true, Prefix} -> - S = io_lib:write_string(Prefix, $"), %" - {[$<,$<, S | "/utf8...>>"], 12 + length(S)}; - false -> - S = io_lib:write(Bin, D), - {{bin,S}, iolist_size(S)} - end; - _ -> - S = io_lib:write(Bin, D), - {{bin,S}, iolist_size(S)} +print_length(Tuple, D, T, RF, Enc, Str) when is_tuple(Tuple) -> + print_length_tuple(Tuple, D, T, RF, Enc, Str); +print_length(Map, D, T, RF, Enc, Str) when is_map(Map) -> + print_length_map(Map, D, T, RF, Enc, Str); +print_length(<<>>, _D, _T, _RF, _Enc, _Str) -> + {"<<>>", 4, 0, no_more}; +print_length(<<_/bitstring>> = Bin, 1, _T, RF, Enc, Str) -> + More = fun(T1, Dd) -> ?FUNCTION_NAME(Bin, 1+Dd, T1, RF, Enc, Str) end, + {"<<...>>", 7, 3, More}; +print_length(<<_/bitstring>> = Bin, D, T, RF, Enc, Str) -> + D1 = D - 1, + case + Str andalso + (bit_size(Bin) rem 8) =:= 0 andalso + printable_bin0(Bin, D1, tsub(T, 6), Enc) + of + {true, List} when is_list(List) -> + S = io_lib:write_string(List, $"), %" + {[$<,$<,S,$>,$>], 4 + length(S), 0, no_more}; + {false, List} when is_list(List) -> + S = io_lib:write_string(List, $"), %" + {[$<,$<,S,"/utf8>>"], 9 + string:length(S), 0, no_more}; + {true, true, Prefix} -> + S = io_lib:write_string(Prefix, $"), %" + More = fun(T1, Dd) -> + ?FUNCTION_NAME(Bin, D+Dd, T1, RF, Enc, Str) + end, + {[$<,$<,S|"...>>"], 7 + length(S), 3, More}; + {false, true, Prefix} -> + S = io_lib:write_string(Prefix, $"), %" + More = fun(T1, Dd) -> + ?FUNCTION_NAME(Bin, D+Dd, T1, RF, Enc, Str) + end, + {[$<,$<,S|"/utf8...>>"], 12 + string:length(S), 3, More}; + false -> + case io_lib:write_binary(Bin, D, T) of + {S, <<>>} -> + {{bin, S}, iolist_size(S), 0, no_more}; + {S, _Rest} -> + More = fun(T1, Dd) -> + ?FUNCTION_NAME(Bin, D+Dd, T1, RF, Enc, Str) + end, + {{bin, S}, iolist_size(S), 3, More} + end end; -print_length(Term, _D, _RF, _Enc, _Str) -> +print_length(Term, _D, _T, _RF, _Enc, _Str) -> S = io_lib:write(Term), %% S can contain unicode, so iolist_size(S) cannot be used here - {S, string:length(S)}. - -print_length_map(_Map, 1, _RF, _Enc, _Str) -> - {"#{...}", 6}; -print_length_map(Map, D, RF, Enc, Str) when is_map(Map) -> - Pairs = print_length_map_pairs(limit_map(maps:iterator(Map), D, []), D, RF, Enc, Str), - {{map, Pairs}, list_length(Pairs, 3)}. - -limit_map(_I, 0, Acc) -> - Acc; -limit_map(I, D, Acc) -> - case maps:next(I) of - {K, V, NextI} -> - limit_map(NextI, D-1, [{K,V} | Acc]); - none -> - Acc - end. - -print_length_map_pairs([], _D, _RF, _Enc, _Str) -> + {S, string:length(S), 0, no_more}. + +print_length_map(Map, 1, _T, RF, Enc, Str) -> + More = fun(T1, Dd) -> ?FUNCTION_NAME(Map, 1+Dd, T1, RF, Enc, Str) end, + {"#{...}", 6, 3, More}; +print_length_map(Map, D, T, RF, Enc, Str) when is_map(Map) -> + Next = maps:next(maps:iterator(Map)), + PairsS = print_length_map_pairs(Next, D, D - 1, tsub(T, 3), RF, Enc, Str), + {Len, Dots} = list_length(PairsS, 3, 0), + {{map, PairsS}, Len, Dots, no_more}. + +print_length_map_pairs(none, _D, _D0, _T, _RF, _Enc, _Str) -> []; -print_length_map_pairs(_Pairs, 1, _RF, _Enc, _Str) -> - {dots, 3}; -print_length_map_pairs([{K, V} | Pairs], D, RF, Enc, Str) -> - [print_length_map_pair(K, V, D - 1, RF, Enc, Str) | - print_length_map_pairs(Pairs, D - 1, RF, Enc, Str)]. - -print_length_map_pair(K, V, D, RF, Enc, Str) -> - {KS, KL} = print_length(K, D, RF, Enc, Str), - {VS, VL} = print_length(V, D, RF, Enc, Str), +print_length_map_pairs(Term, D, D0, T, RF, Enc, Str) when D =:= 1; T =:= 0-> + More = fun(T1, Dd) -> + ?FUNCTION_NAME(Term, D+Dd, D0, T1, RF, Enc, Str) + end, + {dots, 3, 3, More}; +print_length_map_pairs({K, V, Iter}, D, D0, T, RF, Enc, Str) -> + Pair1 = print_length_map_pair(K, V, D0, tsub(T, 1), RF, Enc, Str), + {_, Len1, _, _} = Pair1, + Next = maps:next(Iter), + [Pair1 | + print_length_map_pairs(Next, D - 1, D0, tsub(T, Len1+1), RF, Enc, Str)]. + +print_length_map_pair(K, V, D, T, RF, Enc, Str) -> + {_, KL, KD, _} = P1 = print_length(K, D, T, RF, Enc, Str), KL1 = KL + 4, - {{map_pair, {KS, KL1}, {VS, VL}}, KL1 + VL}. - -print_length_tuple(_Tuple, 1, _RF, _Enc, _Str) -> - {"{...}", 5}; -print_length_tuple(Tuple, D, RF, Enc, Str) -> - L = print_length_list1(tuple_to_list(Tuple), D, RF, Enc, Str), + {_, VL, VD, _} = P2 = print_length(V, D, tsub(T, KL1), RF, Enc, Str), + {{map_pair, P1, P2}, KL1 + VL, KD + VD, no_more}. + +print_length_tuple(Tuple, 1, _T, RF, Enc, Str) -> + More = fun(T1, Dd) -> ?FUNCTION_NAME(Tuple, 1+Dd, T1, RF, Enc, Str) end, + {"{...}", 5, 3, More}; +print_length_tuple(Tuple, D, T, RF, Enc, Str) -> + L = print_length_tuple1(Tuple, 1, D, tsub(T, 2), RF, Enc, Str), IsTagged = is_atom(element(1, Tuple)) and (tuple_size(Tuple) > 1), - {{tuple,IsTagged,L}, list_length(L, 2)}. + {Len, Dots} = list_length(L, 2, 0), + {{tuple,IsTagged,L}, Len, Dots, no_more}. -print_length_record(_Tuple, 1, _RF, _RDefs, _Enc, _Str) -> - {"{...}", 5}; -print_length_record(Tuple, D, RF, RDefs, Enc, Str) -> +print_length_tuple1(Tuple, I, _D, _T, _RF, _Enc, _Str) + when I > tuple_size(Tuple) -> + []; +print_length_tuple1(Tuple, I, D, T, RF, Enc, Str) when D =:= 1; T =:= 0-> + More = fun(T1, Dd) -> ?FUNCTION_NAME(Tuple, I, D+Dd, T1, RF, Enc, Str) end, + {dots, 3, 3, More}; +print_length_tuple1(Tuple, I, D, T, RF, Enc, Str) -> + E = element(I, Tuple), + T1 = tsub(T, 1), + {_, Len1, _, _} = Elem1 = print_length(E, D - 1, T1, RF, Enc, Str), + T2 = tsub(T1, Len1), + [Elem1 | print_length_tuple1(Tuple, I + 1, D - 1, T2, RF, Enc, Str)]. + +print_length_record(Tuple, 1, _T, RF, RDefs, Enc, Str) -> + More = fun(T1, Dd) -> + ?FUNCTION_NAME(Tuple, 1+Dd, T1, RF, RDefs, Enc, Str) + end, + {"{...}", 5, 3, More}; +print_length_record(Tuple, D, T, RF, RDefs, Enc, Str) -> Name = [$# | write_atom(element(1, Tuple), Enc)], - NameL = length(Name), - Elements = tl(tuple_to_list(Tuple)), - L = print_length_fields(RDefs, D - 1, Elements, RF, Enc, Str), - {{record, [{Name,NameL} | L]}, list_length(L, NameL + 2)}. - -print_length_fields([], _D, [], _RF, _Enc, _Str) -> + NameL = string:length(Name), + T1 = tsub(T, NameL+2), + L = print_length_fields(RDefs, D - 1, T1, Tuple, 2, RF, Enc, Str), + {Len, Dots} = list_length(L, NameL + 2, 0), + {{record, [{Name,NameL} | L]}, Len, Dots, no_more}. + +print_length_fields([], _D, _T, Tuple, I, _RF, _Enc, _Str) + when I > tuple_size(Tuple) -> []; -print_length_fields(_, 1, _, _RF, _Enc, _Str) -> - {dots, 3}; -print_length_fields([Def | Defs], D, [E | Es], RF, Enc, Str) -> - [print_length_field(Def, D - 1, E, RF, Enc, Str) | - print_length_fields(Defs, D - 1, Es, RF, Enc, Str)]. - -print_length_field(Def, D, E, RF, Enc, Str) -> +print_length_fields(Term, D, T, Tuple, I, RF, Enc, Str) + when D =:= 1; T =:= 0 -> + More = fun(T1, Dd) -> + ?FUNCTION_NAME(Term, D+Dd, T1, Tuple, I, RF, Enc, Str) + end, + {dots, 3, 3, More}; +print_length_fields([Def | Defs], D, T, Tuple, I, RF, Enc, Str) -> + E = element(I, Tuple), + T1 = tsub(T, 1), + Field1 = print_length_field(Def, D - 1, T1, E, RF, Enc, Str), + {_, Len1, _, _} = Field1, + T2 = tsub(T1, Len1), + [Field1 | + print_length_fields(Defs, D - 1, T2, Tuple, I + 1, RF, Enc, Str)]. + +print_length_field(Def, D, T, E, RF, Enc, Str) -> Name = write_atom(Def, Enc), - {S, L} = print_length(E, D, RF, Enc, Str), - NameL = length(Name) + 3, - {{field, Name, NameL, {S, L}}, NameL + L}. + NameL = string:length(Name) + 3, + {_, Len, Dots, _} = + Field = print_length(E, D, tsub(T, NameL), RF, Enc, Str), + {{field, Name, NameL, Field}, NameL + Len, Dots, no_more}. -print_length_list(List, D, RF, Enc, Str) -> - L = print_length_list1(List, D, RF, Enc, Str), - {{list, L}, list_length(L, 2)}. +print_length_list(List, D, T, RF, Enc, Str) -> + L = print_length_list1(List, D, tsub(T, 2), RF, Enc, Str), + {Len, Dots} = list_length(L, 2, 0), + {{list, L}, Len, Dots, no_more}. -print_length_list1([], _D, _RF, _Enc, _Str) -> +print_length_list1([], _D, _T, _RF, _Enc, _Str) -> []; -print_length_list1(_, 1, _RF, _Enc, _Str) -> - {dots, 3}; -print_length_list1([E | Es], D, RF, Enc, Str) -> - [print_length(E, D - 1, RF, Enc, Str) | - print_length_list1(Es, D - 1, RF, Enc, Str)]; -print_length_list1(E, D, RF, Enc, Str) -> - print_length(E, D - 1, RF, Enc, Str). - -list_length([], Acc) -> - Acc; -list_length([{_, Len} | Es], Acc) -> - list_length_tail(Es, Acc + Len); -list_length({_, Len}, Acc) -> - Acc + Len. - -list_length_tail([], Acc) -> - Acc; -list_length_tail([{_,Len} | Es], Acc) -> - list_length_tail(Es, Acc + 1 + Len); -list_length_tail({_, Len}, Acc) -> - Acc + 1 + Len. +print_length_list1(Term, D, T, RF, Enc, Str) when D =:= 1; T =:= 0-> + More = fun(T1, Dd) -> ?FUNCTION_NAME(Term, D+Dd, T1, RF, Enc, Str) end, + {dots, 3, 3, More}; +print_length_list1([E | Es], D, T, RF, Enc, Str) -> + {_, Len1, _, _} = Elem1 = print_length(E, D - 1, tsub(T, 1), RF, Enc, Str), + [Elem1 | print_length_list1(Es, D - 1, tsub(T, Len1 + 1), RF, Enc, Str)]; +print_length_list1(E, D, T, RF, Enc, Str) -> + print_length(E, D - 1, T, RF, Enc, Str). + +list_length([], Acc, DotsAcc) -> + {Acc, DotsAcc}; +list_length([{_, Len, Dots, _} | Es], Acc, DotsAcc) -> + list_length_tail(Es, Acc + Len, DotsAcc + Dots); +list_length({_, Len, Dots, _}, Acc, DotsAcc) -> + {Acc + Len, DotsAcc + Dots}. + +list_length_tail([], Acc, DotsAcc) -> + {Acc, DotsAcc}; +list_length_tail([{_, Len, Dots, _} | Es], Acc, DotsAcc) -> + list_length_tail(Es, Acc + 1 + Len, DotsAcc + Dots); +list_length_tail({_, Len, Dots, _}, Acc, DotsAcc) -> + {Acc + 1 + Len, DotsAcc + Dots}. %% ?CHARS printable characters has depth 1. -define(CHARS, 4). %% only flat lists are "printable" -printable_list(_L, 1, _Enc) -> +printable_list(_L, 1, _T, _Enc) -> false; -printable_list(L, _D, latin1) -> +printable_list(L, _D, T, latin1) when T < 0 -> io_lib:printable_latin1_list(L); -printable_list(L, _D, _Uni) -> +printable_list(L, _D, T, Enc) when T >= 0 -> + case slice(L, tsub(T, 2)) of + {prefix, ""} -> + false; + {prefix, Prefix} when Enc =:= latin1 -> + io_lib:printable_latin1_list(Prefix) andalso {true, Prefix}; + {prefix, Prefix} -> + %% Probably an overestimation. + io_lib:printable_list(Prefix) andalso {true, Prefix}; + all when Enc =:= latin1 -> + io_lib:printable_latin1_list(L); + all -> + io_lib:printable_list(L) + end; +printable_list(L, _D, T, _Uni) when T < 0-> io_lib:printable_list(L). -printable_bin(Bin, D, Enc) when D >= 0, ?CHARS * D =< byte_size(Bin) -> - printable_bin(Bin, erlang:min(?CHARS * D, byte_size(Bin)), D, Enc); -printable_bin(Bin, D, Enc) -> - printable_bin(Bin, byte_size(Bin), D, Enc). +slice(L, N) -> + case string:length(L) =< N of + true -> + all; + false -> + {prefix, string:slice(L, 0, N)} + end. + +printable_bin0(Bin, D, T, Enc) -> + Len = case D >= 0 of + true -> + %% Use byte_size() also if Enc =/= latin1. + DChars = erlang:min(?CHARS * D, byte_size(Bin)), + case T >= 0 of + true -> + erlang:min(T, DChars); + false -> + DChars + end; + false when T < 0 -> + byte_size(Bin); + false when T >= 0 -> % cannot happen + T + end, + printable_bin(Bin, Len, D, Enc). printable_bin(Bin, Len, D, latin1) -> N = erlang:min(20, Len), @@ -689,28 +871,70 @@ write_string(S, latin1) -> write_string(S, _Uni) -> io_lib:write_string(S, $"). %" +expand({_, _, _Dots=0, no_more} = If, _T, _Dd) -> If; +%% expand({{list,L}, _Len, _, no_more}, T, Dd) -> +%% {NL, NLen, NDots} = expand_list(L, T, Dd, 2), +%% {{list,NL}, NLen, NDots, no_more}; +expand({{tuple,IsTagged,L}, _Len, _, no_more}, T, Dd) -> + {NL, NLen, NDots} = expand_list(L, T, Dd, 2), + {{tuple,IsTagged,NL}, NLen, NDots, no_more}; +expand({{map, Pairs}, _Len, _, no_more}, T, Dd) -> + {NPairs, NLen, NDots} = expand_list(Pairs, T, Dd, 3), + {{map, NPairs}, NLen, NDots, no_more}; +expand({{map_pair, K, V}, _Len, _, no_more}, T, Dd) -> + {_, KL, KD, _} = P1 = expand(K, tsub(T, 1), Dd), + KL1 = KL + 4, + {_, VL, VD, _} = P2 = expand(V, tsub(T, KL1), Dd), + {{map_pair, P1, P2}, KL1 + VL, KD + VD, no_more}; +expand({{record, [{Name,NameL} | L]}, _Len, _, no_more}, T, Dd) -> + {NL, NLen, NDots} = expand_list(L, T, Dd, NameL + 2), + {{record, [{Name,NameL} | NL]}, NLen, NDots, no_more}; +expand({{field, Name, NameL, Field}, _Len, _, no_more}, T, Dd) -> + F = {_S, L, Dots, _} = expand(Field, tsub(T, NameL), Dd), + {{field, Name, NameL, F}, NameL + L, Dots, no_more}; +expand({_, _, _, More}, T, Dd) -> + More(T, Dd). + +expand_list(Ifs, T, Dd, L0) -> + L = expand_list(Ifs, tsub(T, L0), Dd), + {Len, Dots} = list_length(L, L0, 0), + {L, Len, Dots}. + +expand_list([], _T, _Dd) -> + []; +expand_list([If | Ifs], T, Dd) -> + {_, Len1, _, _} = Elem1 = expand(If, tsub(T, 1), Dd), + [Elem1 | expand_list(Ifs, tsub(T, Len1 + 1), Dd)]; +expand_list({_, _, _, More}, T, Dd) -> + More(T, Dd). + +%% Make sure T does not change sign. +tsub(T, _) when T < 0 -> T; +tsub(T, E) when T >= E -> T - E; +tsub(_, _) -> 0. + %% Throw 'no_good' if the indentation exceeds half the line length %% unless there is room for M characters on the line. -cind({_S, Len}, Col, Ll, M, Ind, LD, W) when Len < Ll - Col - LD, - Len + W + LD =< M -> +cind({_S, Len, _, _}, Col, Ll, M, Ind, LD, W) when Len < Ll - Col - LD, + Len + W + LD =< M -> Ind; -cind({{list,L}, _Len}, Col, Ll, M, Ind, LD, W) -> +cind({{list,L}, _Len, _, _}, Col, Ll, M, Ind, LD, W) -> cind_list(L, Col + 1, Ll, M, Ind, LD, W + 1); -cind({{tuple,true,L}, _Len}, Col, Ll, M, Ind, LD, W) -> +cind({{tuple,true,L}, _Len, _ ,_}, Col, Ll, M, Ind, LD, W) -> cind_tag_tuple(L, Col, Ll, M, Ind, LD, W + 1); -cind({{tuple,false,L}, _Len}, Col, Ll, M, Ind, LD, W) -> +cind({{tuple,false,L}, _Len, _, _}, Col, Ll, M, Ind, LD, W) -> cind_list(L, Col + 1, Ll, M, Ind, LD, W + 1); -cind({{map,Pairs},_Len}, Col, Ll, M, Ind, LD, W) -> +cind({{map,Pairs}, _Len, _, _}, Col, Ll, M, Ind, LD, W) -> cind_map(Pairs, Col + 2, Ll, M, Ind, LD, W + 2); -cind({{record,[{_Name,NLen} | L]}, _Len}, Col, Ll, M, Ind, LD, W) -> +cind({{record,[{_Name,NLen} | L]}, _Len, _, _}, Col, Ll, M, Ind, LD, W) -> cind_record(L, NLen, Col, Ll, M, Ind, LD, W + NLen + 1); -cind({{bin,_S}, _Len}, _Col, _Ll, _M, Ind, _LD, _W) -> +cind({{bin,_S}, _Len, _, _}, _Col, _Ll, _M, Ind, _LD, _W) -> Ind; -cind({_S, _Len}, _Col, _Ll, _M, Ind, _LD, _W) -> +cind({_S,_Len,_,_}, _Col, _Ll, _M, Ind, _LD, _W) -> Ind. -cind_tag_tuple([{_Tag,Tlen} | L], Col, Ll, M, Ind, LD, W) -> +cind_tag_tuple([{_Tag,Tlen,_,_} | L], Col, Ll, M, Ind, LD, W) -> TagInd = Tlen + 2, Tcol = Col + TagInd, if @@ -732,9 +956,9 @@ cind_map([P | Ps], Col, Ll, M, Ind, LD, W) -> PW = cind_pair(P, Col, Ll, M, Ind, last_depth(Ps, LD), W), cind_pairs_tail(Ps, Col, Col + PW, Ll, M, Ind, LD, W + PW); cind_map(_, _Col, _Ll, _M, Ind, _LD, _W) -> - Ind. + Ind. % cannot happen -cind_pairs_tail([{_, Len}=P | Ps], Col0, Col, Ll, M, Ind, LD, W) -> +cind_pairs_tail([{_, Len, _, _} = P | Ps], Col0, Col, Ll, M, Ind, LD, W) -> LD1 = last_depth(Ps, LD), ELen = 1 + Len, if @@ -748,7 +972,7 @@ cind_pairs_tail([{_, Len}=P | Ps], Col0, Col, Ll, M, Ind, LD, W) -> cind_pairs_tail(_, _Col0, _Col, _Ll, _M, Ind, _LD, _W) -> Ind. -cind_pair({{map_pair, _Key, _Value}, Len}=Pair, Col, Ll, M, _Ind, LD, W) +cind_pair({{map_pair, _Key, _Value}, Len, _, _}=Pair, Col, Ll, M, _Ind, LD, W) when Len < Ll - Col - LD, Len + W + LD =< M -> if ?ATM_PAIR(Pair) -> @@ -756,7 +980,7 @@ cind_pair({{map_pair, _Key, _Value}, Len}=Pair, Col, Ll, M, _Ind, LD, W) true -> Ll end; -cind_pair({{map_pair, K, V}, _Len}, Col0, Ll, M, Ind, LD, W0) -> +cind_pair({{map_pair, K, V}, _Len, _, _}, Col0, Ll, M, Ind, LD, W0) -> cind(K, Col0, Ll, M, Ind, LD, W0), I = map_value_indent(Ind), cind(V, Col0 + I, Ll, M, Ind, LD, 0), @@ -778,7 +1002,7 @@ cind_record([F | Fs], Nlen, Col0, Ll, M, Ind, LD, W0) -> cind_record(_, _Nlen, _Col, _Ll, _M, Ind, _LD, _W) -> Ind. -cind_fields_tail([{_, Len}=F | Fs], Col0, Col, Ll, M, Ind, LD, W) -> +cind_fields_tail([{_, Len, _, _} = F | Fs], Col0, Col, Ll, M, Ind, LD, W) -> LD1 = last_depth(Fs, LD), ELen = 1 + Len, if @@ -792,7 +1016,7 @@ cind_fields_tail([{_, Len}=F | Fs], Col0, Col, Ll, M, Ind, LD, W) -> cind_fields_tail(_, _Col0, _Col, _Ll, _M, Ind, _LD, _W) -> Ind. -cind_field({{field, _N, _NL, _F}, Len}=Fl, Col, Ll, M, _Ind, LD, W) +cind_field({{field, _N, _NL, _F}, Len, _, _}=Fl, Col, Ll, M, _Ind, LD, W) when Len < Ll - Col - LD, Len + W + LD =< M -> if ?ATM_FLD(Fl) -> @@ -800,7 +1024,7 @@ cind_field({{field, _N, _NL, _F}, Len}=Fl, Col, Ll, M, _Ind, LD, W) true -> Ll end; -cind_field({{field, _Name, NameL, F}, _Len}, Col0, Ll, M, Ind, LD, W0) -> +cind_field({{field, _Name, NameL, F},_Len,_,_}, Col0, Ll, M, Ind, LD, W0) -> {Col, W} = cind_rec(NameL, Col0, Ll, M, Ind, W0 + NameL), cind(F, Col, Ll, M, Ind, LD, W), Ll. @@ -823,7 +1047,7 @@ cind_rec(RInd, Col0, Ll, M, Ind, W0) -> throw(no_good) end. -cind_list({dots, _}, _Col0, _Ll, _M, Ind, _LD, _W) -> +cind_list({dots, _, _, _}, _Col0, _Ll, _M, Ind, _LD, _W) -> Ind; cind_list([E | Es], Col0, Ll, M, Ind, LD, W) -> WE = cind_element(E, Col0, Ll, M, Ind, last_depth(Es, LD), W), @@ -831,7 +1055,7 @@ cind_list([E | Es], Col0, Ll, M, Ind, LD, W) -> cind_tail([], _Col0, _Col, _Ll, _M, Ind, _LD, _W) -> Ind; -cind_tail([{_, Len}=E | Es], Col0, Col, Ll, M, Ind, LD, W) -> +cind_tail([{_, Len, _, _} = E | Es], Col0, Col, Ll, M, Ind, LD, W) -> LD1 = last_depth(Es, LD), ELen = 1 + Len, if @@ -842,9 +1066,9 @@ cind_tail([{_, Len}=E | Es], Col0, Col, Ll, M, Ind, LD, W) -> WE = cind_element(E, Col0, Ll, M, Ind, LD1, 0), cind_tail(Es, Col0, Col0 + WE, Ll, M, Ind, LD, WE) end; -cind_tail({dots, _}, _Col0, _Col, _Ll, _M, Ind, _LD, _W) -> +cind_tail({dots, _, _, _}, _Col0, _Col, _Ll, _M, Ind, _LD, _W) -> Ind; -cind_tail({_, Len}=E, _Col0, Col, Ll, M, Ind, LD, W) +cind_tail({_, Len, _, _}=E, _Col0, Col, Ll, M, Ind, LD, W) when Len + 1 < Ll - Col - (LD + 1), Len + 1 + W + (LD + 1) =< M, ?ATM(E) -> @@ -852,7 +1076,7 @@ cind_tail({_, Len}=E, _Col0, Col, Ll, M, Ind, LD, W) cind_tail(E, _Col0, Col, Ll, M, Ind, LD, _W) -> cind(E, Col, Ll, M, Ind, LD + 1, 0). -cind_element({_, Len}=E, Col, Ll, M, _Ind, LD, W) +cind_element({_, Len, _, _}=E, Col, Ll, M, _Ind, LD, W) when Len < Ll - Col - LD, Len + W + LD =< M, ?ATM(E) -> Len; cind_element(E, Col, Ll, M, Ind, LD, W) -> diff --git a/lib/stdlib/src/lists.erl b/lib/stdlib/src/lists.erl index af9d63ddd6..06c90c0280 100644 --- a/lib/stdlib/src/lists.erl +++ b/lib/stdlib/src/lists.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -38,8 +38,8 @@ -export([all/2,any/2,map/2,flatmap/2,foldl/3,foldr/3,filter/2, partition/2,zf/2,filtermap/2, - mapfoldl/3,mapfoldr/3,foreach/2,takewhile/2,dropwhile/2,splitwith/2, - split/2, + mapfoldl/3,mapfoldr/3,foreach/2,takewhile/2,dropwhile/2, + search/2, splitwith/2,split/2, join/2]). %%% BIFs @@ -1399,6 +1399,19 @@ dropwhile(Pred, [Hd|Tail]=Rest) -> end; dropwhile(Pred, []) when is_function(Pred, 1) -> []. +-spec search(Pred, List) -> {value, Value} | false when + Pred :: fun((T) -> boolean()), + List :: [T], + Value :: T. + +search(Pred, [Hd|Tail]) -> + case Pred(Hd) of + true -> {value, Hd}; + false -> search(Pred, Tail) + end; +search(Pred, []) when is_function(Pred, 1) -> + false. + -spec splitwith(Pred, List) -> {List1, List2} when Pred :: fun((T) -> boolean()), List :: [T], diff --git a/lib/stdlib/src/ms_transform.erl b/lib/stdlib/src/ms_transform.erl index 6616e957c0..ec8cfd56c2 100644 --- a/lib/stdlib/src/ms_transform.erl +++ b/lib/stdlib/src/ms_transform.erl @@ -944,6 +944,7 @@ real_guard_function(node,1) -> true; real_guard_function(round,1) -> true; real_guard_function(size,1) -> true; real_guard_function(map_size,1) -> true; +real_guard_function(map_get,2) -> true; real_guard_function(tl,1) -> true; real_guard_function(trunc,1) -> true; real_guard_function(self,0) -> true; @@ -1115,5 +1116,3 @@ normalise_list([H|T]) -> [normalise(H)|normalise_list(T)]; normalise_list([]) -> []. - - diff --git a/lib/stdlib/src/ordsets.erl b/lib/stdlib/src/ordsets.erl index 569407f5ef..939e147ad8 100644 --- a/lib/stdlib/src/ordsets.erl +++ b/lib/stdlib/src/ordsets.erl @@ -19,7 +19,7 @@ -module(ordsets). --export([new/0,is_set/1,size/1,to_list/1,from_list/1]). +-export([new/0,is_set/1,size/1,is_empty/1,to_list/1,from_list/1]). -export([is_element/2,add_element/2,del_element/2]). -export([union/2,union/1,intersection/2,intersection/1]). -export([is_disjoint/2]). @@ -60,6 +60,13 @@ is_set([], _) -> true. size(S) -> length(S). +%% is_empty(OrdSet) -> boolean(). +%% Return 'true' if OrdSet is an empty set, otherwise 'false'. +-spec is_empty(Ordset) -> boolean() when + Ordset :: ordset(_). + +is_empty(S) -> S=:=[]. + %% to_list(OrdSet) -> [Elem]. %% Return the elements in OrdSet as a list. diff --git a/lib/stdlib/src/otp_internal.erl b/lib/stdlib/src/otp_internal.erl index 122b476ddb..a17addcc42 100644 --- a/lib/stdlib/src/otp_internal.erl +++ b/lib/stdlib/src/otp_internal.erl @@ -604,57 +604,14 @@ obsolete_1(filename, find_src, 1) -> obsolete_1(filename, find_src, 2) -> {deprecated, "deprecated; use filelib:find_source/3 instead"}; +obsolete_1(erlang, get_stacktrace, 0) -> + {deprecated, "deprecated; use the new try/catch syntax for retrieving the stack backtrace"}; + %% Removed in OTP 20. obsolete_1(erlang, hash, 2) -> {removed, {erlang, phash2, 2}, "20.0"}; -%% Added in OTP-21 -obsolete_1(string, len, 1) -> - {deprecated, "deprecated; use string:length/3 instead"}; -obsolete_1(string, concat, 2) -> - {deprecated, "deprecated; use [Str1,Str2] instead"}; -obsolete_1(string, str, 2) -> - {deprecated, "deprecated; use string:find/2 instead"}; -obsolete_1(string, rstr, 2) -> - {deprecated, "deprecated; use string:find/3 instead"}; -obsolete_1(string, chr, 2) -> - {deprecated, "deprecated; use string:find/2 instead"}; -obsolete_1(string, rchr, 2) -> - {deprecated, "deprecated; use string:find/3 instead"}; -obsolete_1(string, span, 2) -> - {deprecated, "deprecated; use string:take/2 instead"}; -obsolete_1(string, cspan, 2) -> - {deprecated, "deprecated; use string:take/3 instead"}; -obsolete_1(string, substr, _) -> - {deprecated, "deprecated; use string:slice/3 instead"}; -obsolete_1(string, tokens, 2) -> - {deprecated, "deprecated; use string:lexemes/2 instead"}; -obsolete_1(string, chars, _) -> - {deprecated, "deprecated; use lists:duplicate/2 instead"}; -obsolete_1(string, copies, _) -> - {deprecated, "deprecated; use lists:duplicate/2 instead"}; -obsolete_1(string, words, _) -> - {deprecated, "deprecated; use string:lexemes/2 instead"}; -obsolete_1(string, strip, _) -> - {deprecated, "deprecated; use string:trim/3 instead"}; -obsolete_1(string, sub_word, _) -> - {deprecated, "deprecated; use string:nth_lexeme/3 instead"}; -obsolete_1(string, sub_string, _) -> - {deprecated, "deprecated; use string:slice/3 instead"}; -obsolete_1(string, left, _) -> - {deprecated, "deprecated; use string:pad/3 instead"}; -obsolete_1(string, right, _) -> - {deprecated, "deprecated; use string:pad/3 instead"}; -obsolete_1(string, centre, _) -> - {deprecated, "deprecated; use string:pad/3 instead"}; -obsolete_1(string, join, _) -> - {deprecated, "deprecated; use lists:join/2 instead"}; -obsolete_1(string, to_upper, _) -> - {deprecated, "deprecated; use string:uppercase/1 or string:titlecase/1 instead"}; -obsolete_1(string, to_lower, _) -> - {deprecated, "deprecated; use string:lowercase/1 or string:casefold/1 instead"}; - %% not obsolete obsolete_1(_, _, _) -> diff --git a/lib/stdlib/src/proc_lib.erl b/lib/stdlib/src/proc_lib.erl index 1991585c13..8d01840313 100644 --- a/lib/stdlib/src/proc_lib.erl +++ b/lib/stdlib/src/proc_lib.erl @@ -30,7 +30,7 @@ start/3, start/4, start/5, start_link/3, start_link/4, start_link/5, hibernate/3, init_ack/1, init_ack/2, - init_p/3,init_p/5,format/1,format/2,format/3, + init_p/3,init_p/5,format/1,format/2,format/3,report_cb/1, initial_call/1, translate_initial_call/1, stop/1, stop/3]). @@ -40,6 +40,8 @@ -export_type([spawn_option/0]). +-include("logger.hrl"). + %%----------------------------------------------------------------------------- -type priority_level() :: 'high' | 'low' | 'max' | 'normal'. @@ -503,10 +505,13 @@ crash_report(exit, normal, _, _) -> ok; crash_report(exit, shutdown, _, _) -> ok; crash_report(exit, {shutdown,_}, _, _) -> ok; crash_report(Class, Reason, StartF, Stacktrace) -> - OwnReport = my_info(Class, Reason, StartF, Stacktrace), - LinkReport = linked_info(self()), - Rep = [OwnReport,LinkReport], - error_logger:error_report(crash_report, Rep). + ?LOG_ERROR(#{label=>{proc_lib,crash}, + report=>[my_info(Class, Reason, StartF, Stacktrace), + linked_info(self())]}, + #{domain=>[beam,erlang,otp,sasl], + report_cb=>fun proc_lib:report_cb/1, + logger_formatter=>#{title=>"CRASH REPORT"}, + error_logger=>#{tag=>error_report,type=>crash_report}}). my_info(Class, Reason, [], Stacktrace) -> my_info_1(Class, Reason, Stacktrace); @@ -548,10 +553,10 @@ get_ancestors(Pid) -> %% assumed that all report handlers call proc_lib:format(). get_messages(Pid) -> Messages = get_process_messages(Pid), - {messages, error_logger:limit_term(Messages)}. + {messages, logger:limit_term(Messages)}. get_process_messages(Pid) -> - Depth = error_logger:get_format_depth(), + Depth = logger:get_format_depth(), case Pid =/= self() orelse Depth =:= unlimited of true -> {messages, Messages} = get_process_info(Pid, messages), @@ -581,7 +586,7 @@ get_cleaned_dictionary(Pid) -> cleaned_dict(Dict) -> CleanDict = clean_dict(Dict), - error_logger:limit_term(CleanDict). + logger:limit_term(CleanDict). clean_dict([{'$ancestors',_}|Dict]) -> clean_dict(Dict); @@ -742,9 +747,18 @@ check({badrpc,Error}) -> Error; check(Res) -> Res. %%% ----------------------------------------------------------- -%%% Format (and write) a generated crash info structure. +%%% Format a generated crash info structure. %%% ----------------------------------------------------------- +-spec report_cb(CrashReport) -> {Format,Args} when + CrashReport :: #{label=>{proc_lib,crash},report=>[term()]}, + Format :: io:format(), + Args :: [term()]. +report_cb(#{label:={proc_lib,crash}, + report:=CrashReport}) -> + Depth = logger:get_format_depth(), + get_format_and_args(CrashReport, utf8, Depth). + -spec format(CrashReport) -> string() when CrashReport :: [term()]. format(CrashReport) -> @@ -762,61 +776,74 @@ format(CrashReport, Encoding) -> Encoding :: latin1 | unicode | utf8, Depth :: unlimited | pos_integer(). -format([OwnReport,LinkReport], Encoding, Depth) -> +format(CrashReport, Encoding, Depth) -> + {F,A} = get_format_and_args(CrashReport, Encoding, Depth), + lists:flatten(io_lib:format(F,A)). + +get_format_and_args([OwnReport,LinkReport], Encoding, Depth) -> Extra = {Encoding,Depth}, MyIndent = " ", - OwnFormat = format_report(OwnReport, MyIndent, Extra), - LinkFormat = format_link_report(LinkReport, MyIndent, Extra), - Str = io_lib:format(" crasher:~n~ts neighbours:~n~ts", - [OwnFormat, LinkFormat]), - lists:flatten(Str). + {OwnFormat,OwnArgs} = format_report(OwnReport, MyIndent, Extra, [], []), + {LinkFormat,LinkArgs} = format_link_report(LinkReport, MyIndent, Extra, [], []), + {" crasher:~n"++OwnFormat++" neighbours:~n"++LinkFormat,OwnArgs++LinkArgs}. -format_link_report([Link|Reps], Indent, Extra) -> +format_link_report([], _Indent, _Extra, Format, Args) -> + {lists:flatten(lists:reverse(Format)),lists:append(lists:reverse(Args))}; +format_link_report([Link|Reps], Indent, Extra, Format, Args) -> Rep = case Link of {neighbour,Rep0} -> Rep0; _ -> Link end, LinkIndent = [" ",Indent], - [Indent,"neighbour:\n",format_report(Rep, LinkIndent, Extra)| - format_link_report(Reps, Indent, Extra)]; -format_link_report(Rep, Indent, Extra) -> - format_report(Rep, Indent, Extra). - -format_report(Rep, Indent, Extra) when is_list(Rep) -> - format_rep(Rep, Indent, Extra); -format_report(Rep, Indent, {Enc,unlimited}) -> - io_lib:format("~s~"++modifier(Enc)++"p~n", [Indent, Rep]); -format_report(Rep, Indent, {Enc,Depth}) -> - io_lib:format("~s~"++modifier(Enc)++"P~n", [Indent, Rep, Depth]). - -format_rep([{initial_call,InitialCall}|Rep], Indent, Extra) -> - [format_mfa(Indent, InitialCall, Extra)|format_rep(Rep, Indent, Extra)]; -format_rep([{error_info,{Class,Reason,StackTrace}}|Rep], Indent, Extra) -> - [format_exception(Class, Reason, StackTrace, Extra)| - format_rep(Rep, Indent, Extra)]; -format_rep([{Tag,Data}|Rep], Indent, Extra) -> - [format_tag(Indent, Tag, Data, Extra)|format_rep(Rep, Indent, Extra)]; -format_rep(_, _, _Extra) -> - []. - -format_exception(Class, Reason, StackTrace, {Enc,_}=Extra) -> - PF = pp_fun(Extra), - StackFun = fun(M, _F, _A) -> (M =:= erl_eval) or (M =:= ?MODULE) end, - %% EI = " exception: ", - EI = " ", - [EI, lib:format_exception(1+length(EI), Class, Reason, - StackTrace, StackFun, PF, Enc), "\n"]. + {LinkFormat,LinkArgs} = format_report(Rep, LinkIndent, Extra, [], []), + F = "~sneighbour:\n"++LinkFormat, + A = [Indent|LinkArgs], + format_link_report(Reps, Indent, Extra, [F|Format], [A|Args]); +format_link_report(Rep, Indent, Extra, Format, Args) -> + {F,A} = format_report(Rep, Indent, Extra, [], []), + format_link_report([], Indent, Extra, [F|Format],[A|Args]). + +format_report([], _Indent, _Extra, Format, Args) -> + {lists:flatten(lists:reverse(Format)),lists:append(lists:reverse(Args))}; +format_report([Rep|Reps], Indent, Extra, Format, Args) -> + {F,A} = format_rep(Rep, Indent, Extra), + format_report(Reps, Indent, Extra, [F|Format], [A|Args]); +format_report(Rep, Indent, {Enc,unlimited}=Extra, Format, Args) -> + {F,A} = {"~s~"++modifier(Enc)++"p~n", [Indent, Rep]}, + format_report([], Indent, Extra, [F|Format], [A|Args]); +format_report(Rep, Indent, {Enc,Depth}=Extra, Format, Args) -> + {F,A} = {"~s~"++modifier(Enc)++"P~n", [Indent, Rep, Depth]}, + format_report([], Indent, Extra, [F|Format], [A|Args]). + +format_rep({initial_call,InitialCall}, Indent, Extra) -> + format_mfa(Indent, InitialCall, Extra); +format_rep({error_info,{Class,Reason,StackTrace}}, _Indent, Extra) -> + {lists:flatten(format_exception(Class, Reason, StackTrace, Extra)),[]}; +format_rep({Tag,Data}, Indent, Extra) -> + format_tag(Indent, Tag, Data, Extra). format_mfa(Indent, {M,F,Args}=StartF, {Enc,_}=Extra) -> try A = length(Args), - [Indent,"initial call: ",atom_to_list(M),$:,to_string(F, Enc),$/, - integer_to_list(A),"\n"] + {lists:flatten([Indent,"initial call: ",atom_to_list(M), + $:,to_string(F, Enc),$/,integer_to_list(A),"\n"]),[]} catch error:_ -> format_tag(Indent, initial_call, StartF, Extra) end. +format_tag(Indent, Tag, Data, {Enc,Depth}) -> + {P,Tl} = p(Enc, Depth), + {"~s~p: ~80.18" ++ P ++ "\n", [Indent, Tag, Data|Tl]}. + +format_exception(Class, Reason, StackTrace, {Enc,_}=Extra) -> + PF = pp_fun(Extra), + StackFun = fun(M, _F, _A) -> (M =:= erl_eval) or (M =:= ?MODULE) end, + %% EI = " exception: ", + EI = " ", + [EI, lib:format_exception(1+length(EI), Class, Reason, + StackTrace, StackFun, PF, Enc), "\n"]. + to_string(A, latin1) -> io_lib:write_atom_as_latin1(A); to_string(A, _) -> @@ -828,10 +855,6 @@ pp_fun({Enc,Depth}) -> io_lib:format("~." ++ integer_to_list(I) ++ P, [Term|Tl]) end. -format_tag(Indent, Tag, Data, {Enc,Depth}) -> - {P,Tl} = p(Enc, Depth), - io_lib:format("~s~p: ~80.18" ++ P ++ "\n", [Indent, Tag, Data|Tl]). - p(Encoding, Depth) -> {Letter, Tl} = case Depth of unlimited -> {"p", []}; diff --git a/lib/stdlib/src/sets.erl b/lib/stdlib/src/sets.erl index c65a13b22e..ac0fc80526 100644 --- a/lib/stdlib/src/sets.erl +++ b/lib/stdlib/src/sets.erl @@ -37,7 +37,7 @@ -module(sets). %% Standard interface. --export([new/0,is_set/1,size/1,to_list/1,from_list/1]). +-export([new/0,is_set/1,size/1,is_empty/1,to_list/1,from_list/1]). -export([is_element/2,add_element/2,del_element/2]). -export([union/2,union/1,intersection/2,intersection/1]). -export([is_disjoint/2]). @@ -96,6 +96,12 @@ is_set(_) -> false. Set :: set(). size(S) -> S#set.size. +%% is_empty(Set) -> boolean(). +%% Return 'true' if Set is an empty set, otherwise 'false'. +-spec is_empty(Set) -> boolean() when + Set :: set(). +is_empty(S) -> S#set.size=:=0. + %% to_list(Set) -> [Elem]. %% Return the elements in Set as a list. -spec to_list(Set) -> List when diff --git a/lib/stdlib/src/shell.erl b/lib/stdlib/src/shell.erl index e4153e7899..1be37672e7 100644 --- a/lib/stdlib/src/shell.erl +++ b/lib/stdlib/src/shell.erl @@ -1416,7 +1416,7 @@ pp(V, I, D, RT) -> true end, io_lib_pretty:print(V, ([{column, I}, {line_length, columns()}, - {depth, D}, {max_chars, ?CHAR_MAX}, + {depth, D}, {line_max_chars, ?CHAR_MAX}, {strings, Strings}, {record_print_fun, record_print_fun(RT)}] ++ enc())). diff --git a/lib/stdlib/src/stdlib.appup.src b/lib/stdlib/src/stdlib.appup.src index e4e3fb83e9..8d1cc09a8b 100644 --- a/lib/stdlib/src/stdlib.appup.src +++ b/lib/stdlib/src/stdlib.appup.src @@ -18,7 +18,9 @@ %% %CopyrightEnd% {"%VSN%", %% Up from - max one major revision back - [{<<"3\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-20.* + [{<<"3\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-20.* + {<<"3\\.5(\\.[0-9]+)*">>,[restart_new_emulator]}],% OTP-21.* %% Down to - max one major revision back - [{<<"3\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-20.* + [{<<"3\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-20.* + {<<"3\\.5(\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-20.* }. diff --git a/lib/stdlib/src/string.erl b/lib/stdlib/src/string.erl index 4e89819e41..0736374f21 100644 --- a/lib/stdlib/src/string.erl +++ b/lib/stdlib/src/string.erl @@ -88,16 +88,6 @@ %%% May be removed -export([list_to_float/1, list_to_integer/1]). --deprecated([{len,1},{concat,2}, - {str,2},{chr,2},{rchr,2},{rstr,2}, - {span,2},{cspan,2},{substr,'_'},{tokens,2}, - {chars,'_'}, - {copies,2},{words,'_'},{strip,'_'}, - {sub_word,'_'},{left,'_'},{right,'_'}, - {sub_string,'_'},{centre,'_'},{join,2}, - {to_upper,1}, {to_lower,1} - ]). - %% Uses bifs: string:list_to_float/1 and string:list_to_integer/1 -spec list_to_float(String) -> {Float, Rest} | {'error', Reason} when String :: string(), diff --git a/lib/stdlib/src/supervisor.erl b/lib/stdlib/src/supervisor.erl index e56415650f..eb46ac611a 100644 --- a/lib/stdlib/src/supervisor.erl +++ b/lib/stdlib/src/supervisor.erl @@ -35,6 +35,20 @@ %% For release_handler only -export([get_callback_module/1]). +-include("logger.hrl"). + +-define(report_error(Error, Reason, Child, SupName), + ?LOG_ERROR(#{label=>{supervisor,Error}, + report=>[{supervisor,SupName}, + {errorContext,Error}, + {reason,Reason}, + {offender,extract_child(Child)}]}, + #{domain=>[beam,erlang,otp,sasl], + report_cb=>fun logger:format_otp_report/1, + logger_formatter=>#{title=>"SUPERVISOR REPORT"}, + error_logger=>#{tag=>error_report, + type=>supervisor_report}})). + %%-------------------------------------------------------------------------- -export_type([sup_flags/0, child_spec/0, startchild_ret/0, strategy/0]). @@ -340,7 +354,7 @@ start_children(Children, SupName) -> {ok, Pid, _Extra} -> {update,Child#child{pid = Pid}}; {error, Reason} -> - report_error(start_error, Reason, Child, SupName), + ?report_error(start_error, Reason, Child, SupName), {abort,{failed_to_start_child,Id,Reason}} end end, @@ -565,8 +579,9 @@ handle_info({'EXIT', Pid, Reason}, State) -> end; handle_info(Msg, State) -> - error_logger:error_msg("Supervisor received unexpected message: ~tp~n", - [Msg]), + ?LOG_ERROR("Supervisor received unexpected message: ~tp~n",[Msg], + #{domain=>[beam,erlang,otp], + error_logger=>#{tag=>error}}), {noreply, State}. %% @@ -683,7 +698,7 @@ restart_child(Pid, Reason, State) -> end. do_restart(Reason, Child, State) when ?is_permanent(Child) -> - report_error(child_terminated, Reason, Child, State#state.name), + ?report_error(child_terminated, Reason, Child, State#state.name), restart(Child, State); do_restart(normal, Child, State) -> NState = del_child(Child, State), @@ -695,10 +710,10 @@ do_restart({shutdown, _Term}, Child, State) -> NState = del_child(Child, State), {ok, NState}; do_restart(Reason, Child, State) when ?is_transient(Child) -> - report_error(child_terminated, Reason, Child, State#state.name), + ?report_error(child_terminated, Reason, Child, State#state.name), restart(Child, State); do_restart(Reason, Child, State) when ?is_temporary(Child) -> - report_error(child_terminated, Reason, Child, State#state.name), + ?report_error(child_terminated, Reason, Child, State#state.name), NState = del_child(Child, State), {ok, NState}. @@ -718,7 +733,7 @@ restart(Child, State) -> Other end; {terminate, NState} -> - report_error(shutdown, reached_max_restart_intensity, + ?report_error(shutdown, reached_max_restart_intensity, Child, State#state.name), {shutdown, del_child(Child, NState)} end. @@ -745,7 +760,7 @@ restart(simple_one_for_one, Child, State0) -> NRestarts = State2#state.dynamic_restarts + 1, State3 = State2#state{dynamic_restarts = NRestarts}, NState = dyn_store(ROldPid, A, State3), - report_error(start_error, Error, Child, NState#state.name), + ?report_error(start_error, Error, Child, NState#state.name), {{try_again, ROldPid}, NState} end; restart(one_for_one, #child{id=Id} = Child, State) -> @@ -759,7 +774,7 @@ restart(one_for_one, #child{id=Id} = Child, State) -> {ok, NState}; {error, Reason} -> NState = set_pid(restarting(OldPid), Id, State), - report_error(start_error, Reason, Child, State#state.name), + ?report_error(start_error, Reason, Child, State#state.name), {{try_again,Id}, NState} end; restart(rest_for_one, #child{id=Id} = Child, #state{name=SupName} = State) -> @@ -820,7 +835,7 @@ do_terminate(Child, SupName) when is_pid(Child#child.pid) -> {error, normal} when not (?is_permanent(Child)) -> ok; {error, OtherReason} -> - report_error(shutdown_error, OtherReason, Child, SupName) + ?report_error(shutdown_error, OtherReason, Child, SupName) end, ok; do_terminate(_Child, _SupName) -> @@ -924,7 +939,7 @@ terminate_dynamic_children(State) -> end, %% Unroll stacked errors and report them dict:fold(fun(Reason, Ls, _) -> - report_error(shutdown_error, Reason, + ?report_error(shutdown_error, Reason, Child#child{pid=Ls}, State#state.name) end, ok, EStack). @@ -1385,14 +1400,6 @@ inPeriod(Then, Now, Period) -> %%% ------------------------------------------------------ %%% Error and progress reporting. %%% ------------------------------------------------------ - -report_error(Error, Reason, Child, SupName) -> - ErrorMsg = [{supervisor, SupName}, - {errorContext, Error}, - {reason, Reason}, - {offender, extract_child(Child)}], - error_logger:error_report(supervisor_report, ErrorMsg). - extract_child(Child) when is_list(Child#child.pid) -> [{nb_children, length(Child#child.pid)}, {id, Child#child.id}, @@ -1409,9 +1416,13 @@ extract_child(Child) -> {child_type, Child#child.child_type}]. report_progress(Child, SupName) -> - Progress = [{supervisor, SupName}, - {started, extract_child(Child)}], - error_logger:info_report(progress, Progress). + ?LOG_INFO(#{label=>{supervisor,progress}, + report=>[{supervisor,SupName}, + {started,extract_child(Child)}]}, + #{domain=>[beam,erlang,otp,sasl], + report_cb=>fun logger:format_otp_report/1, + logger_formatter=>#{title=>"PROGRESS REPORT"}, + error_logger=>#{tag=>info_report,type=>progress}}). format_status(terminate, [_PDict, State]) -> State; diff --git a/lib/stdlib/src/supervisor_bridge.erl b/lib/stdlib/src/supervisor_bridge.erl index af1e046d30..39372935fa 100644 --- a/lib/stdlib/src/supervisor_bridge.erl +++ b/lib/stdlib/src/supervisor_bridge.erl @@ -21,6 +21,8 @@ -behaviour(gen_server). +-include("logger.hrl"). + %% External exports -export([start_link/2, start_link/3]). %% Internal exports @@ -129,13 +131,22 @@ terminate_pid(Reason, #state{mod = Mod, child_state = ChildState}) -> Mod:terminate(Reason, ChildState). report_progress(Pid, Mod, StartArgs, SupName) -> - Progress = [{supervisor, SupName}, - {started, [{pid, Pid}, {mfa, {Mod, init, [StartArgs]}}]}], - error_logger:info_report(progress, Progress). + ?LOG_INFO(#{label=>{supervisor,progress}, + report=>[{supervisor, SupName}, + {started, [{pid, Pid}, + {mfa, {Mod, init, [StartArgs]}}]}]}, + #{domain=>[beam,erlang,otp,sasl], + report_cb=>fun logger:format_otp_report/1, + logger_formatter=>#{title=>"PROGRESS REPORT"}, + error_logger=>#{tag=>info_report,type=>progress}}). report_error(Error, Reason, #state{name = Name, pid = Pid, mod = Mod}) -> - ErrorMsg = [{supervisor, Name}, - {errorContext, Error}, - {reason, Reason}, - {offender, [{pid, Pid}, {mod, Mod}]}], - error_logger:error_report(supervisor_report, ErrorMsg). + ?LOG_ERROR(#{label=>{supervisor,error}, + report=>[{supervisor, Name}, + {errorContext, Error}, + {reason, Reason}, + {offender, [{pid, Pid}, {mod, Mod}]}]}, + #{domain=>[beam,erlang,otp,sasl], + report_cb=>fun logger:format_otp_report/1, + logger_formatter=>#{title=>"SUPERVISOR REPORT"}, + error_logger=>#{tag=>error_report,type=>supervisor_report}}). diff --git a/lib/stdlib/src/sys.erl b/lib/stdlib/src/sys.erl index 0c578acf21..0064414d6f 100644 --- a/lib/stdlib/src/sys.erl +++ b/lib/stdlib/src/sys.erl @@ -44,6 +44,7 @@ -type system_event() :: {'in', Msg :: _} | {'in', Msg :: _, From :: _} | {'out', Msg :: _, To :: _} + | {'out', Msg :: _, To :: _, State :: _} | term(). -opaque dbg_opt() :: {'trace', 'true'} | {'log', @@ -56,7 +57,8 @@ MessagesIn :: non_neg_integer(), MessagesOut :: non_neg_integer()}} | {'log_to_file', file:io_device()} - | {Func :: dbg_fun(), FuncState :: term()}. + | {Func :: dbg_fun(), FuncState :: term()} + | {FuncId :: term(), Func :: dbg_fun(), FuncState :: term()}. -type dbg_fun() :: fun((FuncState :: _, Event :: system_event(), ProcState :: _) -> 'done' | (NewFuncState :: _)). @@ -267,33 +269,41 @@ no_debug(Name, Timeout) -> send_system_msg(Name, {debug, no_debug}, Timeout). -spec install(Name, FuncSpec) -> 'ok' when Name :: name(), - FuncSpec :: {Func, FuncState}, + FuncSpec :: {Func, FuncState} | {FuncId, Func, FuncState}, + FuncId :: term(), Func :: dbg_fun(), FuncState :: term(). install(Name, {Func, FuncState}) -> - send_system_msg(Name, {debug, {install, {Func, FuncState}}}). + send_system_msg(Name, {debug, {install, {Func, FuncState}}}); +install(Name, {FuncId, Func, FuncState}) -> + send_system_msg(Name, {debug, {install, {FuncId, Func, FuncState}}}). -spec install(Name, FuncSpec, Timeout) -> 'ok' when Name :: name(), - FuncSpec :: {Func, FuncState}, + FuncSpec :: {Func, FuncState} | {FuncId, Func, FuncState}, + FuncId :: term(), Func :: dbg_fun(), FuncState :: term(), Timeout :: timeout(). install(Name, {Func, FuncState}, Timeout) -> - send_system_msg(Name, {debug, {install, {Func, FuncState}}}, Timeout). + send_system_msg(Name, {debug, {install, {Func, FuncState}}}, Timeout); +install(Name, {FuncId, Func, FuncState}, Timeout) -> + send_system_msg(Name, {debug, {install, {FuncId, Func, FuncState}}}, Timeout). --spec remove(Name, Func) -> 'ok' when +-spec remove(Name, Func | FuncId) -> 'ok' when Name :: name(), - Func :: dbg_fun(). -remove(Name, Func) -> - send_system_msg(Name, {debug, {remove, Func}}). + Func :: dbg_fun(), + FuncId :: term(). +remove(Name, FuncOrFuncId) -> + send_system_msg(Name, {debug, {remove, FuncOrFuncId}}). --spec remove(Name, Func, Timeout) -> 'ok' when +-spec remove(Name, Func | FuncId, Timeout) -> 'ok' when Name :: name(), Func :: dbg_fun(), + FuncId :: term(), Timeout :: timeout(). -remove(Name, Func, Timeout) -> - send_system_msg(Name, {debug, {remove, Func}}, Timeout). +remove(Name, FuncOrFuncId, Timeout) -> + send_system_msg(Name, {debug, {remove, FuncOrFuncId}}, Timeout). %%----------------------------------------------------------------- %% All system messages sent are on the form {system, From, Msg} @@ -387,6 +397,13 @@ handle_debug([{log_to_file, Fd} | T], FormFunc, State, Event) -> handle_debug([{statistics, StatData} | T], FormFunc, State, Event) -> NStatData = stat(Event, StatData), [{statistics, NStatData} | handle_debug(T, FormFunc, State, Event)]; +handle_debug([{FuncId, {Func, FuncState}} | T], FormFunc, State, Event) -> + case catch Func(FuncState, Event, State) of + done -> handle_debug(T, FormFunc, State, Event); + {'EXIT', _} -> handle_debug(T, FormFunc, State, Event); + NFuncState -> + [{FuncId, {Func, NFuncState}} | handle_debug(T, FormFunc, State, Event)] + end; handle_debug([{Func, FuncState} | T], FormFunc, State, Event) -> case catch Func(FuncState, Event, State) of done -> handle_debug(T, FormFunc, State, Event); @@ -544,8 +561,10 @@ debug_cmd(no_debug, Debug) -> {ok, []}; debug_cmd({install, {Func, FuncState}}, Debug) -> {ok, install_debug(Func, FuncState, Debug)}; -debug_cmd({remove, Func}, Debug) -> - {ok, remove_debug(Func, Debug)}; +debug_cmd({install, {FuncId, Func, FuncState}}, Debug) -> + {ok, install_debug(FuncId, {Func, FuncState}, Debug)}; +debug_cmd({remove, FuncOrFuncId}, Debug) -> + {ok, remove_debug(FuncOrFuncId, Debug)}; debug_cmd(_Unknown, Debug) -> {unknown_debug, Debug}. @@ -573,6 +592,7 @@ get_stat(_) -> stat({in, _Msg}, {Time, Reds, In, Out}) -> {Time, Reds, In+1, Out}; stat({in, _Msg, _From}, {Time, Reds, In, Out}) -> {Time, Reds, In+1, Out}; stat({out, _Msg, _To}, {Time, Reds, In, Out}) -> {Time, Reds, In, Out+1}; +stat({out, _Msg, _To, _State}, {Time, Reds, In, Out}) -> {Time, Reds, In, Out+1}; stat(_, StatData) -> StatData. trim(N, LogData) -> @@ -582,9 +602,9 @@ trim(N, LogData) -> %% Debug structure manipulating functions %%----------------------------------------------------------------- install_debug(Item, Data, Debug) -> - case get_debug2(Item, Debug, undefined) of - undefined -> [{Item, Data} | Debug]; - _ -> Debug + case lists:keysearch(Item, 1, Debug) of + false -> [{Item, Data} | Debug]; + _ -> Debug end. remove_debug(Item, Debug) -> lists:keydelete(Item, 1, Debug). @@ -635,7 +655,8 @@ close_log_file(Debug) -> | {'log_to_file', FileName} | {'install', FuncSpec}, FileName :: file:name(), - FuncSpec :: {Func, FuncState}, + FuncSpec :: {Func, FuncState} | {FuncId, Func, FuncState}, + FuncId :: term(), Func :: dbg_fun(), FuncState :: term(). debug_options(Options) -> @@ -658,6 +679,8 @@ debug_options([{log_to_file, FileName} | T], Debug) -> end; debug_options([{install, {Func, FuncState}} | T], Debug) -> debug_options(T, install_debug(Func, FuncState, Debug)); +debug_options([{install, {FuncId, Func, FuncState}} | T], Debug) -> + debug_options(T, install_debug(FuncId, {Func, FuncState}, Debug)); debug_options([_ | T], Debug) -> debug_options(T, Debug); debug_options([], Debug) -> diff --git a/lib/stdlib/test/Makefile b/lib/stdlib/test/Makefile index 8490770f3d..ae2e3d0e2b 100644 --- a/lib/stdlib/test/Makefile +++ b/lib/stdlib/test/Makefile @@ -95,7 +95,8 @@ MODULES= \ random_unicode_list \ random_iolist \ error_logger_forwarder \ - maps_SUITE + maps_SUITE \ + zzz_SUITE ERL_FILES= $(MODULES:%=%.erl) diff --git a/lib/stdlib/test/calendar_SUITE.erl b/lib/stdlib/test/calendar_SUITE.erl index 20053dfe54..55118e251c 100644 --- a/lib/stdlib/test/calendar_SUITE.erl +++ b/lib/stdlib/test/calendar_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2016. All Rights Reserved. +%% Copyright Ericsson AB 1997-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -30,7 +30,8 @@ leap_years/1, last_day_of_the_month/1, local_time_to_universal_time_dst/1, - iso_week_number/1]). + iso_week_number/1, + system_time/1, rfc3339/1]). -define(START_YEAR, 1947). -define(END_YEAR, 2012). @@ -40,7 +41,8 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [gregorian_days, gregorian_seconds, day_of_the_week, day_of_the_week_calibrate, leap_years, - last_day_of_the_month, local_time_to_universal_time_dst, iso_week_number]. + last_day_of_the_month, local_time_to_universal_time_dst, + iso_week_number, system_time, rfc3339]. groups() -> []. @@ -157,10 +159,163 @@ local_time_to_universal_time_dst_x(Config) when is_list(Config) -> iso_week_number(Config) when is_list(Config) -> check_iso_week_number(). +system_time(Config) when is_list(Config) -> + EpochDate = {{1970,1,1}, {0,0,0}}, + Epoch = calendar:datetime_to_gregorian_seconds(EpochDate), + Y0 = {{0,1,1},{0,0,0}}, + + EpochDate = calendar:system_time_to_universal_time(0, second), + 0 = calendar:datetime_to_gregorian_seconds(Y0), + Y0 = calendar:system_time_to_universal_time(-Epoch, second), + + T = erlang:system_time(second), + UDate = calendar:system_time_to_universal_time(T, second), + LDate = erlang:universaltime_to_localtime(UDate), + LDate = calendar:system_time_to_local_time(T, second), + + ok. + +rfc3339(Config) when is_list(Config) -> + Ms = [{unit, millisecond}], + Mys = [{unit, microsecond}], + Ns = [{unit, nanosecond}], + S = [{unit, second}], + D = [{time_designator, $\s}], + Z = [{offset, "Z"}], + + "1985-04-12T23:20:50.52Z" = test_parse("1985-04-12T23:20:50.52Z", Ms), + "1985-04-12T23:20:50.52Z" = test_parse("1985-04-12t23:20:50.52z", Ms), + "1985-04-12T21:20:50.52Z" = + test_parse("1985-04-12T23:20:50.52+02:00", Ms), + "1985-04-12T23:20:50Z" = test_parse("1985-04-12T23:20:50.52Z", S), + "1985-04-12T23:20:50.52Z" = test_parse("1985-04-12T23:20:50.52Z", Ms), + "1985-04-12T23:20:50.52Z" = test_parse("1985-04-12t23:20:50.52z", Mys), + "1985-04-12 21:20:50.52Z" = + test_parse("1985-04-12 23:20:50.52+02:00", Ns++D), + "1985-04-12T23:20:50Z" = test_parse("1985-04-12T23:20:50.52Z"), + "1996-12-20T00:39:57Z" = test_parse("1996-12-19T16:39:57-08:00"), + "1991-01-01T00:00:00Z" = test_parse("1990-12-31T23:59:60Z"), + "1991-01-01T08:00:00Z" = test_parse("1990-12-31T23:59:60-08:00"), + + "1996-12-20T00:39:57Z" = test_parse("1996-12-19T16:39:57-08:00"), + %% The leap second is not handled: + "1991-01-01T00:00:00Z" = test_parse("1990-12-31T23:59:60Z"), + + "9999-12-31T23:59:59Z" = do_format_z(253402300799, []), + "9999-12-31T23:59:59.999Z" = do_format_z(253402300799*1000+999, Ms), + "9999-12-31T23:59:59.999999Z" = + do_format_z(253402300799*1000000+999999, Mys), + "9999-12-31T23:59:59.999999999Z" = + do_format_z(253402300799*1000000000+999999999, Ns), + {'EXIT', _} = (catch do_format_z(253402300799+1, [])), + {'EXIT', _} = (catch do_parse("9999-12-31T23:59:60Z", [])), + {'EXIT', _} = (catch do_format_z(253402300799*1000000000+999999999+1, Ns)), + 253402300799 = do_parse("9999-12-31T23:59:59Z", []), + + "0000-01-01T00:00:00Z" = test_parse("0000-01-01T00:00:00.0+00:00"), + "9999-12-31T00:00:00Z" = test_parse("9999-12-31T00:00:00.0+00:00"), + "1584-03-04T00:00:00Z" = test_parse("1584-03-04T00:00:00.0+00:00"), + "1900-01-01T00:00:00Z" = test_parse("1900-01-01T00:00:00.0+00:00"), + "2016-01-24T00:00:00Z" = test_parse("2016-01-24T00:00:00.0+00:00"), + "1970-01-01T00:00:00Z" = test_parse("1970-01-01T00:00:00Z"), + "1970-01-02T00:00:00Z" = test_parse("1970-01-01T23:59:60Z"), + "1970-01-02T00:00:00Z" = test_parse("1970-01-01T23:59:60.5Z"), + "1970-01-02T00:00:00Z" = test_parse("1970-01-01T23:59:60.55Z"), + "1970-01-02T00:00:00.55Z" = test_parse("1970-01-01T23:59:60.55Z", Ms), + "1970-01-02T00:00:00.55Z" = test_parse("1970-01-01T23:59:60.55Z", Mys), + "1970-01-02T00:00:00.55Z" = test_parse("1970-01-01T23:59:60.55Z", Ns), + "1970-01-02T00:00:00.999999Z" = + test_parse("1970-01-01T23:59:60.999999Z", Mys), + "1970-01-02T00:00:01Z" = + test_parse("1970-01-01T23:59:60.999999Z", Ms), + "1970-01-01T00:00:00Z" = test_parse("1970-01-01T00:00:00+00:00"), + "1970-01-01T00:00:00Z" = test_parse("1970-01-01T00:00:00-00:00"), + "1969-12-31T00:01:00Z" = test_parse("1970-01-01T00:00:00+23:59"), + "1918-11-11T09:00:00Z" = test_parse("1918-11-11T11:00:00+02:00", Mys), + "1970-01-01T00:00:00.000001Z" = + test_parse("1970-01-01T00:00:00.000001Z", Mys), + + test_time(erlang:system_time(second), []), + test_time(erlang:system_time(second), Z), + test_time(erlang:system_time(second), Z ++ S), + test_time(erlang:system_time(second), [{offset, "+02:20"}]), + test_time(erlang:system_time(millisecond), Ms), + test_time(erlang:system_time(microsecond), Mys++[{offset, "-02:20"}]), + + T = erlang:system_time(second), + TS = do_format(T, []), + TS = do_format(T * 1000, Ms), + TS = do_format(T * 1000 * 1000, Mys), + TS = do_format(T * 1000 * 1000 * 1000, Ns), + + 946720800 = TO = do_parse("2000-01-01 10:00:00Z", []), + Str = "2000-01-01T10:02:00+00:02", + Str = do_format(TO, [{offset, 120}]), + Str = do_format(TO * 1000, [{offset, 120 * 1000}]++Ms), + Str = do_format(TO * 1000 * 1000, [{offset, 120 * 1000 * 1000}]++Mys), + Str = do_format(TO * 1000 * 1000 * 1000, + [{offset, 120 * 1000 * 1000 * 1000}]++Ns), + + NStr = "2000-01-01T09:58:00-00:02", + NStr = do_format(TO, [{offset, -120}]), + NStr = do_format(TO * 1000, [{offset, -120 * 1000}]++Ms), + NStr = do_format(TO * 1000 * 1000, [{offset, -120 * 1000 * 1000}]++Mys), + NStr = do_format(TO * 1000 * 1000 * 1000, + [{offset, -120 * 1000 * 1000 * 1000}]++Ns), + + 543210000 = do_parse("1970-01-01T00:00:00.54321Z", Ns), + 54321000 = do_parse("1970-01-01T00:00:00.054321Z", Ns), + 543210 = do_parse("1970-01-01T00:00:00.54321Z", Mys), + 543 = do_parse("1970-01-01T00:00:00.54321Z", Ms), + 0 = do_parse("1970-01-01T00:00:00.000001Z", Ms), + 1 = do_parse("1970-01-01T00:00:00.000001Z", Mys), + 1000 = do_parse("1970-01-01T00:00:00.000001Z", Ns), + 0 = do_parse("1970-01-01Q00:00:00.00049Z", Ms), + 1 = do_parse("1970-01-01Q00:00:00.0005Z", Ms), + 6543210 = do_parse("1970-01-01T00:00:06.54321Z", Mys), + 298815132000000 = do_parse("1979-06-21T12:12:12Z", Mys), + -1613826000000000 = do_parse("1918-11-11T11:00:00Z", Mys), + -1613833200000000 = do_parse("1918-11-11T11:00:00+02:00", Mys), + -1613833200000000 = do_parse("1918-11-11T09:00:00Z", Mys), + + "1970-01-01T00:00:00Z" = do_format_z(0, Mys), + "1970-01-01T00:00:01Z" = do_format_z(1, S), + "1970-01-01T00:00:00.001Z" = do_format_z(1, Ms), + "1970-01-01T00:00:00.000001Z" = do_format_z(1, Mys), + "1970-01-01T00:00:00.000000001Z" = do_format_z(1, Ns), + "1970-01-01T00:00:01Z" = do_format_z(1000000, Mys), + "1970-01-01T00:00:00.54321Z" = do_format_z(543210, Mys), + "1970-01-01T00:00:00.543Z" = do_format_z(543, Ms), + "1970-01-01T00:00:00.54321Z" = do_format_z(543210000, Ns), + "1970-01-01T00:00:06.54321Z" = do_format_z(6543210, Mys), + "1979-06-21T12:12:12Z" = do_format_z(298815132000000, Mys), + "1918-11-11T13:00:00Z" = do_format_z(-1613818800000000, Mys), + ok. + %% %% LOCAL FUNCTIONS %% +test_parse(String) -> + test_parse(String, []). + +test_parse(String, Options) -> + T = do_parse(String, Options), + calendar:system_time_to_rfc3339(T, [{offset, "Z"} | Options]). + +do_parse(String, Options) -> + calendar:rfc3339_to_system_time(String, Options). + +test_time(Time, Options) -> + F = calendar:system_time_to_rfc3339(Time, Options), + Time = calendar:rfc3339_to_system_time(F, Options). + +do_format_z(Time, Options) -> + do_format(Time, [{offset, "Z"}|Options]). + +do_format(Time, Options) -> + calendar:system_time_to_rfc3339(Time, Options). + %% check_gregorian_days %% check_gregorian_days(Days, MaxDays) when Days < MaxDays -> diff --git a/lib/stdlib/test/dets_SUITE.erl b/lib/stdlib/test/dets_SUITE.erl index d667bd82a2..7d82790b82 100644 --- a/lib/stdlib/test/dets_SUITE.erl +++ b/lib/stdlib/test/dets_SUITE.erl @@ -3275,16 +3275,16 @@ otp_8856(Config) when is_list(Config) -> {ok, _} = dets:open_file(Tab, [{type, bag}, {file, File}]), spawn(fun()-> Me ! {1, dets:insert(Tab, [])} end), spawn(fun()-> Me ! {2, dets:insert_new(Tab, [])} end), - ok = dets:close(Tab), receive {1, ok} -> ok end, receive {2, true} -> ok end, + ok = dets:close(Tab), file:delete(File), {ok, _} = dets:open_file(Tab, [{type, set}, {file, File}]), spawn(fun() -> dets:delete(Tab, 0) end), spawn(fun() -> Me ! {3, dets:insert_new(Tab, {0,0})} end), - ok = dets:close(Tab), receive {3, true} -> ok end, + ok = dets:close(Tab), file:delete(File), ok. diff --git a/lib/stdlib/test/erl_eval_SUITE.erl b/lib/stdlib/test/erl_eval_SUITE.erl index 8eb85cab8e..f4019d477b 100644 --- a/lib/stdlib/test/erl_eval_SUITE.erl +++ b/lib/stdlib/test/erl_eval_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2017. All Rights Reserved. +%% Copyright Ericsson AB 1998-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -47,7 +47,8 @@ eval_expr_5/1, zero_width/1, eep37/1, - eep43/1]). + eep43/1, + otp_15035/1]). %% %% Define to run outside of test server @@ -87,7 +88,7 @@ all() -> otp_6539, otp_6543, otp_6787, otp_6977, otp_7550, otp_8133, otp_10622, otp_13228, otp_14826, funs, try_catch, eval_expr_5, zero_width, - eep37, eep43]. + eep37, eep43, otp_15035]. groups() -> []. @@ -1606,6 +1607,55 @@ eep43(Config) when is_list(Config) -> error_check("(#{})#{nonexisting:=value}.", {badkey,nonexisting}), ok. +otp_15035(Config) when is_list(Config) -> + check(fun() -> + fun() when #{} -> + a; + () when #{a => b} -> + b; + () when #{a => b} =:= #{a => b} -> + c + end() + end, + "fun() when #{} -> + a; + () when #{a => b} -> + b; + () when #{a => b} =:= #{a => b} -> + c + end().", + c), + check(fun() -> + F = fun(M) when M#{} -> + a; + (M) when M#{a => b} -> + b; + (M) when M#{a := b} -> + c; + (M) when M#{a := b} =:= M#{a := b} -> + d; + (M) when M#{a => b} =:= M#{a => b} -> + e + end, + {F(#{}), F(#{a => b})} + end, + "fun() -> + F = fun(M) when M#{} -> + a; + (M) when M#{a => b} -> + b; + (M) when M#{a := b} -> + c; + (M) when M#{a := b} =:= M#{a := b} -> + d; + (M) when M#{a => b} =:= M#{a => b} -> + e + end, + {F(#{}), F(#{a => b})} + end().", + {e, d}), + ok. + %% Check the string in different contexts: as is; in fun; from compiled code. check(F, String, Result) -> check1(F, String, Result), diff --git a/lib/stdlib/test/erl_lint_SUITE.erl b/lib/stdlib/test/erl_lint_SUITE.erl index e40f5e9a5d..f9ab83a120 100644 --- a/lib/stdlib/test/erl_lint_SUITE.erl +++ b/lib/stdlib/test/erl_lint_SUITE.erl @@ -66,7 +66,7 @@ otp_11851/1,otp_11879/1,otp_13230/1, record_errors/1, otp_11879_cont/1, non_latin1_module/1, otp_14323/1, - get_stacktrace/1, stacktrace_syntax/1, + stacktrace_syntax/1, otp_14285/1, otp_14378/1]). suite() -> @@ -88,7 +88,7 @@ all() -> maps, maps_type, maps_parallel_match, otp_11851, otp_11879, otp_13230, record_errors, otp_11879_cont, non_latin1_module, otp_14323, - get_stacktrace, stacktrace_syntax, otp_14285, otp_14378]. + stacktrace_syntax, otp_14285, otp_14378]. groups() -> [{unused_vars_warn, [], @@ -4055,82 +4055,6 @@ otp_14323(Config) -> [] = run(Config, Ts), ok. -get_stacktrace(Config) -> - Ts = [{old_catch, - <<"t1() -> - catch error(foo), - erlang:get_stacktrace(). - ">>, - [], - {warnings,[{3,erl_lint,{get_stacktrace,after_old_catch}}]}}, - {nowarn_get_stacktrace, - <<"t1() -> - catch error(foo), - erlang:get_stacktrace(). - ">>, - [nowarn_get_stacktrace], - []}, - {try_catch, - <<"t1(X) -> - try abs(X) of - _ -> - erlang:get_stacktrace() - catch - _:_ -> ok - end. - - t2() -> - try error(foo) - catch _:_ -> ok - end, - erlang:get_stacktrace(). - - t3() -> - try error(foo) - catch _:_ -> - try error(bar) - catch _:_ -> - ok - end, - erlang:get_stacktrace() - end. - - no_warning(X) -> - try - abs(X) - catch - _:_ -> - erlang:get_stacktrace() - end. - ">>, - [], - {warnings,[{4,erl_lint,{get_stacktrace,wrong_part_of_try}}, - {13,erl_lint,{get_stacktrace,after_try}}, - {22,erl_lint,{get_stacktrace,after_try}}]}}, - {multiple_catch_clauses, - <<"maybe_error(Arg) -> - try 5 / Arg - catch - error:badarith -> - _Stacktrace = erlang:get_stacktrace(), - try io:nl() - catch - error:_ -> io:format('internal error') - end; - error:badarg -> - _Stacktrace = erlang:get_stacktrace(), - try io:format(qwe) - catch - error:_ -> io:format('internal error') - end - end. - ">>, - [], - []}], - - run(Config, Ts), - ok. - stacktrace_syntax(Config) -> Ts = [{guard, <<"t1() -> diff --git a/lib/stdlib/test/error_logger_h_SUITE.erl b/lib/stdlib/test/error_logger_h_SUITE.erl index 9dc04f27a1..d533305939 100644 --- a/lib/stdlib/test/error_logger_h_SUITE.erl +++ b/lib/stdlib/test/error_logger_h_SUITE.erl @@ -62,6 +62,7 @@ logfile(Config) -> error_logger:logfile({open,Log}), ok = rpc:call(Node, erlang, apply, [fun gen_events/1,[Ev]]), AtNode = iolist_to_binary(["** at node ",atom_to_list(Node)," **"]), + timer:sleep(1000), % some time get all log events in the log error_logger:logfile(close), analyse_events(Log, Ev, [AtNode], unlimited), @@ -124,6 +125,7 @@ tty(Config) -> ok = rpc:call(Node, erlang, apply, [fun gen_events/1,[Ev]]), tty_log_close(), AtNode = iolist_to_binary(["** at node ",atom_to_list(Node)," **"]), + timer:sleep(1000), % some time get all log events in the log analyse_events(Log, Ev, [AtNode], unlimited), test_server:stop_node(Node), @@ -207,7 +209,7 @@ event_templates() -> gen_events(Ev) -> io:format("node = ~p\n", [node()]), io:format("group leader = ~p\n", [group_leader()]), - io:format("~p\n", [gen_event:which_handlers(error_logger)]), + io:format("~p\n", [error_logger:which_report_handlers()]), call_error_logger(Ev), {Pid,Ref} = spawn_monitor(fun() -> error(ouch) end), @@ -240,6 +242,7 @@ analyse_events(Log, Ev, AtNode, Depth) -> call_error_logger([{F,Args}|T]) -> apply(error_logger, F, Args), + timer:sleep(10), call_error_logger(T); call_error_logger([]) -> ok. diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index 8b651f4b43..02211fa8df 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -55,6 +55,7 @@ -export([t_repair_continuation/1]). -export([t_match_spec_run/1]). -export([t_bucket_disappears/1]). +-export([t_named_select/1]). -export([otp_5340/1]). -export([otp_6338/1]). -export([otp_6842_select_1000/1]). @@ -124,6 +125,7 @@ all() -> t_init_table, t_whitebox, t_delete_all_objects, t_insert_list, t_test_ms, t_select_delete, t_select_replace, t_ets_dets, memory, t_select_reverse, t_bucket_disappears, + t_named_select, select_fail, t_insert_new, t_repair_continuation, otp_5340, otp_6338, otp_6842_select_1000, otp_7665, otp_8732, meta_wb, grow_shrink, grow_pseudo_deleted, @@ -205,6 +207,38 @@ t_bucket_disappears_do(Opts) -> true = ets:delete(abcd), verify_etsmem(EtsMem). +%% OTP-21: Test that select/1 fails if named table was deleted and recreated +%% and succeeds if table was renamed. +t_named_select(_Config) -> + repeat_for_opts(fun t_named_select_do/1). + +t_named_select_do(Opts) -> + EtsMem = etsmem(), + T = t_name_tid_select, + ets_new(T, [named_table | Opts]), + ets:insert(T, {1,11}), + ets:insert(T, {2,22}), + ets:insert(T, {3,33}), + MS = [{{'$1', 22}, [], ['$1']}], + {[2], Cont1} = ets:select(T, MS, 1), + ets:delete(T), + {'EXIT',{badarg,_}} = (catch ets:select(Cont1)), + ets_new(T, [named_table | Opts]), + {'EXIT',{badarg,_}} = (catch ets:select(Cont1)), + + true = ets:insert_new(T, {1,22}), + true = ets:insert_new(T, {2,22}), + true = ets:insert_new(T, {4,22}), + {[A,B], Cont2} = ets:select(T, MS, 2), + ets:rename(T, abcd), + {[C], '$end_of_table'} = ets:select(Cont2), + 7 = A + B + C, + + true = ets:delete(abcd), + verify_etsmem(EtsMem). + + + %% Check ets:match_spec_run/2. t_match_spec_run(Config) when is_list(Config) -> @@ -700,7 +734,7 @@ whitebox_2(Opts) -> ets:delete(T2), ok. -select_bound_chunk(Config) -> +select_bound_chunk(_Config) -> repeat_for_opts(fun select_bound_chunk_do/1, [all_types]). select_bound_chunk_do(Opts) -> @@ -3649,7 +3683,7 @@ verify_rescheduling_exit(Config, ForEachData, Flags, Fix, NOTabs, NOProcs) -> XScheds = count_exit_sched(TP), io:format("~p XScheds=~p~n", [TP, XScheds]), - true = XScheds >= 5 + true = XScheds >= 3 end, TPs), stop_loopers(LPs), diff --git a/lib/stdlib/test/gen_statem_SUITE.erl b/lib/stdlib/test/gen_statem_SUITE.erl index 7c8a386116..053233df9b 100644 --- a/lib/stdlib/test/gen_statem_SUITE.erl +++ b/lib/stdlib/test/gen_statem_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2016-2017. All Rights Reserved. +%% Copyright Ericsson AB 2016-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -60,7 +60,8 @@ tcs(start) -> tcs(stop) -> [stop1, stop2, stop3, stop4, stop5, stop6, stop7, stop8, stop9, stop10]; tcs(abnormal) -> - [abnormal1, abnormal1clean, abnormal1dirty, abnormal2]; + [abnormal1, abnormal1clean, abnormal1dirty, + abnormal2, abnormal3, abnormal4]; tcs(sys) -> [sys1, call_format_status, error_format_status, terminate_crash_format, @@ -524,6 +525,43 @@ abnormal2(Config) -> process_flag(trap_exit, OldFl), ok = verify_empty_msgq(). +%% Check that bad return actions makes the stm crash. Note that we must +%% trap exit since we must link to get the real bad_return_ error +abnormal3(Config) -> + OldFl = process_flag(trap_exit, true), + {ok,Pid} = gen_statem:start_link(?MODULE, start_arg(Config, []), []), + + %% bad return value in the gen_statem loop + {{{bad_action_from_state_function,badaction},_},_} = + ?EXPECT_FAILURE(gen_statem:call(Pid, badaction), Reason), + receive + {'EXIT',Pid,{{bad_action_from_state_function,badaction},_}} -> ok + after 5000 -> + ct:fail(gen_statem_did_not_die) + end, + + process_flag(trap_exit, OldFl), + ok = verify_empty_msgq(). + +%% Check that bad timeout actions makes the stm crash. Note that we must +%% trap exit since we must link to get the real bad_return_ error +abnormal4(Config) -> + OldFl = process_flag(trap_exit, true), + {ok,Pid} = gen_statem:start_link(?MODULE, start_arg(Config, []), []), + + %% bad return value in the gen_statem loop + BadTimeout = {badtimeout,4711,ouch}, + {{{bad_action_from_state_function,BadTimeout},_},_} = + ?EXPECT_FAILURE(gen_statem:call(Pid, BadTimeout), Reason), + receive + {'EXIT',Pid,{{bad_action_from_state_function,BadTimeout},_}} -> ok + after 5000 -> + ct:fail(gen_statem_did_not_die) + end, + + process_flag(trap_exit, OldFl), + ok = verify_empty_msgq(). + shutdown(Config) -> process_flag(trap_exit, true), @@ -832,9 +870,14 @@ event_types(_Config) -> %% Abusing the internal format of From... #{init => fun () -> - {ok, start, undefined} + {ok, start1, undefined, + [{next_event,internal,0}]} end, - start => + start1 => + fun (internal, 0, undefined) -> + {next_state, start2, undefined} + end, + start2 => fun ({call,_} = Call, Req, undefined) -> {next_state, state1, undefined, [{next_event,internal,1}, @@ -1801,10 +1844,12 @@ idle(cast, {connect,Pid}, Data) -> idle({call,From}, connect, Data) -> gen_statem:reply(From, accept), {next_state,wfor_conf,Data,infinity}; % NoOp timeout just to test API -idle(cast, badreturn, _Data) -> - badreturn; idle({call,_From}, badreturn, _Data) -> badreturn; +idle({call,_From}, badaction, Data) -> + {keep_state, Data, [badaction]}; +idle({call,_From}, {badtimeout,_,_} = BadTimeout, Data) -> + {keep_state, Data, BadTimeout}; idle({call,From}, {delayed_answer,T}, Data) -> receive after T -> diff --git a/lib/stdlib/test/io_SUITE.erl b/lib/stdlib/test/io_SUITE.erl index 6f4e7ad7e0..9f48fbf5e3 100644 --- a/lib/stdlib/test/io_SUITE.erl +++ b/lib/stdlib/test/io_SUITE.erl @@ -31,9 +31,9 @@ otp_10836/1, io_lib_width_too_small/1, io_with_huge_message_queue/1, format_string/1, maps/1, coverage/1, otp_14178_unicode_atoms/1, otp_14175/1, - otp_14285/1, limit_term/1]). + otp_14285/1, limit_term/1, otp_14983/1]). --export([pretty/2]). +-export([pretty/2, trf/3]). %%-define(debug, true). @@ -63,7 +63,7 @@ all() -> io_lib_print_binary_depth_one, otp_10302, otp_10755, otp_10836, io_lib_width_too_small, io_with_huge_message_queue, format_string, maps, coverage, otp_14178_unicode_atoms, otp_14175, - otp_14285, limit_term]. + otp_14285, limit_term, otp_14983]. %% Error cases for output. error_1(Config) when is_list(Config) -> @@ -1750,7 +1750,7 @@ printable_range(Suite) when is_list(Suite) -> PrettyOptions = [{column,1}, {line_length,109}, {depth,30}, - {max_chars,60}, + {line_max_chars,60}, {record_print_fun, fun(_,_) -> no end}, {encoding,unicode}], @@ -1886,7 +1886,7 @@ otp_10302(Suite) when is_list(Suite) -> pretty(Term, Depth) when is_integer(Depth) -> Opts = [{column, 1}, {line_length, 20}, - {depth, Depth}, {max_chars, 60}, + {depth, Depth}, {line_max_chars, 60}, {record_print_fun, fun rfd/2}, {encoding, unicode}], pretty(Term, Opts); @@ -2053,19 +2053,19 @@ maps(_Config) -> %% in a map with more than one element. "#{}" = fmt("~w", [#{}]), - "#{a=>b}" = fmt("~w", [#{a=>b}]), - re_fmt(<<"#\\{(a=>b|c=>d),[.][.][.]=>[.][.][.]\\}">>, - "~W", [#{a=>b,c=>d},2]), - re_fmt(<<"#\\{(a=>b|c=>d|e=>f),[.][.][.]=>[.][.][.],[.][.][.]\\}">>, - "~W", [#{a=>b,c=>d,e=>f},2]), + "#{a => b}" = fmt("~w", [#{a=>b}]), + re_fmt(<<"#\\{(a => b),[.][.][.]\\}">>, + "~W", [#{a => b,c => d},2]), + re_fmt(<<"#\\{(a => b),[.][.][.]\\}">>, + "~W", [#{a => b,c => d,e => f},2]), "#{}" = fmt("~p", [#{}]), - "#{a => b}" = fmt("~p", [#{a=>b}]), - "#{...}" = fmt("~P", [#{a=>b},1]), + "#{a => b}" = fmt("~p", [#{a => b}]), + "#{...}" = fmt("~P", [#{a => b},1]), re_fmt(<<"#\\{(a => b|c => d),[.][.][.]\\}">>, - "~P", [#{a=>b,c=>d},2]), + "~P", [#{a => b,c => d},2]), re_fmt(<<"#\\{(a => b|c => d|e => f),[.][.][.]\\}">>, - "~P", [#{a=>b,c=>d,e=>f},2]), + "~P", [#{a => b,c => d,e => f},2]), List = [{I,I*I} || I <- lists:seq(1, 20)], Map = maps:from_list(List), @@ -2441,7 +2441,7 @@ limit_term(_Config) -> {_, 1} = limt(T, 0), {_, 2} = limt(T, 1), {_, 2} = limt(T, 2), - {_, 1} = limt(T, 3), + {_, 2} = limt(T, 3), {_, 1} = limt(T, 4), T2 = #{[] => {},{} => []}, {_, 2} = limt(T2, 1), @@ -2489,3 +2489,129 @@ limt_pp(Term, Depth) when is_integer(Depth) -> pp(Term, Depth) -> lists:flatten(io_lib:format("~P", [Term, Depth])). + +otp_14983(_Config) -> + trunc_depth(-1, fun trp/3), + trunc_depth(10, fun trp/3), + trunc_depth(-1, fun trw/3), + trunc_depth(10, fun trw/3), + trunc_depth_p(-1), + trunc_depth_p(10), + trunc_string(), + ok. + +trunc_string() -> + "str " = trf("str ", [], 10), + "str ..." = trf("str ~s", ["str"], 6), + "str str" = trf("str ~s", ["str"], 7), + "str ..." = trf("str ~8s", ["str"], 6), + Pa = filename:dirname(code:which(?MODULE)), + {ok, UNode} = test_server:start_node(printable_range_unicode, slave, + [{args, " +pc unicode -pa " ++ Pa}]), + U = "кирилли́ческий атом", + UFun = fun(Format, Args, CharsLimit) -> + rpc:call(UNode, + ?MODULE, trf, [Format, Args, CharsLimit]) + end, + "str кир" = UFun("str ~3ts", [U], 7), + "str ..." = UFun("str ~3ts", [U], 6), + "str ..." = UFun("str ~30ts", [U], 6), + "str кир..." = UFun("str ~30ts", [U], 10), + "str кирилл..." = UFun("str ~30ts", [U], 13), + "str кирилли́..." = UFun("str ~30ts", [U], 14), + "str кирилли́ч..." = UFun("str ~30ts", [U], 15), + "\"кирилли́ческ\"..." = UFun("~tp", [U], 13), + BU = <<"кирилли́ческий атом"/utf8>>, + "<<\"кирилли́\"/utf8...>>" = UFun("~tp", [BU], 20), + "<<\"кирилли́\"/utf8...>>" = UFun("~tp", [BU], 21), + "<<\"кирилли́ческ\"/utf8...>>" = UFun("~tp", [BU], 22), + test_server:stop_node(UNode). + +trunc_depth(D, Fun) -> + "..." = Fun("", D, 0), + "[]" = Fun("", D, 1), + + "#{}" = Fun(#{}, D, 1), + "#{a => 1}" = Fun(#{a => 1}, D, 7), + "#{...}" = Fun(#{a => 1}, D, 5), + "#{a => 1}" = Fun(#{a => 1}, D, 6), + A = lists:seq(1, 1000), + M = #{A => A, {A,A} => {A,A}}, + "#{...}" = Fun(M, D, 6), + "#{{...} => {...},...}" = Fun(M, D, 7), + "#{{[...],...} => {[...],...},...}" = Fun(M, D, 22), + "#{{[...],...} => {[...],...},[...] => [...]}" = Fun(M, D, 31), + "#{{[...],...} => {[...],...},[...] => [...]}" = Fun(M, D, 33), + "#{{[1|...],[...]} => {[1|...],[...]},[1,2|...] => [...]}" = + Fun(M, D, 50), + + "..." = Fun({c, 1, 2}, D, 0), + "{...}" = Fun({c, 1, 2}, D, 1), + + "..." = Fun({}, D, 0), + "{}" = Fun({}, D, 1), + T = {A, A, A}, + "{...}" = Fun(T, D, 5), + "{[...],...}" = Fun(T, D, 6), + "{[1|...],[...],...}" = Fun(T, D, 12), + "{[1,2|...],[1|...],...}" = Fun(T, D, 20), + "{[1,2|...],[1|...],[...]}" = Fun(T, D, 21), + "{[1,2,3|...],[1,2|...],[1|...]}" = Fun(T, D, 28), + + "{[1],[1,2|...]}" = Fun({[1],[1,2,3,4]}, D, 14). + +trunc_depth_p(D) -> + UOpts = [{record_print_fun, fun rfd/2}, + {encoding, unicode}], + LOpts = [{record_print_fun, fun rfd/2}, + {encoding, latin1}], + trunc_depth_p(D, UOpts), + trunc_depth_p(D, LOpts). + +trunc_depth_p(D, Opts) -> + "[...]" = trp("abcdefg", D, 4, Opts), + "\"abc\"..." = trp("abcdefg", D, 5, Opts), + "\"abcdef\"..." = trp("abcdefg", D, 8, Opts), + "\"abcdefg\"" = trp("abcdefg", D, 9, Opts), + "\"abcdefghijkl\"" = trp("abcdefghijkl", D, -1, Opts), + AZ = lists:seq($A, $Z), + AZb = list_to_binary(AZ), + AZbS = "<<\"" ++ AZ ++ "\">>", + AZbS = trp(AZb, D, -1), + "<<\"ABCDEFGH\"...>>" = trp(AZb, D, 17, Opts), % 4 chars even if D = -1... + "<<\"ABCDEFGHIJKL\"...>>" = trp(AZb, D, 18, Opts), + B1 = <<"abcdef",0:8>>, + "<<\"ab\"...>>" = trp(B1, D, 8, Opts), + "<<\"abcdef\"...>>" = trp(B1, D, 14, Opts), + "<<97,98,99,100,...>>" = trp(B1, D, 16, Opts), + "<<97,98,99,100,101,102,0>>" = trp(B1, D, -1, Opts), + B2 = <<AZb/binary,0:8>>, + "<<\"AB\"...>>" = trp(B2, D, 8, Opts), + "<<\"ABCDEFGH\"...>>" = trp(B2, D, 14, Opts), + "<<65,66,67,68,69,70,71,72,0>>" = trp(<<"ABCDEFGH",0:8>>, D, -1, Opts), + "<<97,0,107,108,...>>" = trp(<<"a",0:8,"kllkjlksdjfsj">>, D, 20, Opts), + + A = lists:seq(1, 1000), + "#c{...}" = trp({c, 1, 2}, D, 6), + "#c{...}" = trp({c, 1, 2}, D, 7), + "#c{f1 = [...],...}" = trp({c, A, A}, D, 18), + "#c{f1 = [1|...],f2 = [...]}" = trp({c, A, A}, D, 19), + "#c{f1 = [1,2|...],f2 = [1|...]}" = trp({c, A, A}, D, 31), + "#c{f1 = [1,2,3|...],f2 = [1,2|...]}" = trp({c, A, A}, D, 32). + +trp(Term, D, T) -> + trp(Term, D, T, [{record_print_fun, fun rfd/2}]). + +trp(Term, D, T, Opts) -> + R = io_lib_pretty:print(Term, [{depth, D}, + {chars_limit, T}|Opts]), + lists:flatten(io_lib:format("~s", [R])). + +trw(Term, D, T) -> + lists:flatten(io_lib:format("~W", [Term, D], [{chars_limit, T}])). + +trf(Format, Args, T) -> + trf(Format, Args, T, [{record_print_fun, fun rfd/2}]). + +trf(Format, Args, T, Opts) -> + lists:flatten(io_lib:format(Format, Args, [{chars_limit, T}|Opts])). diff --git a/lib/stdlib/test/lists_SUITE.erl b/lib/stdlib/test/lists_SUITE.erl index 7c99244b36..837ab4e97e 100644 --- a/lib/stdlib/test/lists_SUITE.erl +++ b/lib/stdlib/test/lists_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2017. All Rights Reserved. +%% Copyright Ericsson AB 1997-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -57,7 +57,7 @@ filter_partition/1, join/1, otp_5939/1, otp_6023/1, otp_6606/1, otp_7230/1, - suffix/1, subtract/1, droplast/1, hof/1]). + suffix/1, subtract/1, droplast/1, search/1, hof/1]). %% Sort randomized lists until stopped. %% @@ -121,7 +121,7 @@ groups() -> {zip, [parallel], [zip_unzip, zip_unzip3, zipwith, zipwith3]}, {misc, [parallel], [reverse, member, dropwhile, takewhile, filter_partition, suffix, subtract, join, - hof, droplast]} + hof, droplast, search]} ]. init_per_suite(Config) -> @@ -2615,6 +2615,20 @@ droplast(Config) when is_list(Config) -> ok. +%% Test lists:search/2 +search(Config) when is_list(Config) -> + F = fun(I) -> I rem 2 =:= 0 end, + F2 = fun(A, B) -> A > B end, + + {value, 2} = lists:search(F, [1,2,3,4]), + false = lists:search(F, [1,3,5,7]), + false = lists:search(F, []), + + %% Error cases. + {'EXIT',{function_clause,_}} = (catch lists:search(badfun, [])), + {'EXIT',{function_clause,_}} = (catch lists:search(F2, [])), + ok. + %% Briefly test the common high-order functions to ensure they %% are covered. hof(Config) when is_list(Config) -> diff --git a/lib/stdlib/test/proc_lib_SUITE.erl b/lib/stdlib/test/proc_lib_SUITE.erl index fbdcb518b2..81bf9020b8 100644 --- a/lib/stdlib/test/proc_lib_SUITE.erl +++ b/lib/stdlib/test/proc_lib_SUITE.erl @@ -542,16 +542,17 @@ system_terminate(Reason,_Parent,_Deb,_State) -> t_format(_Config) -> - error_logger:tty(false), + logger:add_handler_filter(logger_std_h,stop_all,{fun(_,_) -> stop end,ok}), + error_logger:add_report_handler(?MODULE, self()), try t_format() after - error_logger:tty(true) + error_logger:delete_report_handler(?MODULE), + logger:remove_handler_filter(logger_std_h,stop_all) end, ok. t_format() -> - error_logger:add_report_handler(?MODULE, self()), Pid = proc_lib:spawn(fun '\x{aaa}t_format_looper'/0), HugeData = gb_sets:from_list(lists:seq(1, 100)), SomeData1 = list_to_atom([246]), @@ -584,11 +585,11 @@ t_format() -> ok. t_format_arbitrary(_Config) -> - error_logger:tty(false), + logger:add_handler_filter(logger_std_h,stop_all,{fun(_,_) -> stop end,ok}), try t_format_arbitrary() after - error_logger:tty(true) + logger:remove_handler_filter(logger_std_h,stop_all) end, ok. diff --git a/lib/stdlib/test/sets_SUITE.erl b/lib/stdlib/test/sets_SUITE.erl index bec38000b2..7066d07e19 100644 --- a/lib/stdlib/test/sets_SUITE.erl +++ b/lib/stdlib/test/sets_SUITE.erl @@ -28,7 +28,7 @@ init_per_testcase/2,end_per_testcase/2, create/1,add_element/1,del_element/1, subtract/1,intersection/1,union/1,is_subset/1, - is_set/1,fold/1,filter/1, + is_set/1,is_empty/1,fold/1,filter/1, take_smallest/1,take_largest/1, iterate/1]). -include_lib("common_test/include/ct.hrl"). @@ -48,7 +48,7 @@ suite() -> all() -> [create, add_element, del_element, subtract, intersection, union, is_subset, is_set, fold, filter, - take_smallest, take_largest, iterate]. + take_smallest, take_largest, iterate, is_empty]. groups() -> []. @@ -345,6 +345,17 @@ is_set_1(M) -> false = M(is_set, {}), M(empty, []). +is_empty(Config) when is_list(Config) -> + test_all(fun is_empty_1/1). + +is_empty_1(M) -> + S = M(from_list, [blurf]), + Empty = M(empty, []), + + true = M(is_empty, Empty), + false = M(is_empty, S), + M(empty, []). + fold(Config) when is_list(Config) -> test_all([{0,71},{125,129},{254,259},{510,513},{1023,1025},{9999,10001}], fun fold_1/2). diff --git a/lib/stdlib/test/sets_test_lib.erl b/lib/stdlib/test/sets_test_lib.erl index 9f153822a2..93d027704b 100644 --- a/lib/stdlib/test/sets_test_lib.erl +++ b/lib/stdlib/test/sets_test_lib.erl @@ -32,7 +32,7 @@ new(Mod, Eq) -> (from_list, L) -> Mod:from_list(L); (intersection, {S1,S2}) -> intersection(Mod, Eq, S1, S2); (intersection, Ss) -> intersection(Mod, Eq, Ss); - (is_empty, S) -> is_empty(Mod, S); + (is_empty, S) -> Mod:is_empty(S); (is_set, S) -> Mod:is_set(S); (is_subset, {S,Set}) -> is_subset(Mod, Eq, S, Set); (iterator, S) -> Mod:iterator(S); @@ -56,7 +56,7 @@ singleton(Mod, E) -> add_element(Mod, El, S0) -> S = Mod:add_element(El, S0), true = Mod:is_element(El, S), - false = is_empty(Mod, S), + false = Mod:is_empty(S), true = Mod:is_set(S), S. @@ -66,17 +66,10 @@ del_element(Mod, El, S0) -> true = Mod:is_set(S), S. -is_empty(Mod, S) -> - true = Mod:is_set(S), - case erlang:function_exported(Mod, is_empty, 1) of - true -> Mod:is_empty(S); - false -> Mod:size(S) == 0 - end. - intersection(Mod, Equal, S1, S2) -> S = Mod:intersection(S1, S2), true = Equal(S, Mod:intersection(S2, S1)), - Disjoint = is_empty(Mod, S), + Disjoint = Mod:is_empty(S), Disjoint = Mod:is_disjoint(S1, S2), Disjoint = Mod:is_disjoint(S2, S1), S. diff --git a/lib/stdlib/test/stdlib_bench_SUITE.erl b/lib/stdlib/test/stdlib_bench_SUITE.erl index 294898a932..2364e8376f 100644 --- a/lib/stdlib/test/stdlib_bench_SUITE.erl +++ b/lib/stdlib/test/stdlib_bench_SUITE.erl @@ -348,12 +348,16 @@ do_tests(Test, ParamSet, Config) -> {Parallelism, Message} = bench_params(ParamSet), Fun = create_clients(Message, ServerMod, Client, Parallelism), {TotalLoops, AllPidTime} = run_test(Fun), - PerSecond = ?CALLS_PER_LOOP * round((1000 * TotalLoops) / AllPidTime), - ct_event:notify( - #event{ - name = benchmark_data, - data = [{suite,BenchmarkSuite},{value,PerSecond}]}), - PerSecond. + try ?CALLS_PER_LOOP * round((1000 * TotalLoops) / AllPidTime) of + PerSecond -> + ct_event:notify( + #event{ + name = benchmark_data, + data = [{suite,BenchmarkSuite},{value,PerSecond}]}), + PerSecond + catch error:badarith -> + "Time measurement is not working" + end. -define(COUNTER, n). diff --git a/lib/stdlib/test/sys_SUITE.erl b/lib/stdlib/test/sys_SUITE.erl index b44df0fbda..439a23d82d 100644 --- a/lib/stdlib/test/sys_SUITE.erl +++ b/lib/stdlib/test/sys_SUITE.erl @@ -84,7 +84,7 @@ stats(Config) when is_list(Config) -> {ok,-44} = public_call(44), {ok,Stats} = sys:statistics(?server,get), true = lists:member({messages_in,1}, Stats), - true = lists:member({messages_out,0}, Stats), + true = lists:member({messages_out,1}, Stats), ok = sys:statistics(?server,false), {status,_Pid,{module,_Mod},[_PDict,running,Self,_,_]} = sys:get_status(?server), @@ -133,7 +133,8 @@ install(Config) when is_list(Config) -> Master ! {spy_got,{request,Arg},ProcState}; Other -> io:format("Trigged other=~p\n",[Other]) - end + end, + func_state end, sys:install(?server,{SpyFun,func_state}), {ok,-1} = (catch public_call(1)), @@ -142,10 +143,27 @@ install(Config) when is_list(Config) -> sys:install(?server,{SpyFun,func_state}), sys:install(?server,{SpyFun,func_state}), {ok,-3} = (catch public_call(3)), - sys:remove(?server,SpyFun), {ok,-4} = (catch public_call(4)), + sys:remove(?server,SpyFun), + {ok,-5} = (catch public_call(5)), + [{spy_got,{request,1},sys_SUITE_server}, + {spy_got,{request,3},sys_SUITE_server}, + {spy_got,{request,4},sys_SUITE_server}] = get_messages(), + + sys:install(?server,{id1, SpyFun, func_state}), + sys:install(?server,{id1, SpyFun, func_state}), %% should not be installed + sys:install(?server,{id2, SpyFun, func_state}), + {ok,-1} = (catch public_call(1)), + %% We have two SpyFun installed: [{spy_got,{request,1},sys_SUITE_server}, - {spy_got,{request,3},sys_SUITE_server}] = get_messages(), + {spy_got,{request,1},sys_SUITE_server}] = get_messages(), + sys:remove(?server, id1), + {ok,-1} = (catch public_call(1)), + %% We have one SpyFun installed: + [{spy_got,{request,1},sys_SUITE_server}] = get_messages(), + sys:no_debug(?server), + {ok,-1} = (catch public_call(1)), + [] = get_messages(), stop(), ok. diff --git a/lib/stdlib/test/zzz_SUITE.erl b/lib/stdlib/test/zzz_SUITE.erl new file mode 100644 index 0000000000..59c7fd7404 --- /dev/null +++ b/lib/stdlib/test/zzz_SUITE.erl @@ -0,0 +1,37 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2006-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(zzz_SUITE). + +%% The sole purpose of this test suite is for things we want to run last +%% before the VM terminates. + +-export([all/0]). + +-export([lc_graph/1]). + + +all() -> + [lc_graph]. + +lc_graph(_Config) -> + %% Create "lc_graph" file in current working dir + %% if lock checker is enabled. + erts_debug:lc_graph(), + ok. diff --git a/lib/stdlib/vsn.mk b/lib/stdlib/vsn.mk index 8391389fc4..09a4d6fb50 100644 --- a/lib/stdlib/vsn.mk +++ b/lib/stdlib/vsn.mk @@ -1 +1 @@ -STDLIB_VSN = 3.4.4 +STDLIB_VSN = 3.4.5 diff --git a/lib/syntax_tools/doc/src/Makefile b/lib/syntax_tools/doc/src/Makefile index 1ce620b3d6..a346b9a0bd 100644 --- a/lib/syntax_tools/doc/src/Makefile +++ b/lib/syntax_tools/doc/src/Makefile @@ -63,10 +63,11 @@ BOOK_FILES = book.xml XML_FILES=\ - $(BOOK_FILES) $(XML_CHAPTER_FILES) \ - $(XML_PART_FILES) $(XML_REF3_FILES) $(XML_APPLICATION_FILES) \ + $(BOOK_FILES) $(XML_PART_FILES) $(XML_APPLICATION_FILES) \ $(XML_NOTES_FILES) +XML_GEN_FILES = $(XML_REF3_FILES:%=$(XMLDIR)/%) $(XML_CHAPTER_FILES:%=$(XMLDIR)/%) + # ---------------------------------------------------- INFO_FILE = ../../info @@ -108,11 +109,13 @@ html: gifs $(HTML_REF_MAN_FILE) man: $(MAN3_FILES) -$(XML_REF3_FILES): - escript $(DOCGEN)/priv/bin/xml_from_edoc.escript $(SRC_DIR)/$(@:%.xml=%.erl) +$(XML_REF3_FILES:%=$(XMLDIR)/%): + $(gen_verbose)escript $(DOCGEN)/priv/bin/xml_from_edoc.escript \ + -dir $(XMLDIR) $(SRC_DIR)/$(@:$(XMLDIR)/%.xml=%.erl) -$(XML_CHAPTER_FILES): - escript $(DOCGEN)/priv/bin/xml_from_edoc.escript -def vsn $(VSN) -chapter ../overview.edoc +$(XML_CHAPTER_FILES:%=$(XMLDIR)/%): + $(gen_verbose)escript $(DOCGEN)/priv/bin/xml_from_edoc.escript -def vsn $(VSN) \ + -chapter -dir $(XMLDIR) ../overview.edoc gifs: $(GIF_FILES:%=$(HTMLDIR)/%) diff --git a/lib/tftp/AUTHORS b/lib/tftp/AUTHORS new file mode 100644 index 0000000000..71fc97c4e0 --- /dev/null +++ b/lib/tftp/AUTHORS @@ -0,0 +1,11 @@ +Original Authors: + +Håkan Mattsson - tftp + +Contributors: + +Ingela Anderton Andin +Martin Gustafsson +Johan Blom +Torbjörn Törnkvist +Joe Armstrong
\ No newline at end of file diff --git a/lib/tftp/Makefile b/lib/tftp/Makefile new file mode 100644 index 0000000000..5c3ed52b28 --- /dev/null +++ b/lib/tftp/Makefile @@ -0,0 +1,78 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 1996-2016. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# %CopyrightEnd% +# +# +include $(ERL_TOP)/make/target.mk +include $(ERL_TOP)/make/$(TARGET)/otp.mk + +# ---------------------------------------------------- +# Macros +# ---------------------------------------------------- + +SUB_DIRECTORIES = src doc/src + +include vsn.mk +VSN = $(TFTP_VSN) + +SPECIAL_TARGETS = + +DIA_PLT = ./priv/plt/$(APPLICATION).plt +DIA_ANALYSIS = $(basename $(DIA_PLT)).dialyzer_analysis + + +# ---------------------------------------------------- +# Default Subdir Targets +# ---------------------------------------------------- +include $(ERL_TOP)/make/otp_subdir.mk + +.PHONY: info gclean dialyzer dialyzer_plt dclean + +info: + @echo "OS: $(OS)" + @echo "DOCB: $(DOCB)" + @echo "" + @echo "TFTP_VSN: $(TFTP_VSN)" + @echo "APP_VSN: $(APP_VSN)" + @echo "" + @echo "DIA_PLT: $(DIA_PLT)" + @echo "DIA_ANALYSIS: $(DIA_ANALYSIS)" + @echo "" + +gclean: + git clean -fXd + +dclean: + rm -f $(DIA_PLT) + rm -f $(DIA_ANALYSIS) + +dialyzer_plt: $(DIA_PLT) + +$(DIA_PLT): + @echo "Building $(APPLICATION) plt file" + @dialyzer --build_plt \ + --output_plt $@ \ + -r ../$(APPLICATION)/ebin \ + --output $(DIA_ANALYSIS) \ + --verbose + +dialyzer: $(DIA_PLT) + @echo "Running dialyzer on $(APPLICATION)" + @dialyzer --plt $< \ + ../$(APPLICATION)/ebin \ + --verbose diff --git a/lib/tftp/doc/html/.gitignore b/lib/tftp/doc/html/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/tftp/doc/html/.gitignore diff --git a/lib/tftp/doc/man3/.gitignore b/lib/tftp/doc/man3/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/tftp/doc/man3/.gitignore diff --git a/lib/tftp/doc/man6/.gitignore b/lib/tftp/doc/man6/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/tftp/doc/man6/.gitignore diff --git a/lib/tftp/doc/pdf/.gitignore b/lib/tftp/doc/pdf/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/tftp/doc/pdf/.gitignore diff --git a/lib/tftp/doc/src/Makefile b/lib/tftp/doc/src/Makefile new file mode 100644 index 0000000000..a2fdcf6325 --- /dev/null +++ b/lib/tftp/doc/src/Makefile @@ -0,0 +1,154 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 1997-2018. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# %CopyrightEnd% +# +# +include $(ERL_TOP)/make/target.mk +include $(ERL_TOP)/make/$(TARGET)/otp.mk + +# ---------------------------------------------------- +# Application version +# ---------------------------------------------------- +include ../../vsn.mk +VSN=$(TFTP_VSN) + +# ---------------------------------------------------- +# Release directory specification +# ---------------------------------------------------- +RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN) + + +# ---------------------------------------------------- +# Target Specs +# ---------------------------------------------------- +XML_APPLICATION_FILES = ref_man.xml + +XML_CHAPTER_FILES = \ + introduction.xml \ + getting_started.xml \ + notes.xml + +XML_REF3_FILES = \ + tftp.xml + +XML_PART_FILES = \ + usersguide.xml + +BOOK_FILES = book.xml + +XML_FILES = \ + $(BOOK_FILES) \ + $(XML_CHAPTER_FILES) \ + $(XML_PART_FILES) \ + $(XML_REF6_FILES) \ + $(XML_REF3_FILES) \ + $(XML_APPLICATION_FILES) + +# GIF_FILES = tftp.gif + + +# ---------------------------------------------------- + +HTML_FILES = \ + $(XML_APPLICATION_FILES:%.xml=$(HTMLDIR)/%.html) \ + $(XML_PART_FILES:%.xml=$(HTMLDIR)/%.html) + +INFO_FILE = ../../info +EXTRA_FILES = \ + $(XML_REF3_FILES:%.xml=$(HTMLDIR)/%.html) \ + $(XML_REF6_FILES:%.xml=$(HTMLDIR)/%.html) \ + $(XML_CHAPTER_FILES:%.xml=$(HTMLDIR)/%.html) + +MAN3_FILES = $(XML_REF3_FILES:%.xml=$(MAN3DIR)/%.3) + +HTML_REF_MAN_FILE = $(HTMLDIR)/index.html + +TOP_PDF_FILE = $(PDFDIR)/$(APPLICATION)-$(VSN).pdf + +# ---------------------------------------------------- +# FLAGS +# ---------------------------------------------------- +XML_FLAGS += +DVIPS_FLAGS += + +# ---------------------------------------------------- +# Targets +# ---------------------------------------------------- +$(HTMLDIR)/%.gif: %.gif + $(INSTALL_DATA) $< $@ + +docs: pdf html man + +ldocs: local_docs + +$(TOP_PDF_FILE): $(XML_FILES) + +pdf: $(TOP_PDF_FILE) + +html: gifs $(HTML_REF_MAN_FILE) + +clean clean_docs: clean_html clean_man clean_pdf + rm -f errs core *~ + +man: $(MAN3_FILES) + +gifs: $(GIF_FILES:%=$(HTMLDIR)/%) + +debug opt: + +clean_pdf: + rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) + +clean_html: + rm -rf $(TOP_HTML_FILES) $(HTMLDIR)/* + +clean_man: + rm -f $(MAN3_FILES) + + +# ---------------------------------------------------- +# Release Target +# ---------------------------------------------------- +include $(ERL_TOP)/make/otp_release_targets.mk + +release_docs_spec: docs + $(INSTALL_DIR) "$(RELSYSDIR)/doc/pdf" + $(INSTALL_DATA) $(TOP_PDF_FILE) "$(RELSYSDIR)/doc/pdf" + $(INSTALL_DIR) "$(RELSYSDIR)/doc/html" + $(INSTALL_DATA) $(HTMLDIR)/* "$(RELSYSDIR)/doc/html" + $(INSTALL_DATA) $(INFO_FILE) "$(RELSYSDIR)" + $(INSTALL_DIR) "$(RELEASE_PATH)/man/man3" + $(INSTALL_DATA) $(MAN3DIR)/* "$(RELEASE_PATH)/man/man3" + +release_spec: + +info: + @echo "GIF_FILES:\n$(GIF_FILES)" + @echo "" + @echo "EXTRA_FILES:\n$(EXTRA_FILES)" + @echo "" + @echo "HTML_FILES:\n$(HTML_FILES)" + @echo "" + @echo "TOP_HTML_FILES:\n$(TOP_HTML_FILES)" + @echo "" + @echo "XML_REF3_FILES:\n$(XML_REF3_FILES)" + @echo "" + @echo "XML_REF6_FILES:\n$(XML_REF6_FILES)" + @echo "" + @echo "XML_CHAPTER_FILES:\n$(XML_CHAPTER_FILES)" + @echo "" diff --git a/lib/tftp/doc/src/book.xml b/lib/tftp/doc/src/book.xml new file mode 100644 index 0000000000..c0b551d517 --- /dev/null +++ b/lib/tftp/doc/src/book.xml @@ -0,0 +1,49 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE book SYSTEM "book.dtd"> + +<book xmlns:xi="http://www.w3.org/2001/XInclude"> + <header titlestyle="normal"> + <copyright> + <year>1997</year><year>2018</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>TFTP</title> + <prepared>Péter Dimitrov</prepared> + <docno></docno> + <date>2018-03-22</date> + <rev>1.0</rev> + <file>book.sgml</file> + </header> + <insidecover> + </insidecover> + <pagetext>TFTP</pagetext> + <preamble> + <contents level="2"></contents> + </preamble> + <parts lift="no"> + <xi:include href="usersguide.xml"/> + </parts> + <applications> + <xi:include href="ref_man.xml"/> + </applications> + <releasenotes> + <xi:include href="notes.xml"/> + </releasenotes> + <listofterms></listofterms> + <index></index> +</book> diff --git a/lib/tftp/doc/src/getting_started.xml b/lib/tftp/doc/src/getting_started.xml new file mode 100644 index 0000000000..9bce52dbe0 --- /dev/null +++ b/lib/tftp/doc/src/getting_started.xml @@ -0,0 +1,81 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>1997</year> + <year>2018</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>Getting Started</title> + <prepared></prepared> + <docno></docno> + <approved></approved> + <date></date> + <rev></rev> + <file>getting_started.xml</file> + </header> + + <section> + <title>General Information</title> + <p>The <seealso marker="tftp#start/1">start/1</seealso> function starts + a daemon process listening for UDP packets on a port. When it + receives a request for read or write, it spawns a temporary server + process handling the transfer.</p> + <p>On the client side, + function <seealso marker="tftp#read_file/3">read_file/3</seealso> + and <seealso marker="tftp#write_file/3">write_file/3</seealso> + spawn a temporary client process establishing + contact with a TFTP daemon and perform the file transfer.</p> + <p><c>tftp</c> uses a callback module to handle the file + transfer. Two such callback modules are provided, + <c>tftp_binary</c> and <c>tftp_file</c>. See + <seealso marker="tftp#read_file/3">read_file/3</seealso> and + <seealso marker="tftp#write_file/3">write_file/3</seealso> for details. + You can also implement your own callback modules, see + <seealso marker="tftp#tftp_callback">CALLBACK FUNCTIONS</seealso>. + A callback module provided by + the user is registered using option <c>callback</c>, see + <seealso marker="tftp#options">DATA TYPES</seealso>.</p> + </section> + + <section> + <title>Using the TFTP client and server</title> + <p>This is a simple example of starting the TFTP server and reading the content + of a sample file using the TFTP client.</p> + + <p><em>Step 1.</em> Create a sample file to be used for the transfer:</p> + <code> + $ echo "Erlang/OTP 21" > file.txt + </code> + + <p><em>Step 2.</em> Start the TFTP server:</p> + <code type="erl" > + 1> {ok, Pid} = tftp:start([{port, 19999}]). + <![CDATA[{ok,<0.65.0>}]]> + </code> + + <p><em>Step 3.</em> Start the TFTP client (in another shell):</p> + <code type="erl" > + 1> tftp:read_file("file.txt", binary, [{port, 19999}]). + <![CDATA[{ok,<<"Erlang/OTP 21\n">>}]]> + </code> + </section> + +</chapter> diff --git a/lib/tftp/doc/src/introduction.xml b/lib/tftp/doc/src/introduction.xml new file mode 100644 index 0000000000..70761db0dc --- /dev/null +++ b/lib/tftp/doc/src/introduction.xml @@ -0,0 +1,62 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>1997</year><year>2018</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + </legalnotice> + + <title>Introduction</title> + <prepared>Péter Dimitrov</prepared> + <responsible></responsible> + <docno></docno> + <approved></approved> + <checked></checked> + <date>2018-03-22</date> + <rev>A</rev> + <file>introduction.xml</file> + </header> + + <section> + <title>Purpose</title> + <p>The Trivial File Transfer Protocol or TFTP is a very simple protocol + used to transfer files.</p> + <p>It has been implemented on top of the User Datagram protocol (UDP) so + it may be used to move files between machines on different networks + implementing UDP. It is designed to be small and easy to implement. + Therefore, it lacks most of the features of a regular FTP. The only + thing it can do is read and write files (or mail) from/to a remote server. + It cannot list directories, and currently has no provisions for user + authentication.</p> + <p>The <c>tftp</c> application implements the following IETF standards:</p> + <list type="bulleted"> + <item>RFC 1350, The TFTP Protocol (revision 2)</item> + <item>RFC 2347, TFTP Option Extension</item> + <item>RFC 2348, TFTP Blocksize Option</item> + <item>RFC 2349, TFTP Timeout Interval and Transfer Size Options</item> + </list> + <p>The only feature that not is implemented is the <c>netascii</c> transfer mode.</p> + </section> + + <section> + <title>Prerequisites</title> + <p>It is assumed that the reader is familiar with the Erlang + programming language, concepts of OTP, and has a basic + understanding of the TFTP protocol.</p> + </section> +</chapter> diff --git a/lib/tftp/doc/src/notes.xml b/lib/tftp/doc/src/notes.xml new file mode 100644 index 0000000000..3a4d97a008 --- /dev/null +++ b/lib/tftp/doc/src/notes.xml @@ -0,0 +1,53 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>2002</year><year>2018</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>TFTP Release Notes</title> + <prepared></prepared> + <responsible></responsible> + <docno></docno> + <approved></approved> + <checked></checked> + <date>2018-03-22</date> + <rev>A</rev> + <file>notes.xml</file> + </header> + + <section><title>TFTP 1.0</title> + + <section><title>First released version</title> + <list> + <item> + <p> + Inets application was split into multiple smaller protocol specific applications. + The TFTP application is a standalone TFTP client and server with the same functionality as + TFTP in Inets.</p> + <p> + Own Id: OTP-14113</p> + </item> + </list> + </section> + +</section> + +</chapter> diff --git a/lib/tftp/doc/src/ref_man.xml b/lib/tftp/doc/src/ref_man.xml new file mode 100644 index 0000000000..41a6cc6d52 --- /dev/null +++ b/lib/tftp/doc/src/ref_man.xml @@ -0,0 +1,36 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE application SYSTEM "application.dtd"> + +<application xmlns:xi="http://www.w3.org/2001/XInclude"> + <header> + <copyright> + <year>1997</year><year>2018</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>TFTP Reference Manual</title> + <prepared>Péter Dimitrov</prepared> + <docno></docno> + <date>2018-03-22</date> + <rev>1.0</rev> + <file>ref_man.xml</file> + </header> + <description> + <p>The <c>TFTP</c> application.</p> + </description> + <xi:include href="tftp.xml"/> +</application> diff --git a/lib/inets/doc/src/tftp.xml b/lib/tftp/doc/src/tftp.xml index 10398f5088..481e5446ad 100644 --- a/lib/inets/doc/src/tftp.xml +++ b/lib/tftp/doc/src/tftp.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2006</year><year>2015</year> + <year>2006</year><year>2018</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -31,57 +31,9 @@ <module>tftp</module> <modulesummary>Trivial FTP.</modulesummary> <description> - <p>This is a complete implementation of the following IETF standards:</p> - <list type="bulleted"> - <item>RFC 1350, The TFTP Protocol (revision 2)</item> - <item>RFC 2347, TFTP Option Extension</item> - <item>RFC 2348, TFTP Blocksize Option</item> - <item>RFC 2349, TFTP Timeout Interval and Transfer Size Options</item> - </list> - <p>The only feature that not is implemented is - the "netascii" transfer mode.</p> - <p>The <seealso marker="#start/1">start/1</seealso> function starts - a daemon process listening for UDP packets on a port. When it - receives a request for read or write, it spawns a temporary server - process handling the transfer.</p> - <p>On the client side, - function <seealso marker="#read_file/3">read_file/3</seealso> - and <seealso marker="#write_file/3">write_file/3</seealso> - spawn a temporary client process establishing - contact with a TFTP daemon and perform the file transfer.</p> - <p><c>tftp</c> uses a callback module to handle the file - transfer. Two such callback modules are provided, - <c>tftp_binary</c> and <c>tftp_file</c>. See - <seealso marker="#read_file/3">read_file/3</seealso> and - <seealso marker="#write_file/3">write_file/3</seealso> for details. - You can also implement your own callback modules, see - <seealso marker="#tftp_callback">CALLBACK FUNCTIONS</seealso>. - A callback module provided by - the user is registered using option <c>callback</c>, see - <seealso marker="#options">DATA TYPES</seealso>.</p> + <p>Interface module for the <c>tftp</c> application.</p> </description> - - <section> - <title>TFTP SERVER SERVICE START/STOP</title> - - <p>A TFTP server can be configured to start statically when starting - the <c>Inets</c> application. Alternatively, it can be started dynamically - (when <c>Inets</c> is already started) by calling the <c>Inets</c> application - API <c>inets:start(tftpd, ServiceConfig)</c> or - <c>inets:start(tftpd, ServiceConfig, How)</c>, - see <seealso marker="inets">inets(3)</seealso> for details. - The <c>ServiceConfig</c> for TFTP is described in - the <seealso marker="#options">DATA TYPES</seealso> - section.</p> - - <p>The TFTP server can be stopped using <c>inets:stop(tftpd, Pid)</c>, - see <seealso marker="inets">inets(3)</seealso> for details.</p> - <p>The TPFT client is of such a temporary nature that it is not - handled as a service in the <c>Inets</c> service framework.</p> - - </section> - <section> <marker id="options"></marker> <title>DATA TYPES</title> diff --git a/lib/tftp/doc/src/usersguide.xml b/lib/tftp/doc/src/usersguide.xml new file mode 100644 index 0000000000..eb7f7d17c3 --- /dev/null +++ b/lib/tftp/doc/src/usersguide.xml @@ -0,0 +1,37 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE part SYSTEM "part.dtd"> + +<part xmlns:xi="http://www.w3.org/2001/XInclude"> + <header> + <copyright> + <year>2004</year><year>2018</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>TFTP User's Guide</title> + <prepared>Péter Dimitrov</prepared> + <docno></docno> + <date>2018-03-22</date> + <rev>A</rev> + <file>part.sgml</file> + </header> + <description> + <p>The <c>TFTP</c> application provides a TFTP client and server.</p> + </description> + <xi:include href="introduction.xml"/> + <xi:include href="getting_started.xml"/> +</part> diff --git a/lib/tftp/ebin/.gitignore b/lib/tftp/ebin/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/tftp/ebin/.gitignore diff --git a/lib/tftp/info b/lib/tftp/info new file mode 100644 index 0000000000..1220454351 --- /dev/null +++ b/lib/tftp/info @@ -0,0 +1,2 @@ +group: comm +short: TFTP application diff --git a/lib/inets/src/tftp/Makefile b/lib/tftp/src/Makefile index 4eaa959cce..ed1551ba04 100644 --- a/lib/inets/src/tftp/Makefile +++ b/lib/tftp/src/Makefile @@ -20,30 +20,27 @@ # include $(ERL_TOP)/make/target.mk -EBIN = ../../ebin include $(ERL_TOP)/make/$(TARGET)/otp.mk # ---------------------------------------------------- # Application version # ---------------------------------------------------- -include ../../vsn.mk - -VSN = $(INETS_VSN) - +include ../vsn.mk +VSN = $(TFTP_VSN) # ---------------------------------------------------- # Release directory specification # ---------------------------------------------------- RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN) - # ---------------------------------------------------- # Target Specs # ---------------------------------------------------- -BEHAVIOUR_MODULES= \ - tftp +BEHAVIOUR_MODULES= MODULES = \ + tftp \ + tftp_app \ tftp_binary \ tftp_engine \ tftp_file \ @@ -61,18 +58,18 @@ TARGET_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) BEHAVIOUR_TARGET_FILES= $(BEHAVIOUR_MODULES:%=$(EBIN)/%.$(EMULATOR)) +APP_FILE= tftp.app +APPUP_FILE= tftp.appup + +APP_SRC= $(APP_FILE).src +APP_TARGET= $(EBIN)/$(APP_FILE) +APPUP_SRC= $(APPUP_FILE).src +APPUP_TARGET= $(EBIN)/$(APPUP_FILE) + # ---------------------------------------------------- # FLAGS # ---------------------------------------------------- -include ../inets_app/inets.mk - -ERL_COMPILE_FLAGS += \ - $(INETS_FLAGS) \ - $(INETS_ERL_COMPILE_FLAGS) \ - -I../../include \ - -I../inets_app - # ---------------------------------------------------- # Targets @@ -80,12 +77,18 @@ ERL_COMPILE_FLAGS += \ $(TARGET_FILES): $(BEHAVIOUR_TARGET_FILES) -debug opt: $(TARGET_FILES) +debug opt: $(TARGET_FILES) $(APP_TARGET) $(APPUP_TARGET) clean: - rm -f $(TARGET_FILES) $(BEHAVIOUR_TARGET_FILES) + rm -f $(TARGET_FILES) $(APP_TARGET) $(APPUP_TARGET) $(BEHAVIOUR_TARGET_FILES) rm -f core +$(APP_TARGET): $(APP_SRC) ../vsn.mk + $(vsn_verbose)sed -e 's;%VSN%;$(VSN);' $< > $@ + +$(APPUP_TARGET): $(APPUP_SRC) ../vsn.mk + $(vsn_verbose)sed -e 's;%VSN%;$(VSN);' $< > $@ + docs: # ---------------------------------------------------- @@ -94,16 +97,14 @@ docs: include $(ERL_TOP)/make/otp_release_targets.mk release_spec: opt - $(INSTALL_DIR) "$(RELSYSDIR)/src" - $(INSTALL_DIR) "$(RELSYSDIR)/src/tftp" - $(INSTALL_DATA) $(HRL_FILES) $(ERL_FILES) "$(RELSYSDIR)/src/tftp" - $(INSTALL_DIR) "$(RELSYSDIR)/ebin" - $(INSTALL_DATA) $(TARGET_FILES) $(BEHAVIOUR_TARGET_FILES) "$(RELSYSDIR)/ebin" + $(INSTALL_DIR) "$(RELSYSDIR)/src" + $(INSTALL_DATA) $(ERL_FILES) $(HRL_FILES) "$(RELSYSDIR)/src" + $(INSTALL_DIR) "$(RELSYSDIR)/ebin" + $(INSTALL_DATA) $(BEHAVIOUR_TARGET_FILES) $(TARGET_FILES) $(APP_TARGET) \ + $(APPUP_TARGET) "$(RELSYSDIR)/ebin" release_docs_spec: info: @echo "APPLICATION = $(APPLICATION)" - @echo "INETS_DEBUG = $(INETS_DEBUG)" - @echo "INETS_FLAGS = $(INETS_FLAGS)" @echo "ERL_COMPILE_FLAGS = $(ERL_COMPILE_FLAGS)" diff --git a/lib/tftp/src/tftp.app.src b/lib/tftp/src/tftp.app.src new file mode 100644 index 0000000000..2a87d39ada --- /dev/null +++ b/lib/tftp/src/tftp.app.src @@ -0,0 +1,22 @@ +{application, tftp, + [{description, "TFTP application"}, + {vsn, "1.0"}, + {registered, []}, + {mod, { tftp_app, []}}, + {applications, + [kernel, + stdlib + ]}, + {env,[]}, + {modules, [ + tftp, + tftp_app, + tftp_binary, + tftp_engine, + tftp_file, + tftp_lib, + tftp_logger, + tftp_sup + ]}, + {runtime_dependencies, ["stdlib-3.5","kernel-6.0"]} + ]}. diff --git a/lib/tftp/src/tftp.appup.src b/lib/tftp/src/tftp.appup.src new file mode 100644 index 0000000000..06a0f0f9dc --- /dev/null +++ b/lib/tftp/src/tftp.appup.src @@ -0,0 +1,26 @@ +%% -*- erlang -*- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1999-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +{"%VSN%", + [ + {<<".*">>,[{restart_application, tftp}]} + ], + [ + {<<".*">>,[{restart_application, tftp}]} + ] +}. diff --git a/lib/inets/src/tftp/tftp.erl b/lib/tftp/src/tftp.erl index c8804ea55f..27ed13694b 100644 --- a/lib/inets/src/tftp/tftp.erl +++ b/lib/tftp/src/tftp.erl @@ -213,7 +213,8 @@ start/1, info/1, change_config/2, - start/0 + start/0, + stop/0 ]). %% Application local functions @@ -373,7 +374,17 @@ change_config(Pid, Options) -> %%------------------------------------------------------------------- start() -> - application:start(inets). + application:start(tftp). + +%%------------------------------------------------------------------- +%% stop() -> ok | {error, Reason} +%% +%% Reason = term() +%% +%% Stop the application +%%------------------------------------------------------------------- +stop() -> + application:stop(tftp). %%------------------------------------------------------------------- %% Inets service behavior diff --git a/lib/inets/src/tftp/tftp.hrl b/lib/tftp/src/tftp.hrl index 25543e0b9e..25543e0b9e 100644 --- a/lib/inets/src/tftp/tftp.hrl +++ b/lib/tftp/src/tftp.hrl diff --git a/lib/tftp/src/tftp_app.erl b/lib/tftp/src/tftp_app.erl new file mode 100644 index 0000000000..80d54c6cbe --- /dev/null +++ b/lib/tftp/src/tftp_app.erl @@ -0,0 +1,56 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2002-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% + +%%%------------------------------------------------------------------- +%% @doc ftp public API +%% @end +%%%------------------------------------------------------------------- + +-module(tftp_app). + +-behaviour(application). + +%% Application callbacks +-export([start/2, stop/1]). + +%%==================================================================== +%% API +%%==================================================================== + +start(_StartType, _StartArgs) -> + Config = get_configuration(), + tftp_sup:start_link(Config). + +%%-------------------------------------------------------------------- +stop(_State) -> + ok. + +%%==================================================================== +%% Internal functions +%%==================================================================== + +get_configuration() -> + case (catch application:get_env(tftp, services)) of + {ok, Services} -> + Services; + _ -> + [] + end. diff --git a/lib/inets/src/tftp/tftp_binary.erl b/lib/tftp/src/tftp_binary.erl index 09adcfc41f..09adcfc41f 100644 --- a/lib/inets/src/tftp/tftp_binary.erl +++ b/lib/tftp/src/tftp_binary.erl diff --git a/lib/inets/src/tftp/tftp_engine.erl b/lib/tftp/src/tftp_engine.erl index fb2c9749e5..f14354ad6a 100644 --- a/lib/inets/src/tftp/tftp_engine.erl +++ b/lib/tftp/src/tftp_engine.erl @@ -203,19 +203,19 @@ daemon_loop(#daemon_state{config = DaemonConfig, Fun = fun(#server_info{pid = Pid}, Acc) -> [{server, Pid} | Acc] end, ServerInfo = ets:foldl(Fun, [], ServerTab), Info = internal_info(DaemonConfig, daemon) ++ [{n_conn, N}] ++ ServerInfo, - reply({ok, Info}, Ref, FromPid), + _ = reply({ok, Info}, Ref, FromPid), ?MODULE:daemon_loop(State); {{change_config, Options}, Ref, FromPid} when is_pid(FromPid) -> case catch tftp_lib:parse_config(Options, DaemonConfig) of {'EXIT', Reason} -> - reply({error, Reason}, Ref, FromPid), + _ = reply({error, Reason}, Ref, FromPid), ?MODULE:daemon_loop(State); DaemonConfig2 when is_record(DaemonConfig2, config) -> - reply(ok, Ref, FromPid), + _ = reply(ok, Ref, FromPid), ?MODULE:daemon_loop(State#daemon_state{config = DaemonConfig2}) end; {udp, Socket, RemoteHost, RemotePort, Bin} when is_binary(Bin) -> - inet:setopts(Socket, [{active, once}]), + _ = inet:setopts(Socket, [{active, once}]), ServerConfig = DaemonConfig#config{parent_pid = self(), udp_host = RemoteHost, udp_port = RemotePort}, @@ -449,14 +449,14 @@ client_prepare(Config, Callback, Req) when is_record(Req, tftp_msg_req) -> transfer(Config2, Callback2, Req2, Req2, LocalAccess, BlockNo, #prepared{}), client_open(Config3, Callback3, Req2, BlockNo, TransferRes); {error, {Code, Text}} -> - callback({abort, {Code, Text}}, Config, Callback2, Req), + _ = callback({abort, {Code, Text}}, Config, Callback2, Req), terminate(Config, Req, ?ERROR(post_verify_options, Code, Text, Req#tftp_msg_req.filename)) end; {undefined, #tftp_msg_error{code = Code, text = Text}} -> terminate(Config, Req, ?ERROR(client_prepare, Code, Text, Req#tftp_msg_req.filename)) end; {error, {Code, Text}} -> - callback({abort, {Code, Text}}, Config, Callback, Req), + _ = callback({abort, {Code, Text}}, Config, Callback, Req), terminate(Config, Req, ?ERROR(pre_verify_options, Code, Text, Req#tftp_msg_req.filename)) end. @@ -500,10 +500,10 @@ client_open(Config, Callback, Req, BlockNo, #transfer_res{status = Status, decod %% Req2 = Req#tftp_msg_req{options = []}, %% client_prepare(Config, Callback, Req2); #tftp_msg_error{code = Code, text = Text} -> - callback({abort, {Code, Text}}, Config, Callback, Req), + _ = callback({abort, {Code, Text}}, Config, Callback, Req), terminate(Config, Req, ?ERROR(client_open, Code, Text, Req#tftp_msg_req.filename)); {'EXIT', #tftp_msg_error{code = Code, text = Text}} -> - callback({abort, {Code, Text}}, Config, Callback, Req), + _ = callback({abort, {Code, Text}}, Config, Callback, Req), terminate(Config, Req, ?ERROR(client_open, Code, Text, Req#tftp_msg_req.filename)); Msg when is_tuple(Msg) -> Code = badop, @@ -516,7 +516,7 @@ client_open(Config, Callback, Req, BlockNo, #transfer_res{status = Status, decod end; error when is_record(Prepared, tftp_msg_error) -> #tftp_msg_error{code = Code, text = Text} = Prepared, - callback({abort, {Code, Text}}, Config, Callback, Req), + _ = callback({abort, {Code, Text}}, Config, Callback, Req), terminate(Config, Req, ?ERROR(client_open, Code, Text, Req#tftp_msg_req.filename)) end. @@ -568,10 +568,10 @@ common_loop(Config, Callback, Req, #transfer_res{status = Status, decoded_msg = #tftp_msg_data{block_no = ActualBlockNo, data = Data} when LocalAccess =:= write -> common_write(Config, Callback, Req, LocalAccess, ExpectedBlockNo, ActualBlockNo, Data, Prepared); #tftp_msg_error{code = Code, text = Text} -> - callback({abort, {Code, Text}}, Config, Callback, Req), + _ = callback({abort, {Code, Text}}, Config, Callback, Req), terminate(Config, Req, ?ERROR(common_loop, Code, Text, Req#tftp_msg_req.filename)); {'EXIT', #tftp_msg_error{code = Code, text = Text} = Error} -> - callback({abort, {Code, Text}}, Config, Callback, Req), + _ = callback({abort, {Code, Text}}, Config, Callback, Req), send_msg(Config, Req, Error), terminate(Config, Req, ?ERROR(common_loop, Code, Text, Req#tftp_msg_req.filename)); Msg when is_tuple(Msg) -> @@ -918,7 +918,7 @@ wait_for_msg(Config, Callback, Req) -> {udp, Socket, RemoteHost, RemotePort, Bin} when is_binary(Bin), Callback#callback.block_no =:= undefined -> %% Client prepare - inet:setopts(Socket, [{active, once}]), + _ = inet:setopts(Socket, [{active, once}]), Config2 = Config#config{udp_host = RemoteHost, udp_port = RemotePort}, DecodedMsg = (catch tftp_lib:decode_msg(Bin)), @@ -927,7 +927,7 @@ wait_for_msg(Config, Callback, Req) -> {udp, Socket, Host, Port, Bin} when is_binary(Bin), Config#config.udp_host =:= Host, Config#config.udp_port =:= Port -> - inet:setopts(Socket, [{active, once}]), + _ = inet:setopts(Socket, [{active, once}]), DecodedMsg = (catch tftp_lib:decode_msg(Bin)), print_debug_info(Config, Req, recv, DecodedMsg), {Config, DecodedMsg}; @@ -938,15 +938,15 @@ wait_for_msg(Config, Callback, Req) -> false -> server end, Info = internal_info(Config, Type), - reply({ok, Info}, Ref, FromPid), + _ = reply({ok, Info}, Ref, FromPid), wait_for_msg(Config, Callback, Req); {{change_config, Options}, Ref, FromPid} when is_pid(FromPid) -> case catch tftp_lib:parse_config(Options, Config) of {'EXIT', Reason} -> - reply({error, Reason}, Ref, FromPid), + _ = reply({error, Reason}, Ref, FromPid), wait_for_msg(Config, Callback, Req); Config2 when is_record(Config2, config) -> - reply(ok, Ref, FromPid), + _ = reply(ok, Ref, FromPid), wait_for_msg(Config2, Callback, Req) end; {system, From, Msg} -> @@ -1076,7 +1076,7 @@ do_callback({open, Type}, Config, Callback, Req) Req#tftp_msg_req.options, Callback#callback.state], PeerInfo = peer_info(Config), - fast_ensure_loaded(Mod), + _ = fast_ensure_loaded(Mod), Args2 = case erlang:function_exported(Mod, Fun, length(Args)) of true -> Args; @@ -1295,7 +1295,7 @@ info_msg(#config{logger = Logger}, F, A) -> safe_apply(Logger, info_msg, [F, A]). safe_apply(Mod, Fun, Args) -> - fast_ensure_loaded(Mod), + _ = fast_ensure_loaded(Mod), apply(Mod, Fun, Args). fast_ensure_loaded(Mod) -> diff --git a/lib/inets/src/tftp/tftp_file.erl b/lib/tftp/src/tftp_file.erl index 7664324808..43b588f71a 100644 --- a/lib/inets/src/tftp/tftp_file.erl +++ b/lib/tftp/src/tftp_file.erl @@ -215,13 +215,13 @@ read(#state{access = read} = State) -> Count = State#state.count + size(Bin), {more, Bin, State#state{count = Count}}; {ok, Bin} when is_binary(Bin), size(Bin) < BlkSize -> - file:close(State#state.fd), + _ = file:close(State#state.fd), Count = State#state.count + size(Bin), {last, Bin, Count}; eof -> {last, <<>>, State#state.count}; {error, Reason} -> - file:close(State#state.fd), + _ = file:close(State#state.fd), {error, file_error(Reason)} end; read(State) -> @@ -255,12 +255,12 @@ write(Bin, #state{access = write} = State) when is_binary(Bin) -> Count = State#state.count + Size, {more, State#state{count = Count}}; ok when Size < BlkSize-> - file:close(State#state.fd), + _ = file:close(State#state.fd), Count = State#state.count + Size, {last, Count}; {error, Reason} -> - file:close(State#state.fd), - file:delete(State#state.filename), + _ = file:close(State#state.fd), + _ = file:delete(State#state.filename), {error, file_error(Reason)} end; write(Bin, State) -> @@ -281,7 +281,7 @@ write(Bin, State) -> %%------------------------------------------------------------------- abort(_Code, _Text, #state{fd = Fd, access = Access} = State) -> - file:close(Fd), + _ = file:close(Fd), case Access of write -> ok = file:delete(State#state.filename); diff --git a/lib/inets/src/tftp/tftp_lib.erl b/lib/tftp/src/tftp_lib.erl index 454754f0a3..454754f0a3 100644 --- a/lib/inets/src/tftp/tftp_lib.erl +++ b/lib/tftp/src/tftp_lib.erl diff --git a/lib/inets/src/tftp/tftp_logger.erl b/lib/tftp/src/tftp_logger.erl index a869958484..a869958484 100644 --- a/lib/inets/src/tftp/tftp_logger.erl +++ b/lib/tftp/src/tftp_logger.erl diff --git a/lib/inets/src/tftp/tftp_sup.erl b/lib/tftp/src/tftp_sup.erl index 40b67c499c..0475e53e42 100644 --- a/lib/inets/src/tftp/tftp_sup.erl +++ b/lib/tftp/src/tftp_sup.erl @@ -19,7 +19,7 @@ %% %% %%---------------------------------------------------------------------- -%% Purpose: The top supervisor for tftp hangs under inets_sup. +%% Purpose: The top supervisor for tftp %%---------------------------------------------------------------------- -module(tftp_sup). diff --git a/lib/tftp/test/Makefile b/lib/tftp/test/Makefile new file mode 100644 index 0000000000..99f36256b0 --- /dev/null +++ b/lib/tftp/test/Makefile @@ -0,0 +1,250 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 1997-2018. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# %CopyrightEnd% +# +# +# For an outline of how this all_SUITE_data stuff works, see the +# make file ../../ssl/test/Makefile. +# +include $(ERL_TOP)/make/target.mk +include $(ERL_TOP)/make/$(TARGET)/otp.mk + +# ---------------------------------------------------- +# Application version +# ---------------------------------------------------- +include ../vsn.mk +VSN = $(TFTP_VSN) + + +# ---------------------------------------------------- +# Release directory specification +# ---------------------------------------------------- +RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN) + + +# ---------------------------------------------------- +# Target Specs +# ---------------------------------------------------- +INCLUDES = -I. \ + -I$(ERL_TOP)/lib/tftp/src + +CP = cp + +ifeq ($(TESTROOT_DIR),) +TESTROOT_DIR = /ldisk/tests/$(USER)/tftp +endif + +ifeq ($(TFTP_DATA_DIR),) +TFTP_DATA_DIR = $(TESTROOT_DIR)/data_dir +endif + +ifeq ($(TFTP_PRIV_DIR),) +TFTP_PRIV_DIR = $(TESTROOT_DIR)/priv_dir +endif + +TFTP_FLAGS = -Dtftp__data_dir='"$(TFTP_DATA_DIR)"' \ + -Dtftp_priv_dir='"$(TFTP_PRIV_DIR)"' + + +### +### test suite debug flags +### +ifeq ($(TFTP_DEBUG_CLIENT),) + TFTP_DEBUG_CLIENT = y +endif + +ifeq ($(TFTP_DEBUG_CLIENT),) + TFTP_FLAGS += -Dtftp_debug_client +endif + +ifeq ($(TFTP_TRACE_CLIENT),) + TFTP_DEBUG_CLIENT = y +endif + +ifeq ($(TFTP_TRACE_CLIENT),y) + TFTP_FLAGS += -Dtftp_trace_client +endif + +ifneq ($(TFTP_DEBUG),) + TFTP_DEBUG = s +endif + +ifeq ($(TFTP_DEBUG),l) + TFTP_FLAGS += -Dtftp_log +endif + +ifeq ($(TFTP_DEBUG),d) + TFTP_FLAGS += -Dtftp_debug -Dtftp_log +endif + + +TFTP_FLAGS += -pa ../tftp/ebin + +TFTP_ROOT = ../tftp + +MODULES = \ + tftp_SUITE \ + tftp_test_lib + + +EBIN = . + +HRL_FILES = \ + ../src/tftp.hrl \ + tftp_test_lib.hrl + +ERL_FILES = $(MODULES:%=%.erl) + +SOURCE = $(ERL_FILES) $(HRL_FILES) + +TARGET_FILES = $(MODULES:%=$(EBIN)/%.$(EMULATOR)) + +TFTP_SPECS = tftp.spec tftp_bench.spec +COVER_FILE = tftp.cover +TFTP_FILES = tftp.config $(TFTP_SPECS) + + +TFTP_DATADIRS = tftp_SUITE_data + +DATADIRS = $(TFTP_DATADIRS) + +EMAKEFILE = Emakefile +MAKE_EMAKE = $(wildcard $(ERL_TOP)/make/make_emakefile) + +ifeq ($(MAKE_EMAKE),) +BUILDTARGET = $(TARGET_FILES) +RELTEST_FILES = $(COVER_FILE) $(TFTP_SPECS) $(SOURCE) +else +BUILDTARGET = emakebuild +RELTEST_FILES = $(EMAKEFILE) $(COVER_FILE) $(TFTP_SPECS) $(SOURCE) +endif + + +# ---------------------------------------------------- +# Release directory specification +# ---------------------------------------------------- + +RELTESTSYSDIR = "$(RELEASE_PATH)/tftp_test" +RELTESTSYSALLDATADIR = $(RELTESTSYSDIR)/all_SUITE_data +RELTESTSYSBINDIR = $(RELTESTSYSALLDATADIR)/bin + + +# ---------------------------------------------------- +# FLAGS +# The path to the test_server ebin dir is needed when +# running the target "targets". +# ---------------------------------------------------- +ERL_COMPILE_FLAGS += \ + $(INCLUDES) \ + $(TFTP_FLAGS) + +# ---------------------------------------------------- +# Targets +# erl -sname kalle -pa ../ebin +# If you intend to run the test suite locally (private), then +# there is some requirements: +# 1) TFTP_PRIV_DIR must be created +# ---------------------------------------------------- + +tests debug opt: $(BUILDTARGET) + +targets: $(TARGET_FILES) + +.PHONY: emakebuild + +emakebuild: $(EMAKEFILE) + +$(EMAKEFILE): + $(MAKE_EMAKE) $(ERL_COMPILE_FLAGS) -o$(EBIN) '*_SUITE_make' | grep -v Warning > $(EMAKEFILE) + $(MAKE_EMAKE) $(ERL_COMPILE_FLAGS) -o$(EBIN) $(MODULES) | grep -v Warning >> $(EMAKEFILE) + +clean: + rm -f $(EMAKEFILE) + rm -f $(TARGET_FILES) + rm -f core *~ + +docs: + + +# ---------------------------------------------------- +# Release Target +# ---------------------------------------------------- +include $(ERL_TOP)/make/otp_release_targets.mk + +release_spec: opt + $(INSTALL_DIR) "$(RELSYSDIR)/test" + $(INSTALL_DATA) $(HRL_FILES) $(ERL_FILES) "$(RELSYSDIR)/test" + $(INSTALL_DATA) $(TFTP_FILES) "$(RELSYSDIR)/test" + @for d in $(DATADIRS); do \ + echo "installing data dir $$d"; \ + if test -f $$d/TAR.exclude; then \ + echo $$d/TAR.exclude2 > $$d/TAR.exclude2; \ + cat $$d/TAR.exclude >> $$d/TAR.exclude2; \ + find $$d -name '*.contrib*' >> $$d/TAR.exclude2; \ + find $$d -name '*.keep*' >> $$d/TAR.exclude2; \ + find $$d -name '*.mkelem*' >> $$d/TAR.exclude2; \ + find $$d -name '*~' >> $$d/TAR.exclude2; \ + find $$d -name 'erl_crash.dump' >> $$d/TAR.exclude2; \ + find $$d -name 'core' >> $$d/TAR.exclude2; \ + find $$d -name '.cmake.state' >> $$d/TAR.exclude2; \ + tar cfX - $$d/TAR.exclude2 $$d | (cd "$(RELSYSDIR)/test"; tar xf -); \ + else \ + tar cf - $$d | (cd "$(RELSYSDIR)/test"; tar xf -); \ + fi; \ + done + +release_tests_spec: opt + $(INSTALL_DIR) $(RELTESTSYSDIR) + $(INSTALL_DATA) $(RELTEST_FILES) $(RELTESTSYSDIR) + chmod -R u+w $(RELTESTSYSDIR) + tar chf - $(DATADIRS) | (cd $(RELTESTSYSDIR); tar xf -) + $(INSTALL_DIR) $(RELTESTSYSALLDATADIR) + $(INSTALL_DIR) $(RELTESTSYSBINDIR) + chmod -R +x $(RELTESTSYSBINDIR) + $(INSTALL_DIR) $(RELTESTSYSALLDATADIR)/win32/lib + +release_docs_spec: + +info: + @echo "MAKE_EMAKE = $(MAKE_EMAKE)" + @echo "EMAKEFILE = $(EMAKEFILE)" + @echo "BUILDTARGET = $(BUILDTARGET)" + @echo "" + @echo "MODULES = $(MODULES)" + @echo "ERL_FILES = $(ERL_FILES)" + @echo "SOURCE = $(SOURCE)" + @echo "TARGET_FILES = $(TARGET_FILES)" + @echo "" + @echo "TFTP_SPECS = $(TFTP_SPECS)" + @echo "TFTP_FILES = $(TFTP_FILES)" + @echo "" + @echo "RELEASE_PATH = "$(RELEASE_PATH)"" + @echo "RELSYSDIR = "$(RELSYSDIR)"" + @echo "RELTESTSYSDIR = $(RELTESTSYSDIR)" + @echo "RELTESTSYSALLDATADIR = $(RELTESTSYSALLDATADIR)" + @echo "RELTESTSYSBINDIR = $(RELTESTSYSBINDIR)" + @echo "" + @echo "DATADIRS = $(DATADIRS)" + @echo "REL_DATADIRS = $(REL_DATADIRS)" + @echo "" + @echo "TFTP_DATA_DIR = $(TFTP_DATA_DIR)" + @echo "TFTP_PRIV_DIR = $(TFTP_PRIV_DIR)" + @echo "TFTP_ROOT = $(TFTP_ROOT)" + @echo "TFTP_FLAGS = $(TFTP_FLAGS)" + + diff --git a/lib/tftp/test/tftp.config b/lib/tftp/test/tftp.config new file mode 100644 index 0000000000..2600237da9 --- /dev/null +++ b/lib/tftp/test/tftp.config @@ -0,0 +1 @@ +[].
\ No newline at end of file diff --git a/lib/tftp/test/tftp.cover b/lib/tftp/test/tftp.cover new file mode 100644 index 0000000000..22ef5d0dda --- /dev/null +++ b/lib/tftp/test/tftp.cover @@ -0,0 +1,2 @@ +{incl_app,tftp,details}. + diff --git a/lib/tftp/test/tftp.spec b/lib/tftp/test/tftp.spec new file mode 100644 index 0000000000..f3537bc652 --- /dev/null +++ b/lib/tftp/test/tftp.spec @@ -0,0 +1 @@ +{suites,"../tftp_test", all}. diff --git a/lib/inets/test/tftp_SUITE.erl b/lib/tftp/test/tftp_SUITE.erl index 09049e36af..fd1d209c25 100644 --- a/lib/inets/test/tftp_SUITE.erl +++ b/lib/tftp/test/tftp_SUITE.erl @@ -26,21 +26,22 @@ %% Includes and defines %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +-include_lib("common_test/include/ct.hrl"). -include("tftp_test_lib.hrl"). --define(START_DAEMON(PortX, OptionsX), - fun(Port, Options) -> - {ok, Pid} = ?VERIFY({ok, _Pid}, tftp:start([{port, Port} | Options])), - if - Port == 0 -> - {ok, ActualOptions} = ?IGNORE(tftp:info(Pid)), - {value, {port, ActualPort}} = - lists:keysearch(port, 1, ActualOptions), - {ActualPort, Pid}; - true -> - {Port, Pid} - end - end(PortX, OptionsX)). +-define(START_DAEMON(Port, Options), + begin + {ok, Pid} = ?VERIFY({ok, _Pid}, tftp:start([{port, Port} | Options])), + if + Port == 0 -> + {ok, ActualOptions} = ?IGNORE(tftp:info(Pid)), + {value, {port, ActualPort}} = + lists:keysearch(port, 1, ActualOptions), + {ActualPort, Pid}; + true -> + {Port, Pid} + end + end). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% API @@ -74,9 +75,18 @@ end_per_testcase(Case, Config) when is_list(Config) -> suite() -> [{ct_hooks,[ts_install_cth]}]. -all() -> - [simple, extra, reuse_connection, resend_client, - resend_server, large_file]. +all() -> + [ + simple, + extra, + reuse_connection, + resend_client, + resend_server, + large_file, + app, + appup, + start_tftpd + ]. groups() -> []. @@ -94,6 +104,48 @@ end_per_group(_GroupName, Config) -> Config. +app() -> + [{doc, "Test that the tftp app file is ok"}]. +app(Config) when is_list(Config) -> + ok = ?t:app_test(tftp). + +%%-------------------------------------------------------------------- +appup() -> + [{doc, "Test that the tftp appup file is ok"}]. +appup(Config) when is_list(Config) -> + ok = ?t:appup_test(tftp). + +start_tftpd() -> + [{doc, "Start/stop of tfpd service"}]. +start_tftpd(Config) when is_list(Config) -> + process_flag(trap_exit, true), + ok = tftp:start(), + {ok, Pid0} = tftp:start_service([{host, "localhost"}, {port, 0}]), + Pids0 = [ServicePid || {_, ServicePid} <- tftp:services()], + true = lists:member(Pid0, Pids0), + {ok, [_|_]} = tftp:service_info(Pid0), + tftp:stop_service(Pid0), + ct:sleep(100), + Pids1 = [ServicePid || {_, ServicePid} <- tftp:services()], + false = lists:member(Pid0, Pids1), + + {ok, Pid1} = + tftp:start_standalone([{host, "localhost"}, {port, 0}]), + Pids2 = [ServicePid || {_, ServicePid} <- tftp:services()], + false = lists:member(Pid1, Pids2), + %% Standalone service is not supervised + {error,not_found} = tftp:stop_service(Pid1), + ok = tftp:stop(), + + application:load(tftp), + application:set_env(tftp, services, [{tftpd, [{host, "localhost"}, + {port, 0}]}]), + ok = tftp:start(), + 1 = length(tftp:services()), + application:unset_env(tftp, services), + ok = tftp:stop(). + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Simple %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -103,7 +155,7 @@ simple(doc) -> simple(suite) -> []; simple(Config) when is_list(Config) -> - ?VERIFY(ok, application:start(inets)), + ?VERIFY(ok, application:start(tftp)), {Port, DaemonPid} = ?IGNORE(?START_DAEMON(0, [{debug, brief}])), @@ -128,7 +180,7 @@ simple(Config) when is_list(Config) -> exit(DaemonPid, kill), ?VERIFY(ok, file:delete(LocalFilename)), ?VERIFY(ok, file:delete(RemoteFilename)), - ?VERIFY(ok, application:stop(inets)), + ?VERIFY(ok, application:stop(tftp)), ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -910,7 +962,7 @@ large_file(doc) -> large_file(suite) -> []; large_file(Config) when is_list(Config) -> - ?VERIFY(ok, application:start(inets)), + ?VERIFY(ok, application:start(tftp)), {Port, DaemonPid} = ?IGNORE(?START_DAEMON(0, [{debug, brief}])), @@ -933,7 +985,7 @@ large_file(Config) when is_list(Config) -> exit(DaemonPid, kill), ?VERIFY(ok, file:delete(LocalFilename)), ?VERIFY(ok, file:delete(RemoteFilename)), - ?VERIFY(ok, application:stop(inets)), + ?VERIFY(ok, application:stop(tftp)), ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/tftp/test/tftp_bench.spec b/lib/tftp/test/tftp_bench.spec new file mode 100644 index 0000000000..43fa385c85 --- /dev/null +++ b/lib/tftp/test/tftp_bench.spec @@ -0,0 +1 @@ +{suites,"../tftp_test",[]}. diff --git a/lib/inets/test/tftp_test_lib.erl b/lib/tftp/test/tftp_test_lib.erl index f07795324f..45386389cb 100644 --- a/lib/inets/test/tftp_test_lib.erl +++ b/lib/tftp/test/tftp_test_lib.erl @@ -30,11 +30,11 @@ init_per_testcase(_Case, Config) when is_list(Config) -> io:format("\n ", []), - ?IGNORE(application:stop(inets)), + ?IGNORE(application:stop(tftp)), Config. end_per_testcase(_Case, Config) when is_list(Config) -> - ?IGNORE(application:stop(inets)), + ?IGNORE(application:stop(tftp)), Config. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/inets/test/tftp_test_lib.hrl b/lib/tftp/test/tftp_test_lib.hrl index e7a5a37d2c..e7a5a37d2c 100644 --- a/lib/inets/test/tftp_test_lib.hrl +++ b/lib/tftp/test/tftp_test_lib.hrl diff --git a/lib/tftp/vsn.mk b/lib/tftp/vsn.mk new file mode 100644 index 0000000000..f1b0851a8f --- /dev/null +++ b/lib/tftp/vsn.mk @@ -0,0 +1,24 @@ +#-*-makefile-*- ; force emacs to enter makefile-mode + +# %CopyrightBegin% +# +# Copyright Ericsson AB 2001-2018. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# %CopyrightEnd% + +APPLICATION = tftp +TFTP_VSN = 1.0 +PRE_VSN = +APP_VSN = "$(APPLICATION)-$(TFTP_VSN)$(PRE_VSN)" diff --git a/lib/tools/doc/specs/.gitignore b/lib/tools/doc/specs/.gitignore new file mode 100644 index 0000000000..322eebcb06 --- /dev/null +++ b/lib/tools/doc/specs/.gitignore @@ -0,0 +1 @@ +specs_*.xml diff --git a/lib/tools/doc/src/Makefile b/lib/tools/doc/src/Makefile index b554781382..4b663106a0 100644 --- a/lib/tools/doc/src/Makefile +++ b/lib/tools/doc/src/Makefile @@ -84,11 +84,20 @@ HTML_REF_MAN_FILE = $(HTMLDIR)/index.html TOP_PDF_FILE = $(PDFDIR)/$(APPLICATION)-$(VSN).pdf +SPECS_FILES = $(XML_REF3_FILES:%.xml=$(SPECDIR)/specs_%.xml) + +TOP_SPECS_FILE = specs.xml + # ---------------------------------------------------- # FLAGS # ---------------------------------------------------- XML_FLAGS += +TOOLS_SRC=$(ERL_TOP)/lib/tools/src +TOOLS_INCLUDE=$(ERL_TOP)/lib/tools/include + +SPECS_FLAGS = -I$(TOOLS_SRC) -I$(TOOLS_INCLUDE) + # ---------------------------------------------------- # Targets # ---------------------------------------------------- @@ -113,8 +122,13 @@ clean clean_docs: rm -rf $(HTMLDIR)/* rm -f $(MAN3DIR)/* rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) + rm -f $(SPECDIR)/* rm -f errs core *~ +# erlang_mode doesn't have erlang source so we generate a dummy file for it. +$(SPECDIR)/specs_erlang_mode.xml: + echo '<module name="erlang_mode"/>' > $(SPECDIR)/specs_erlang_mode.xml + # ---------------------------------------------------- # Release Target # ---------------------------------------------------- diff --git a/lib/tools/doc/src/instrument.xml b/lib/tools/doc/src/instrument.xml index bb6f9b6100..9fd9332373 100644 --- a/lib/tools/doc/src/instrument.xml +++ b/lib/tools/doc/src/instrument.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1998</year><year>2016</year> + <year>1998</year><year>2018</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -41,387 +41,190 @@ <note> <p>Note that this whole module is experimental, and the representations used as well as the functionality is likely to change in the future.</p> - <p>The <c>instrument</c> module interface was slightly changed in - Erlang/OTP R9C.</p> </note> - <p>To start an Erlang runtime system with instrumentation, use the - <c>+Mi*</c> set of command-line arguments to the <c>erl</c> command (see - the erts_alloc(3) and erl(1) man pages).</p> - <p>The basic object of study in the case of memory allocation is a memory - allocation map. A memory allocation map contains a list of descriptors - for each allocated memory block. Currently, a descriptor is a 4-tuple</p> - <pre> - {TypeNo, Address, Size, PidDesc} </pre> - <p>where <c>TypeNo</c> is the memory block type number, <c>Address</c> - is its place in memory, and <c>Size</c> is its size, in bytes. - <c>PidDesc</c> is either a tuple <c>{X,Y,Z}</c> identifying the - process which was executing when the block was allocated, or - <c>undefined</c> if no process was executing. The pid tuple - <c>{X,Y,Z}</c> can be transformed into a real pid by usage of the - <c>c:pid/3</c> function.</p> - <p>Various details about memory allocation:</p> - <p>Memory blocks are allocated both on the heap segment and on other memory - segments. This can cause the instrumentation functionality to report - very large holes. Currently the instrumentation functionality doesn't - provide any support for distinguishing between holes between memory - segments, and holes between allocated blocks inside memory segments. - The current size of the process cannot be obtained from within Erlang, - but can be seen with one of the system statistics tools, e.g., - <c>ps</c> or <c>top</c>. The Solaris utility <c>pmap</c> can be - useful. It reports currently mapped memory segments. </p> - <p>Overhead for instrumentation: When the emulator has been started with - the <seealso marker="erts:erts_alloc#Mim">"+Mim true"</seealso> - flag, each block is preceded by a 24 bytes large - header on a 32-bit machine and a 48 bytes large header on a 64-bit - machine. When the emulator has been started with the - <seealso marker="erts:erts_alloc#Mis">"+Mis true"</seealso> - flag, each block is preceded by an 8 bytes large header. These are the header - sizes used by the Erlang 5.3/OTP R9C emulator. Other versions of the - emulator may use other header sizes. The function - <seealso marker="#block_header_size/1">block_header_size/1</seealso> - can be used for retrieving the header size used for a specific memory - allocation map. The time overhead for managing the instrumentation - data is small.</p> - <p>Sizes presented by the instrumentation functionality are (by the - emulator) requested sizes, i.e. neither instrumentation headers nor - headers used by allocators are included.</p> </description> + <datatypes> + <datatype> + <name name="block_histogram"/> + <desc> + <p>A histogram of block sizes where each interval's upper bound is + twice as high as the one before it.</p> + <p>The upper bound of the first interval is provided by the function + that returned the histogram, and the last interval has no upper + bound.</p> + </desc> + </datatype> + <datatype> + <name name="allocation_summary"/> + <desc> + <p>A summary of allocated block sizes (including their headers) grouped + by their <c><anno>Origin</anno></c> and <c><anno>Type</anno></c>.</p> + <p><c><anno>Origin</anno></c> is generally which NIF or driver that + allocated the blocks, or 'system' if it could not be determined.</p> + <p><c><anno>Type</anno></c> is the allocation category that the blocks + belong to, e.g. <c>db_term</c>, <c>message</c> or <c>binary</c>.</p> + <p>If one or more carriers could not be scanned in full without harming + the responsiveness of the system, <c><anno>UnscannedSize</anno></c> + is the number of bytes that had to be skipped.</p> + </desc> + </datatype> + <datatype> + <name name="carrier_info_list"/> + <desc> + <p><c><anno>AllocatorType</anno></c> is the type of the allocator that + employs this carrier.</p> + <p><c><anno>TotalSize</anno></c> is the total size of the carrier, + including its header.</p> + <p><c><anno>AllocatedSize</anno></c> is the combined size of the + carrier's allocated blocks, including their headers.</p> + <p><c><anno>AllocatedCount</anno></c> is the number of allocated + blocks in the carrier.</p> + <p><c><anno>InPool</anno></c> is whether the carrier is in the + migration pool.</p> + <p><c><anno>FreeBlocks</anno></c> is a histogram of the free block + sizes in the carrier.</p> + <p>If the carrier could not be scanned in full without harming the + responsiveness of the system, <c><anno>UnscannedSize</anno></c> is + the number of bytes that had to be skipped.</p> + </desc> + </datatype> + </datatypes> <funcs> + <func> - <name>allocator_descr(MemoryData, TypeNo) -> AllocDescr | invalid_type | "unknown"</name> - <fsummary>Returns a allocator description</fsummary> - <type> - <v>MemoryData = {term(), AllocList}</v> - <v>AllocList = [Desc]</v> - <v>Desc = {int(), int(), int(), PidDesc}</v> - <v>PidDesc = {int(), int(), int()} | undefined</v> - <v>TypeNo = int()</v> - <v>AllocDescr = atom() | string()</v> - </type> - <desc> - <p>Returns the allocator description of the allocator that - manages memory blocks of type number <c>TypeNo</c> used in - <c>MemoryData</c>. - Valid <c>TypeNo</c>s are in the range returned by - <seealso marker="#type_no_range/1">type_no_range/1</seealso> on - this specific memory allocation map. If <c>TypeNo</c> is an - invalid integer, <c>invalid_type</c> is returned.</p> - </desc> - </func> - <func> - <name>block_header_size(MemoryData) -> int()</name> - <fsummary>Returns the memory block header size used by the emulator that generated the memory allocation map</fsummary> - <type> - <v>MemoryData = {term(), AllocList}</v> - <v>AllocList = [Desc]</v> - <v>Desc = {int(), int(), int(), PidDesc}</v> - <v>PidDesc = {int(), int(), int()} | undefined</v> - </type> - <desc> - <marker id="block_header_size_1"></marker> - <p>Returns the memory block header size used by the - emulator that generated the memory allocation map. The block - header size may differ between different emulators.</p> - </desc> - </func> - <func> - <name>class_descr(MemoryData, TypeNo) -> ClassDescr | invalid_type | "unknown"</name> - <fsummary>Returns a allocator description</fsummary> - <type> - <v>MemoryData = {term(), AllocList}</v> - <v>AllocList = [Desc]</v> - <v>Desc = {int(), int(), int(), PidDesc}</v> - <v>PidDesc = {int(), int(), int()} | undefined</v> - <v>TypeNo = int()</v> - <v>ClassDescr = atom() | string()</v> - </type> - <desc> - <p>Returns the class description of the class that - the type number <c>TypeNo</c> used in <c>MemoryData</c> belongs - to. - Valid <c>TypeNo</c>s are in the range returned by - <seealso marker="#type_no_range/1">type_no_range/1</seealso> on - this specific memory allocation map. If <c>TypeNo</c> is an - invalid integer, <c>invalid_type</c> is returned.</p> - </desc> - </func> - <func> - <name>descr(MemoryData) -> DescrMemoryData</name> - <fsummary>Replace type numbers in memory allocation map with type descriptions</fsummary> - <type> - <v>MemoryData = {term(), AllocList}</v> - <v>AllocList = [Desc]</v> - <v>Desc = {int(), int(), int(), PidDesc}</v> - <v>PidDesc = {int(), int(), int()} | undefined</v> - <v>DescrMemoryData = {term(), DescrAllocList}</v> - <v>DescrAllocList = [DescrDesc]</v> - <v>DescrDesc = {TypeDescr, int(), int(), DescrPidDesc}</v> - <v>TypeDescr = atom() | string()</v> - <v>DescrPidDesc = pid() | undefined</v> - </type> - <desc> - <p>Returns a memory allocation map where the type numbers (first - element of <c>Desc</c>) have been replaced by type descriptions, - and pid tuples (fourth element of <c>Desc</c>) have been - replaced by real pids.</p> - </desc> - </func> - <func> - <name>holes(MemoryData) -> ok</name> - <fsummary>Print out the sizes of unused memory blocks</fsummary> - <type> - <v>MemoryData = {term(), AllocList}</v> - <v>AllocList = [Desc]</v> - <v>Desc = {int(), int(), int(), PidDesc}</v> - <v>PidDesc = {int(), int(), int()} | undefined</v> - </type> - <desc> - <p>Prints out the size of each hole (i.e., the space between - allocated blocks) on the terminal. <em>NOTE:</em> Really large holes - are probably holes between memory segments. - The memory allocation map has to be sorted (see - <seealso marker="#sort/1">sort/1</seealso>).</p> - </desc> - </func> - <func> - <name>mem_limits(MemoryData) -> {Low, High}</name> - <fsummary>Return lowest and highest memory address used</fsummary> - <type> - <v>MemoryData = {term(), AllocList}</v> - <v>AllocList = [Desc]</v> - <v>Desc = {int(), int(), int(), PidDesc}</v> - <v>PidDesc = {int(), int(), int()} | undefined</v> - <v>Low = High = int()</v> - </type> - <desc> - <p>Returns a tuple <c>{Low, High}</c> indicating - the lowest and highest address used. - The memory allocation map has to be sorted (see - <seealso marker="#sort/1">sort/1</seealso>).</p> - </desc> - </func> - <func> - <name>memory_data() -> MemoryData | false</name> - <fsummary>Return the current memory allocation map</fsummary> - <type> - <v>MemoryData = {term(), AllocList}</v> - <v>AllocList = [Desc]</v> - <v>Desc = {int(), int(), int(), PidDesc}</v> - <v>PidDesc = {int(), int(), int()} | undefined</v> - </type> - <desc> - <p>Returns <c>MemoryData</c> (a the memory allocation map) - if the emulator has been started with the "<c>+Mim true</c>" - command-line argument; otherwise, <c>false</c>. <em>NOTE:</em><c>memory_data/0</c> blocks execution of other processes while - the data is collected. The time it takes to collect the data can - be substantial.</p> - </desc> - </func> - <func> - <name>memory_status(StatusType) -> [StatusInfo] | false</name> - <fsummary>Return current memory allocation status</fsummary> - <type> - <v>StatusType = total | allocators | classes | types</v> - <v>StatusInfo = {About, [Info]}</v> - <v>About = atom()</v> - <v>Info = {InfoName, Current, MaxSinceLast, MaxEver}</v> - <v>InfoName = sizes|blocks</v> - <v>Current = int()</v> - <v>MaxSinceLast = int()</v> - <v>MaxEver = int()</v> - </type> - <desc> - <p>Returns a list of <c>StatusInfo</c> if the emulator has been - started with the "<c>+Mis true</c>" or "<c>+Mim true</c>" - command-line argument; otherwise, <c>false</c>. </p> - <p>See the - <seealso marker="#read_memory_status/1">read_memory_status/1</seealso> - function for a description of the <c>StatusInfo</c> term.</p> - </desc> - </func> - <func> - <name>read_memory_data(File) -> MemoryData | {error, Reason}</name> - <fsummary>Read memory allocation map</fsummary> - <type> - <v>File = string()</v> - <v>MemoryData = {term(), AllocList}</v> - <v>AllocList = [Desc]</v> - <v>Desc = {int(), int(), int(), PidDesc}</v> - <v>PidDesc = {int(), int(), int()} | undefined</v> - </type> + <name name="allocations" arity="0"/> + <fsummary>Return a summary of all allocations in the system.</fsummary> <desc> - <marker id="read_memory_data_1"></marker> - <p>Reads a memory allocation map from the file <c>File</c> and - returns it. The file is assumed to have been created by - <c>store_memory_data/1</c>. The error codes are the same as for - <c>file:consult/1</c>.</p> + <p>Shorthand for + <seealso marker="#allocations/1"><c>allocations(#{})</c>.</seealso></p> </desc> </func> + <func> - <name>read_memory_status(File) -> MemoryStatus | {error, Reason}</name> - <fsummary>Read memory allocation status from a file</fsummary> - <type> - <v>File = string()</v> - <v>MemoryStatus = [{StatusType, [StatusInfo]}]</v> - <v>StatusType = total | allocators | classes | types</v> - <v>StatusInfo = {About, [Info]}</v> - <v>About = atom()</v> - <v>Info = {InfoName, Current, MaxSinceLast, MaxEver}</v> - <v>InfoName = sizes|blocks</v> - <v>Current = int()</v> - <v>MaxSinceLast = int()</v> - <v>MaxEver = int()</v> - </type> - <desc> - <marker id="read_memory_status_1"></marker> - <p>Reads memory allocation status from the file <c>File</c> and - returns it. The file is assumed to have been created by - <c>store_memory_status/1</c>. The error codes are the same as - for <c>file:consult/1</c>.</p> - <p>When <c>StatusType</c> is <c>allocators</c>, <c>About</c> is - the allocator that the information is about. When - <c>StatusType</c> is <c>types</c>, <c>About</c> is - the memory block type that the information is about. Memory - block types are not described other than by their name and may - vary between emulators. When <c>StatusType</c> is <c>classes</c>, - <c>About</c> is the memory block type class that information is - presented about. Memory block types are classified after their - use. Currently the following classes exist:</p> + <name name="allocations" arity="1"/> + <fsummary>Return a summary of all allocations filtered by allocator type + and scheduler id.</fsummary> + <desc> + <p>Returns a summary of all tagged allocations in the system, + optionally filtered by allocator type and scheduler id.</p> + <p>Only binaries and allocations made by NIFs and drivers are tagged by + default, but this can be configured an a per-allocator basis with the + <seealso marker="erts:erts_alloc#M_atags"><c>+M<S>atags</c> + </seealso> emulator option.</p> + <p>If tagged allocations are not enabled on any of the specified + allocator types, the call will fail with + <c>{error, not_enabled}</c>.</p> + <p>The following options can be used:</p> <taglist> - <tag><c>process_data</c></tag> - <item>Erlang process specific data.</item> - <tag><c>binary_data</c></tag> - <item>Erlang binaries.</item> - <tag><c>atom_data</c></tag> - <item>Erlang atoms.</item> - <tag><c>code_data</c></tag> - <item>Erlang code.</item> - <tag><c>system_data</c></tag> - <item>Other data used by the system</item> + <tag><c>allocator_types</c></tag> + <item> + <p>The allocator types that will be searched. Defaults to all + <c>alloc_util</c> allocators.</p> + </item> + <tag><c>scheduler_ids</c></tag> + <item> + <p>The scheduler ids whose allocator instances will be searched. A + scheduler id of 0 will refer to the global instance that is not + tied to any particular scheduler. Defaults to all schedulers and + the global instance.</p> + </item> + <tag><c>histogram_start</c></tag> + <item> + <p>The upper bound of the first interval in the allocated block + size histograms. Defaults to 128.</p> + </item> + <tag><c>histogram_width</c></tag> + <item> + <p>The number of intervals in the allocated block size histograms. + Defaults to 18.</p> + </item> </taglist> - <p>When <c>InfoName</c> is <c>sizes</c>, <c>Current</c>, - <c>MaxSinceLast</c>, and <c>MaxEver</c> are, respectively, current - size, maximum size since last call to - <c>store_memory_status/1</c> or <c>memory_status/1</c> with the - specific <c>StatusType</c>, and maximum size since the emulator - was started. When <c>InfoName</c> is <c>blocks</c>, <c>Current</c>, - <c>MaxSinceLast</c>, and <c>MaxEver</c> are, respectively, current - number of blocks, maximum number of blocks since last call to - <c>store_memory_status/1</c> or <c>memory_status/1</c> with the - specific <c>StatusType</c>, and maximum number of blocks since the - emulator was started. </p> - <p><em>NOTE:</em>A memory block is accounted for at - "the first level" allocator. E.g. <c>fix_alloc</c> allocates its - memory pools via <c>ll_alloc</c>. When a <c>fix_alloc</c> block - is allocated, neither the block nor the pool in which it resides - are accounted for as memory allocated via <c>ll_alloc</c> even - though it is.</p> - </desc> - </func> - <func> - <name>sort(MemoryData) -> MemoryData</name> - <fsummary>Sort the memory allocation list</fsummary> - <type> - <v>MemoryData = {term(), AllocList}</v> - <v>AllocList = [Desc]</v> - <v>Desc = {int(), int(), int(), PidDesc}</v> - <v>PidDesc = {int(), int(), int()} | undefined</v> - </type> - <desc> - <marker id="sort_1"></marker> - <p>Sorts a memory allocation map so that the addresses are in - ascending order.</p> - </desc> - </func> - <func> - <name>store_memory_data(File) -> true|false</name> - <fsummary>Store the current memory allocation map on a file</fsummary> - <type> - <v>File = string()</v> - </type> - <desc> - <p>Stores the current memory allocation map on the file - <c>File</c>. Returns <c>true</c> if the emulator has been - started with the "<c>+Mim true</c>" command-line argument, and - the map was successfully stored; otherwise, <c>false</c>. The - contents of the file can later be read using - <seealso marker="#read_memory_data/1">read_memory_data/1</seealso>. - <em>NOTE:</em><c>store_memory_data/0</c> blocks execution of - other processes while the data is collected. The time it takes - to collect the data can be substantial.</p> - </desc> - </func> - <func> - <name>store_memory_status(File) -> true|false</name> - <fsummary>Store the current memory allocation status on a file</fsummary> - <type> - <v>File = string()</v> - </type> - <desc> - <p>Stores the current memory status on the file - <c>File</c>. Returns <c>true</c> if the emulator has been - started with the "<c>+Mis true</c>", or "<c>+Mim true</c>" - command-line arguments, and the data was successfully stored; - otherwise, <c>false</c>. The contents of the file can later be - read using - <seealso marker="#read_memory_status/1">read_memory_status/1</seealso>.</p> - </desc> - </func> - <func> - <name>sum_blocks(MemoryData) -> int()</name> - <fsummary>Return the total amount of memory used</fsummary> - <type> - <v>MemoryData = {term(), AllocList}</v> - <v>AllocList = [Desc]</v> - <v>Desc = {int(), int(), int(), PidDesc}</v> - <v>PidDesc = {int(), int(), int()} | undefined</v> - </type> - <desc> - <p>Returns the total size of the memory blocks in the list.</p> + <p><em>Example:</em></p> + <code type="none"><![CDATA[ +> instrument:allocations(#{ histogram_start => 128, histogram_width => 15 }). +{ok,{128,0, + #{udp_inet => + #{driver_event_state => {0,0,0,0,0,0,0,0,0,1,0,0,0,0,0}}, + system => + #{heap => {0,0,0,0,20,4,2,2,2,3,0,1,0,0,1}, + db_term => {271,3,1,52,80,1,0,0,0,0,0,0,0,0,0}, + code => {0,0,0,5,3,6,11,22,19,20,10,2,1,0,0}, + binary => {18,0,0,0,7,0,0,1,0,0,0,0,0,0,0}, + message => {0,40,78,2,2,0,0,0,0,0,0,0,0,0,0}, + ... } + spawn_forker => + #{driver_select_data_state => + {1,0,0,0,0,0,0,0,0,0,0,0,0,0,0}}, + ram_file_drv => #{drv_binary => {0,0,0,0,0,0,1,0,0,0,0,0,0,0,0}}, + prim_file => + #{process_specific_data => {2,0,0,0,0,0,0,0,0,0,0,0,0,0,0}, + nif_trap_export_entry => {0,4,0,0,0,0,0,0,0,0,0,0,0,0,0}, + monitor_extended => {0,1,0,0,0,0,0,0,0,0,0,0,0,0,0}, + drv_binary => {0,0,0,0,0,0,1,0,3,5,0,0,0,1,0}, + binary => {0,4,0,0,0,0,0,0,0,0,0,0,0,0,0}}, + prim_buffer => + #{nif_internal => {0,4,0,0,0,0,0,0,0,0,0,0,0,0,0}, + binary => {0,4,0,0,0,0,0,0,0,0,0,0,0,0,0}}}}} + ]]></code> </desc> </func> + <func> - <name>type_descr(MemoryData, TypeNo) -> TypeDescr | invalid_type</name> - <fsummary>Returns a type description</fsummary> - <type> - <v>MemoryData = {term(), AllocList}</v> - <v>AllocList = [Desc]</v> - <v>Desc = {int(), int(), int(), PidDesc}</v> - <v>PidDesc = {int(), int(), int()} | undefined</v> - <v>TypeNo = int()</v> - <v>TypeDescr = atom() | string()</v> - </type> + <name name="carriers" arity="0"/> + <fsummary>Return a list of all carriers in the system.</fsummary> <desc> - <p>Returns the type description of a type number used in - <c>MemoryData</c>. - Valid <c>TypeNo</c>s are in the range returned by - <seealso marker="#type_no_range/1">type_no_range/1</seealso> on - this specific memory allocation map. If <c>TypeNo</c> is an - invalid integer, <c>invalid_type</c> is returned.</p> + <p>Shorthand for + <seealso marker="#carriers/1"><c>carriers(#{})</c>.</seealso></p> </desc> </func> + <func> - <name>type_no_range(MemoryData) -> {Min, Max}</name> - <fsummary>Returns the memory block type numbers</fsummary> - <type> - <v>MemoryData = {term(), AllocList}</v> - <v>AllocList = [Desc]</v> - <v>Desc = {int(), int(), int(), PidDesc}</v> - <v>PidDesc = {int(), int(), int()} | undefined</v> - <v>Min = int()</v> - <v>Max = int()</v> - </type> + <name name="carriers" arity="1"/> + <fsummary>Return a list of all carriers filtered by allocator type and + scheduler id.</fsummary> <desc> - <marker id="type_no_range_1"></marker> - <p>Returns the memory block type number range used in - <c>MemoryData</c>. When the memory allocation map was generated - by an Erlang 5.3/OTP R9C or newer emulator, all integers <c>T</c> - that satisfy <c>Min</c> <= <c>T</c> <= <c>Max</c> are - valid type numbers. When the memory allocation map was generated - by a pre Erlang 5.3/OTP R9C emulator, all integers in the - range are <em>not</em> valid type numbers.</p> + <p>Returns a summary of all carriers in the system, optionally filtered + by allocator type and scheduler id.</p> + <p>If the specified allocator types are not enabled, the call will fail + with <c>{error, not_enabled}</c>.</p> + <p>The following options can be used:</p> + <taglist> + <tag><c>allocator_types</c></tag> + <item> + <p>The allocator types that will be searched. Defaults to all + <c>alloc_util</c> allocators.</p> + </item> + <tag><c>scheduler_ids</c></tag> + <item> + <p>The scheduler ids whose allocator instances will be searched. A + scheduler id of 0 will refer to the global instance that is not + tied to any particular scheduler. Defaults to all schedulers and + the global instance.</p> + </item> + <tag><c>histogram_start</c></tag> + <item> + <p>The upper bound of the first interval in the free block size + histograms. Defaults to 512.</p> + </item> + <tag><c>histogram_width</c></tag> + <item> + <p>The number of intervals in the free block size histograms. + Defaults to 14.</p> + </item> + </taglist> + <p><em>Example:</em></p> + <code type="none"><![CDATA[ +> instrument:carriers(#{ histogram_start => 512, histogram_width => 8 }). +{ok,{512, + [{ll_alloc,1048576,0,1048344,71,false,{0,0,0,0,0,0,0,0}}, + {binary_alloc,1048576,0,324640,13,false,{3,0,0,1,0,0,0,2}}, + {eheap_alloc,2097152,0,1037200,45,false,{2,1,1,3,4,3,2,2}}, + {fix_alloc,32768,0,29544,82,false,{22,0,0,0,0,0,0,0}}, + {...}|...]}} + ]]></code> </desc> </func> + </funcs> <section> diff --git a/lib/tools/doc/src/specs.xml b/lib/tools/doc/src/specs.xml new file mode 100644 index 0000000000..0b5b7b171c --- /dev/null +++ b/lib/tools/doc/src/specs.xml @@ -0,0 +1,12 @@ +<?xml version="1.0" encoding="utf-8" ?> +<specs xmlns:xi="http://www.w3.org/2001/XInclude"> + <xi:include href="../specs/specs_fprof.xml"/> + <xi:include href="../specs/specs_make.xml"/> + <xi:include href="../specs/specs_lcnt.xml"/> + <xi:include href="../specs/specs_eprof.xml"/> + <xi:include href="../specs/specs_tags.xml"/> + <xi:include href="../specs/specs_cover.xml"/> + <xi:include href="../specs/specs_xref.xml"/> + <xi:include href="../specs/specs_instrument.xml"/> + <xi:include href="../specs/specs_erlang_mode.xml"/> +</specs> diff --git a/lib/tools/emacs/erlang.el b/lib/tools/emacs/erlang.el index b3411c3ce7..fd51aca861 100644 --- a/lib/tools/emacs/erlang.el +++ b/lib/tools/emacs/erlang.el @@ -825,6 +825,7 @@ resulting regexp is surrounded by \\_< and \\_>." "list_to_tuple" "load_module" "make_ref" + "map_get" "map_size" "max" "min" @@ -884,7 +885,6 @@ resulting regexp is surrounded by \\_< and \\_>." "alloc_sizes" "append" "append_element" - "await_proc_exit" "bump_reductions" "call_on_load_function" "cancel_timer" @@ -903,7 +903,6 @@ resulting regexp is surrounded by \\_< and \\_>." "dist_ctrl_input_handler" "dist_ctrl_put_data" "dmonitor_node" - "dmonitor_p" "dt_append_vm_tag_data" "dt_get_tag" "dt_get_tag_data" @@ -912,6 +911,7 @@ resulting regexp is surrounded by \\_< and \\_>." "dt_restore_tag" "dt_spread_tag" "convert_time_unit" + "exit_signal" "external_size" "finish_after_on_load" "finish_loading" diff --git a/lib/tools/src/instrument.erl b/lib/tools/src/instrument.erl index 055f4a7afb..0203fefe13 100644 --- a/lib/tools/src/instrument.erl +++ b/lib/tools/src/instrument.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2016. All Rights Reserved. +%% Copyright Ericsson AB 1998-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -19,410 +19,140 @@ %% -module(instrument). --export([holes/1, mem_limits/1, memory_data/0, read_memory_data/1, - sort/1, store_memory_data/1, sum_blocks/1, - descr/1, type_descr/2, allocator_descr/2, class_descr/2, - type_no_range/1, block_header_size/1, store_memory_status/1, - read_memory_status/1, memory_status/1]). - - --define(OLD_INFO_SIZE, 32). %% (sizeof(mem_link) in pre R9C utils.c) - --define(IHMARKER(H), element(1, H)). --define(VSN(H), element(2, H)). --define(INFO_SIZE(H), element(3, H)). --define(TYPEMAP(H), element(4, H)). - --define(IHDR(H), is_tuple(H), ?IHMARKER(H) =:= instr_hdr). --define(IHDRVSN(H, V), ?IHDR(H), ?VSN(H) =:= V). - -memory_data() -> - case catch erlang:system_info(allocated) of - {'EXIT',{Error,_}} -> - erlang:error(Error, []); - {'EXIT',Error} -> - erlang:error(Error, []); - Res -> - Res +-export([allocations/0, allocations/1, + carriers/0, carriers/1]). + +-type block_histogram() :: tuple(). + +-type allocation_summary() :: + {HistogramStart :: non_neg_integer(), + UnscannedSize :: non_neg_integer(), + Allocations :: #{ Origin :: atom() => + #{ Type :: atom() => block_histogram() }}}. + +-spec allocations() -> {ok, Result} | {error, Reason} when + Result :: allocation_summary(), + Reason :: not_enabled. +allocations() -> + allocations(#{}). + +-spec allocations(Options) -> {ok, Result} | {error, Reason} when + Result :: allocation_summary(), + Reason :: not_enabled, + Options :: #{ scheduler_ids => list(non_neg_integer()), + allocator_types => list(atom()), + histogram_start => pos_integer(), + histogram_width => pos_integer() }. +allocations(Options) -> + Ref = make_ref(), + + Defaults = #{ scheduler_ids => lists:seq(0, erlang:system_info(schedulers)), + allocator_types => erlang:system_info(alloc_util_allocators), + histogram_start => 128, + histogram_width => 18 }, + + {HistStart, MsgCount} = + dispatch_gather(maps:merge(Defaults, Options), Ref, + fun erts_internal:gather_alloc_histograms/1), + + alloc_hist_receive(HistStart, MsgCount, Ref). + +alloc_hist_receive(_HistStart, 0, _Ref) -> + {error, not_enabled}; +alloc_hist_receive(HistStart, MsgCount, Ref) when MsgCount > 0 -> + {Unscanned, Histograms} = alloc_hist_receive_1(MsgCount, Ref, 0, #{}), + {ok, {HistStart, Unscanned, Histograms}}. + +alloc_hist_receive_1(0, _Ref, Unscanned, Result) -> + {Unscanned, Result}; +alloc_hist_receive_1(MsgCount, Ref, Unscanned0, Result0) -> + receive + {Ref, Unscanned, Tags} -> + Result = lists:foldl(fun alloc_hist_fold_result/2, Result0, Tags), + alloc_hist_receive_1(MsgCount - 1, Ref, Unscanned0 + Unscanned, Result) end. -store_memory_data(File) -> - case catch erlang:system_info({allocated, File}) of - {'EXIT',{Error,_}} -> - erlang:error(Error, [File]); - {'EXIT',Error} -> - erlang:error(Error, [File]); - Res -> - Res +alloc_hist_fold_result({Id, Type, BlockHist}, Result0) -> + IdAllocs0 = maps:get(Id, Result0, #{}), + MergedHists = case maps:find(Type, IdAllocs0) of + {ok, PrevHist} -> + alloc_hist_merge_hist(tuple_size(BlockHist), + BlockHist, + PrevHist); + error -> + BlockHist + end, + IdAllocs = IdAllocs0#{ Type => MergedHists }, + Result0#{ Id => IdAllocs }. + +alloc_hist_merge_hist(0, A, _B) -> + A; +alloc_hist_merge_hist(Index, A, B) -> + Merged = setelement(Index, A, element(Index, A) + element(Index, B)), + alloc_hist_merge_hist(Index - 1, Merged, B). + +-type carrier_info_list() :: + {HistogramStart :: non_neg_integer(), + Carriers :: [{AllocatorType :: atom(), + TotalSize :: non_neg_integer(), + UnscannedSize :: non_neg_integer(), + AllocatedSize :: non_neg_integer(), + AllocatedCount :: non_neg_integer(), + InPool :: boolean(), + FreeBlocks :: block_histogram()}]}. + +-spec carriers() -> {ok, Result} | {error, Reason} when + Result :: carrier_info_list(), + Reason :: not_enabled. +carriers() -> + carriers(#{}). + +-spec carriers(Options) -> {ok, Result} | {error, Reason} when + Result :: carrier_info_list(), + Reason :: not_enabled, + Options :: #{ scheduler_ids => list(non_neg_integer()), + allocator_types => list(atom()), + histogram_start => pos_integer(), + histogram_width => pos_integer() }. +carriers(Options) -> + Ref = make_ref(), + + Defaults = #{ scheduler_ids => lists:seq(0, erlang:system_info(schedulers)), + allocator_types => erlang:system_info(alloc_util_allocators), + histogram_start => 512, + histogram_width => 14 }, + + {HistStart, MsgCount} = + dispatch_gather(maps:merge(Defaults, Options), Ref, + fun erts_internal:gather_carrier_info/1), + + carrier_info_receive(HistStart, MsgCount, Ref). + +carrier_info_receive(_HistStart, 0, _Ref) -> + {error, not_enabled}; +carrier_info_receive(HistStart, MsgCount, Ref) -> + {ok, {HistStart, carrier_info_receive_1(MsgCount, Ref, [])}}. + +carrier_info_receive_1(0, _Ref, Result) -> + lists:flatten(Result); +carrier_info_receive_1(MsgCount, Ref, Result0) -> + receive + {Ref, Carriers} -> + carrier_info_receive_1(MsgCount - 1, Ref, [Carriers, Result0]) end. -memory_status(Type) when is_atom(Type) -> - case catch erlang:system_info({allocated, status, Type}) of - {'EXIT',{Error,_}} -> - erlang:error(Error, [Type]); - {'EXIT',Error} -> - erlang:error(Error, [Type]); - Res -> - Res - end; -memory_status(Type) -> - erlang:error(badarg, [Type]). - -store_memory_status(File) when is_list(File) -> - case catch erlang:system_info({allocated, status, File}) of - {'EXIT',{Error,_}} -> - erlang:error(Error, [File]); - {'EXIT',Error} -> - erlang:error(Error, [File]); - Res -> - Res - end; -store_memory_status(File) -> - erlang:error(badarg, [File]). - -read_memory_data(File) when is_list(File) -> - case file:consult(File) of - {ok, [Hdr|MD]} when ?IHDR(Hdr) -> - {Hdr, MD}; - {ok, [{T,A,S,undefined}|_] = MD} when is_integer(T), - is_integer(A), - is_integer(S) -> - {{instr_hdr, 1, ?OLD_INFO_SIZE}, MD}; - {ok, [{T,A,S,{X,Y,Z}}|_] = MD} when is_integer(T), - is_integer(A), - is_integer(S), - is_integer(X), - is_integer(Y), - is_integer(Z) -> - {{instr_hdr, 1, ?OLD_INFO_SIZE}, MD}; - {ok, _} -> - {error, eio}; - Error -> - Error - end; -read_memory_data(File) -> - erlang:error(badarg, [File]). - -read_memory_status(File) when is_list(File) -> - case file:consult(File) of - {ok, [{instr_vsn, _}|Stat]} -> - Stat; - {ok, _} -> - {error, eio}; - Error -> - Error - end; -read_memory_status(File) -> - erlang:error(badarg, [File]). - -holes({Hdr, MD}) when ?IHDR(Hdr) -> - check_holes(?INFO_SIZE(Hdr), MD). - -check_holes(_ISz, []) -> - ok; -check_holes(ISz, [E | L]) -> - check_holes(ISz, E, L). - -check_holes(_ISz, _E1, []) -> - io:format("~n"); -check_holes(ISz, E1, [E2 | Rest]) -> - check_hole(ISz, E1, E2), - check_holes(ISz, E2, Rest). - -check_hole(ISz, {_,P1,S1,_}, {_,P2,_,_}) -> - End = P1+S1, - Hole = P2 - (End + ISz), - if - Hole =< 7 -> - ok; - true -> - io:format(" ~p", [Hole]) - end. - -sum_blocks({Hdr, L}) when ?IHDR(Hdr) -> - lists:foldl(fun({_,_,S,_}, Sum) -> S+Sum end, - 0, - L). - -mem_limits({Hdr, L}) when ?IHDR(Hdr) -> - {_, P1, _, _} = hd(L), - {_, P2, S2, _} = lists:last(L), - {P1, P2+S2}. - -sort({Hdr, MD}) when ?IHDR(Hdr) -> - {Hdr, lists:keysort(2, MD)}. - -descr({Hdr, MD} = ID) when ?IHDR(Hdr) -> - {Hdr, lists:map(fun ({TN, Addr, Sz, {0, N, S}}) -> - {type_descr(ID, TN), - Addr, - Sz, - list_to_pid("<0." - ++ integer_to_list(N) - ++ "." - ++ integer_to_list(S) - ++ ">")}; - ({TN, Addr, Sz, undefined}) -> - {type_descr(ID, TN), - Addr, - Sz, - undefined} - end, - MD)}. - -block_header_size({Hdr, _}) when ?IHDR(Hdr) -> - ?INFO_SIZE(Hdr). - -type_descr({Hdr, _}, TypeNo) when ?IHDRVSN(Hdr, 2), - is_integer(TypeNo) -> - case catch element(1, element(TypeNo, ?TYPEMAP(Hdr))) of - {'EXIT', _} -> invalid_type; - Type -> Type - end; -type_descr({Hdr, _}, TypeNo) when ?IHDRVSN(Hdr, 1), - is_integer(TypeNo) -> - type_string(TypeNo). - - -allocator_descr({Hdr, _}, TypeNo) when ?IHDRVSN(Hdr, 2), is_integer(TypeNo) -> - case catch element(2, element(TypeNo, ?TYPEMAP(Hdr))) of - {'EXIT', _} -> invalid_type; - Type -> Type - end; -allocator_descr({Hdr, _}, TypeNo) when ?IHDRVSN(Hdr, 1), is_integer(TypeNo) -> - "unknown". - -class_descr({Hdr, _}, TypeNo) when ?IHDRVSN(Hdr, 2), is_integer(TypeNo) -> - case catch element(3, element(TypeNo, ?TYPEMAP(Hdr))) of - {'EXIT', _} -> invalid_type; - Type -> Type - end; -class_descr({Hdr, _}, TypeNo) when ?IHDRVSN(Hdr, 1), is_integer(TypeNo) -> - "unknown". - -type_no_range({Hdr, _}) when ?IHDRVSN(Hdr, 2) -> - {1, tuple_size(?TYPEMAP(Hdr))}; -type_no_range({Hdr, _}) when ?IHDRVSN(Hdr, 1) -> - {-1, 1000}. - -type_string(-1) -> - "unknown"; -type_string(1) -> - "atom text"; -type_string(11) -> - "atom desc"; -type_string(2) -> - "bignum (big_to_list)"; -type_string(31) -> - "fixalloc"; -type_string(32) -> - "unknown fixalloc block"; -type_string(33) -> - "message buffer"; -type_string(34) -> - "message link"; -type_string(4) -> - "estack"; -type_string(40) -> - "db table vec"; -type_string(41) -> - "db tree select buffer"; -type_string(43) -> - "db hash select buffer"; -type_string(44) -> - "db hash select list"; -type_string(45) -> - "db match prog stack"; -type_string(46) -> - "db match prog heap data"; -type_string(47) -> - "db temp buffer"; -type_string(48) -> - "db error"; -type_string(49) -> - "db error info"; -type_string(50) -> - "db trans tab"; -type_string(51) -> - "db segment"; -type_string(52) -> - "db term"; -type_string(53) -> - "db add_counter"; -type_string(54) -> - "db segment table"; -type_string(55) -> - "db table (fix)"; -type_string(56) -> - "db bindings"; -type_string(57) -> - "db counter"; -type_string(58) -> - "db trace vec"; -type_string(59) -> - "db fixed deletion"; -type_string(60) -> - "binary (external.c)"; -type_string(61) -> - "binary"; -type_string(62) -> - "procbin (fix)"; -type_string(70) -> - "driver alloc (io.c)"; -type_string(71) -> - "binary (io.c)"; -type_string(72) -> - "binary vec (io.c)"; -type_string(73) -> - "binary vec 2 (io.c)"; -type_string(74) -> - "io vec (io.c)"; -type_string(75) -> - "io vec 2 (io.c)"; -type_string(76) -> - "temp io buffer (io.c)"; -type_string(77) -> - "temp io buffer 2 (io.c)"; -type_string(78) -> - "line buffer (io.c)"; -type_string(8) -> - "heap"; -type_string(801) -> - "heap (1)"; -type_string(802) -> - "heap (2)"; -type_string(803) -> - "heap (3)"; -type_string(804) -> - "heap (4)"; -type_string(805) -> - "heap (5)"; -type_string(821) -> - "heap fragment (1)"; -type_string(822) -> - "heap fragment (2)"; -type_string(830) -> - "sequential store buffer (for vectors)"; -type_string(91) -> - "process table"; -type_string(92) -> - "process desc"; -type_string(110) -> - "hash buckets"; -type_string(111) -> - "hash table"; -type_string(120) -> - "index init"; -type_string(121) -> - "index table"; -type_string(130) -> - "temp buffer"; -type_string(140) -> - "timer wheel"; -type_string(150) -> - "distribution cache"; -type_string(151) -> - "dmem"; -type_string(152) -> - "distribution table"; -type_string(153) -> - "distribution table buckets"; -type_string(154) -> - "distribution table entry"; -type_string(155) -> - "node table"; -type_string(156) -> - "node table buckets"; -type_string(157) -> - "node table entry"; -type_string(160) -> - "port table"; -type_string(161) -> - "driver entry"; -type_string(162) -> - "port setup"; -type_string(163) -> - "port wait"; -type_string(170) -> - "module"; -type_string(171) -> - "fundef"; -type_string(180) -> - "file table"; -type_string(181) -> - "driver table"; -type_string(182) -> - "poll struct"; -type_string(190) -> - "inet driver"; -type_string(200) -> - "efile driver"; -type_string(210) -> - "gc root set"; -type_string(220) -> - "breakpoint data"; -type_string(230) -> - "async queue"; -type_string(231) -> - "async (exit)"; -type_string(232) -> - "async (driver)"; -type_string(240) -> - "bits buffer"; -type_string(241) -> - "bits temp buffer"; -type_string(250) -> - "modules (loader)"; -type_string(251) -> - "code (loader)"; -type_string(252) -> - "atom tab (loader)"; -type_string(253) -> - "import tab (loader)"; -type_string(254) -> - "export tab (loader)"; -type_string(255) -> - "lable tab (loader)"; -type_string(256) -> - "gen op (loader)"; -type_string(257) -> - "gen op args (loader)"; -type_string(258) -> - "gen op args 2 (loader)"; -type_string(259) -> - "gen op args 3 (loader)"; -type_string(260) -> - "lambdas (loader)"; -type_string(261) -> - "temp int buffer (loader)"; -type_string(262) -> - "temp heap (loader)"; -type_string(280) -> - "dist ctrl msg buffer"; -type_string(281) -> - "dist_buf"; -type_string(290) -> - "call trace buffer"; -type_string(300) -> - "bif timer rec"; -type_string(310) -> - "argument registers"; -type_string(320) -> - "compressed binary temp buffer"; -type_string(330) -> - "term_to_binary temp buffer"; -type_string(340) -> - "proc dict"; -type_string(350) -> - "trace to port temp buffer"; -type_string(360) -> - "lists subtract temp buffer"; -type_string(370) -> - "link (lh)"; -type_string(380) -> - "port call buffer"; -type_string(400) -> - "definite_alloc block"; -type_string(_) -> - invalid_type. - +dispatch_gather(#{ allocator_types := AllocatorTypes, + scheduler_ids := SchedulerIds, + histogram_start := HistStart, + histogram_width := HistWidth }, Ref, Gather) + when is_list(AllocatorTypes), + is_list(SchedulerIds), + HistStart >= 1, HistStart =< (1 bsl 28), + HistWidth >= 1, HistWidth =< 32 -> + MsgCount = lists:sum( + [Gather({AllocatorType, SchedId, HistWidth, HistStart, Ref}) || + SchedId <- SchedulerIds, + AllocatorType <- AllocatorTypes]), + {HistStart, MsgCount}; +dispatch_gather(_, _, _) -> + error(badarg). diff --git a/lib/tools/test/instrument_SUITE.erl b/lib/tools/test/instrument_SUITE.erl index f37d28c277..8c521b2e1a 100644 --- a/lib/tools/test/instrument_SUITE.erl +++ b/lib/tools/test/instrument_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2016. All Rights Reserved. +%% Copyright Ericsson AB 1998-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -20,89 +20,274 @@ -module(instrument_SUITE). -export([all/0, suite/0]). --export(['+Mim true'/1, '+Mis true'/1]). + +-export([allocations_enabled/1, allocations_disabled/1, allocations_ramv/1, + carriers_enabled/1, carriers_disabled/1]). + +-export([test_all_alloc/2, test_per_alloc/2, test_format/3, test_abort/1, + generate_test_blocks/0, churn_memory/0]). -include_lib("common_test/include/ct.hrl"). suite() -> [{ct_hooks,[ts_install_cth]}, - {timetrap,{seconds,10}}]. + {timetrap,{minutes,5}}]. all() -> - ['+Mim true', '+Mis true']. - - -%% Check that memory data can be read and processed -'+Mim true'(Config) when is_list(Config) -> - Node = start_slave("+Mim true"), - MD = rpc:call(Node, instrument, memory_data, []), - [{total,[{sizes,S1,S2,S3},{blocks,B1,B2,B3}]}] - = rpc:call(Node, instrument, memory_status, [total]), - stop_slave(Node), - true = S1 =< S2, - true = S2 =< S3, - true = B1 =< B2, - true = B2 =< B3, - MDS = instrument:sort(MD), - {Low, High} = instrument:mem_limits(MDS), - true = Low < High, - {_, AL} = MDS, - SumBlocks = instrument:sum_blocks(MD), - case SumBlocks of - N when is_integer(N) -> - N = lists:foldl(fun ({_,_,Size,_}, Sum) -> - Size+Sum - end, 0, AL), - true = N =< S3; - Other -> - ct:fail(Other) + [allocations_enabled, allocations_disabled, allocations_ramv, + carriers_enabled, carriers_disabled]. + +-define(GENERATED_SBC_BLOCK_COUNT, 1000). +-define(GENERATED_MBC_BLOCK_COUNT, ?GENERATED_SBC_BLOCK_COUNT). + +-define(GENERATED_BLOCK_COUNT, (?GENERATED_SBC_BLOCK_COUNT + + ?GENERATED_MBC_BLOCK_COUNT)). +-define(GENERATED_CARRIER_COUNT, ?GENERATED_SBC_BLOCK_COUNT). + +allocations_test(Args, Plain, PerAlloc) -> + run_test(Args, fun(Node) -> + ok = rpc:call(Node, ?MODULE, test_all_alloc, + [fun instrument:allocations/0, Plain]), + ok = rpc:call(Node, ?MODULE, test_per_alloc, + [fun instrument:allocations/1, PerAlloc]), + ok = rpc:call(Node, ?MODULE, test_format, + [#{ histogram_start => 512, + histogram_width => 4 }, + fun instrument:allocations/1, + fun verify_allocations_output/2]), + ok = rpc:call(Node, ?MODULE, test_abort, + [fun erts_internal:gather_alloc_histograms/1]) + end). + +allocations_enabled(Config) when is_list(Config) -> + allocations_test("+Meamax +Muatags true", + fun verify_allocations_enabled/1, + fun verify_allocations_enabled/2). + +allocations_disabled(Config) when is_list(Config) -> + allocations_test("+Meamax +Muatags false", + fun verify_allocations_disabled/1, + fun verify_allocations_disabled/2). + +allocations_ramv(Config) when is_list(Config) -> + allocations_test("+Meamax +Muatags true +Muramv true", + fun verify_allocations_enabled/1, + fun verify_allocations_enabled/2). + +verify_allocations_disabled(_AllocType, Result) -> + verify_allocations_disabled(Result). + +verify_allocations_disabled({error, not_enabled}) -> + ok. + +%% Skip types that have unstable results or are unaffected by +Muatags +verify_allocations_enabled(literal_alloc, _Result) -> ok; +verify_allocations_enabled(exec_alloc, _Result) -> ok; +verify_allocations_enabled(temp_alloc, _Result) -> ok; +verify_allocations_enabled(sl_alloc, _Result) -> ok; +verify_allocations_enabled(_AllocType, Result) -> + verify_allocations_enabled(Result). + +verify_allocations_enabled({ok, {_HistStart, _UnscannedBytes, Allocs}}) -> + true = Allocs =/= #{}. + +verify_allocations_output(#{ histogram_start := HistStart, + histogram_width := HistWidth }, + {ok, {HistStart, _UnscannedBytes, ByOrigin}}) -> + AllHistograms = lists:flatten([maps:values(ByType) || + ByType <- maps:values(ByOrigin)]), + + %% Do the histograms look alright? + HistogramSet = ordsets:from_list(AllHistograms), + Verified = [H || H <- HistogramSet, + tuple_size(H) =:= HistWidth, + hist_sum(H) >= 1], + [] = ordsets:subtract(HistogramSet, Verified), + + %% Do we have at least as many blocks as we've generated? + BlockCount = lists:foldl(fun(Hist, Acc) -> + hist_sum(Hist) + Acc + end, 0, AllHistograms), + GenTotalBlockCount = ?GENERATED_BLOCK_COUNT, + GenSBCBlockCount = ?GENERATED_SBC_BLOCK_COUNT, + if + BlockCount < GenSBCBlockCount -> + ct:fail("Found ~p blocks, required at least ~p (SB)." , + [BlockCount, GenSBCBlockCount]); + BlockCount >= GenTotalBlockCount -> + ct:pal("Found ~p blocks, expected at least ~p (SB + MB).", + [BlockCount, GenTotalBlockCount]); + BlockCount < GenTotalBlockCount -> + ct:pal("Found ~p blocks, expected at least ~p (SB + MB), but this " + "may be due to MBCs being skipped if they're about to be " + "scanned just as they're fetched from the carrier pool.", + [BlockCount, GenTotalBlockCount]) + end, + + ok; +verify_allocations_output(#{}, {error, not_enabled}) -> + ok. + +%% %% %% %% %% %% + +carriers_test(Args, Plain, PerAlloc) -> + run_test(Args, fun(Node) -> + ok = rpc:call(Node, ?MODULE, test_all_alloc, + [fun instrument:carriers/0, Plain]), + ok = rpc:call(Node, ?MODULE, test_per_alloc, + [fun instrument:carriers/1, PerAlloc]), + ok = rpc:call(Node, ?MODULE, test_format, + [#{ histogram_start => 1024, + histogram_width => 4 }, + fun instrument:carriers/1, + fun verify_carriers_output/2]), + ok = rpc:call(Node, ?MODULE, test_abort, + [fun erts_internal:gather_carrier_info/1]) + end). + +carriers_enabled(Config) when is_list(Config) -> + carriers_test("+Meamax", + fun verify_carriers_enabled/1, + fun verify_carriers_enabled/2). + +carriers_disabled(Config) when is_list(Config) -> + carriers_test("+Meamin", + fun verify_carriers_disabled/1, + fun verify_carriers_disabled/2). + +verify_carriers_disabled(_AllocType, Result) -> + verify_carriers_disabled(Result). + +verify_carriers_disabled({error, not_enabled}) -> + ok; +verify_carriers_disabled({ok, {_HistStart, Carriers}}) -> + verify_carriers_disabled_1(Carriers). + +verify_carriers_disabled_1([]) -> + ok; +%% literal_alloc, exec_alloc, and temp_alloc can't be disabled, so we have to +%% accept their presence in carriers_disabled/test_all_alloc. +verify_carriers_disabled_1([Carrier | Rest]) when + element(1, Carrier) =:= literal_alloc; + element(1, Carrier) =:= exec_alloc; + element(1, Carrier) =:= temp_alloc -> + verify_carriers_disabled_1(Rest). + +%% exec_alloc only has a carrier if it's actually used. +verify_carriers_enabled(exec_alloc, _Result) -> ok; +verify_carriers_enabled(_AllocType, Result) -> verify_carriers_enabled(Result). + +verify_carriers_enabled({ok, {_HistStart, Carriers}}) when Carriers =/= [] -> + ok. + +verify_carriers_output(#{ histogram_start := HistStart, + histogram_width := HistWidth }, + {ok, {HistStart, AllCarriers}}) -> + + %% Do the carriers look alright? + CarrierSet = ordsets:from_list(AllCarriers), + Verified = [C || {AllocType, + TotalSize, + UnscannedSize, + AllocatedSize, + AllocatedCount, + InPool, + FreeBlockHist} = C <- CarrierSet, + is_atom(AllocType), + is_integer(TotalSize), TotalSize >= 1, + is_integer(UnscannedSize), UnscannedSize < TotalSize, + UnscannedSize >= 0, + is_integer(AllocatedSize), AllocatedSize < TotalSize, + AllocatedSize >= 0, + is_integer(AllocatedCount), AllocatedCount =< AllocatedSize, + AllocatedCount >= 0, + is_boolean(InPool), + tuple_size(FreeBlockHist) =:= HistWidth, + carrier_block_check(AllocatedCount, FreeBlockHist)], + [] = ordsets:subtract(CarrierSet, Verified), + + %% Do we have at least as many carriers as we've generated? + CarrierCount = length(AllCarriers), + GenSBCCount = ?GENERATED_SBC_BLOCK_COUNT, + if + CarrierCount < GenSBCCount -> + ct:fail("Carrier count is ~p, expected at least ~p (SBC).", + [CarrierCount, GenSBCCount]); + CarrierCount >= GenSBCCount -> + ok end, - lists:foldl( - fun ({TDescr,Addr,Size,Proc}, MinAddr) -> - true = TDescr /= invalid_type, - true = is_integer(TDescr), - true = is_integer(Addr), - true = is_integer(Size), - true = Addr >= MinAddr, - case Proc of - {0, Number, Serial} -> - true = is_integer(Number), - true = is_integer(Serial); - undefined -> - ok; - BadProc -> - ct:fail({badproc, BadProc}) - end, - NextMinAddr = Addr+Size, - true = NextMinAddr =< High, - NextMinAddr - end, Low, AL), - {_, DAL} = instrument:descr(MDS), - lists:foreach( - fun ({TDescr,_,_,Proc}) -> - true = TDescr /= invalid_type, - true = is_atom(TDescr) orelse is_list(TDescr), - true = is_pid(Proc) orelse Proc == undefined - end, DAL), - ASL = lists:map(fun ({_,A,S,_}) -> {A,S} end, AL), - ASL = lists:map(fun ({_,A,S,_}) -> {A,S} end, DAL), - instrument:holes(MDS), - {comment, "total status - sum of blocks = " ++ integer_to_list(S1-SumBlocks)}. - -%% Check that memory data can be read and processed -'+Mis true'(Config) when is_list(Config) -> - Node = start_slave("+Mis true"), - [{total,[{sizes,S1,S2,S3},{blocks,B1,B2,B3}]}] - = rpc:call(Node, instrument, memory_status, [total]), - true = S1 =< S2, - true = S2 =< S3, - true = B1 =< B2, - true = B2 =< B3, - true = is_list(rpc:call(Node,instrument,memory_status,[allocators])), - true = is_list(rpc:call(Node,instrument,memory_status,[classes])), - true = is_list(rpc:call(Node,instrument,memory_status,[types])), + + ok; +verify_carriers_output(#{}, {error, not_enabled}) -> + ok. + +carrier_block_check(AllocCount, FreeHist) -> + %% A carrier must contain at least one block, and th. number of free blocks + %% must not exceed the number of allocated blocks + 1. + FreeCount = hist_sum(FreeHist), + + (AllocCount + FreeCount) >= 1 andalso FreeCount =< (AllocCount + 1). + +%% %% %% %% %% %% + +test_all_alloc(Gather, Verify) -> + Verify(Gather()), ok. +test_per_alloc(Gather, Verify) -> + [begin + Verify(T, Gather(#{ allocator_types => [T] })) + end || T <- erlang:system_info(alloc_util_allocators)], + ok. + +test_format(#{ allocator_types := _ }, _, _) -> + error(badarg); +test_format(Options0, Gather, Verify) -> + %% We limit format checking to binary_alloc since we generated the test + %% vectors there. + Options = Options0#{ allocator_types => [binary_alloc] }, + Verify(Options, Gather(Options)), + ok. + +test_abort(Gather) -> + %% There's no way for us to tell whether this actually aborted or ran to + %% completion, but it might catch a few segfaults. + Runner = self(), + Ref = make_ref(), + spawn_opt(fun() -> + [Gather({Type, SchedId, 1, 1, Ref}) || + Type <- erlang:system_info(alloc_util_allocators), + SchedId <- lists:seq(0, erlang:system_info(schedulers))], + Runner ! Ref + end, [{priority, max}]), + receive + Ref -> ok + end. + +hist_sum(H) -> hist_sum_1(H, tuple_size(H), 0). +hist_sum_1(_H, 0, A) -> A; +hist_sum_1(H, N, A) -> hist_sum_1(H, N - 1, element(N, H) + A). + +%% + +run_test(Args0, Test) -> + %% Override single-block carrier threshold for binaries to ensure we have + %% coverage for that path. generate_test_blocks builds a few binaries that + %% crosses this threshold. + %% + %% We also set the abandon carrier threshold to 70% to provoke more + %% activity in the carrier pool. + Args = Args0 ++ " +MBsbct 1 +Muacul 70", + Node = start_slave(Args), + + ok = rpc:call(Node, ?MODULE, generate_test_blocks, []), + ok = Test(Node), + + ok = rpc:call(Node, ?MODULE, churn_memory, []), + ok = Test(Node), + + true = test_server:stop_node(Node). + start_slave(Args) -> MicroSecs = erlang:monotonic_time(), Name = "instr" ++ integer_to_list(MicroSecs), @@ -112,6 +297,60 @@ start_slave(Args) -> [{args, "-pa " ++ Pa ++ " " ++ Args}]), Node. +generate_test_blocks() -> + Runner = self(), + Ref = make_ref(), + spawn(fun() -> + %% We've set the single-block carrier threshold to 1KB so one + %% ought to land in a SBC and the other in a MBC. Both are kept + %% alive forever. + SBCs = [<<I, 0:(1 bsl 10)/unit:8>> || + I <- lists:seq(1, ?GENERATED_SBC_BLOCK_COUNT)], + MBCs = [<<I, 0:64/unit:8>> || + I <- lists:seq(1, ?GENERATED_MBC_BLOCK_COUNT)], + Runner ! Ref, + receive after infinity -> ok end, + unreachable ! {SBCs, MBCs} + end), + receive + Ref -> ok + end. -stop_slave(Node) -> - true = test_server:stop_node(Node). +churn_memory() -> + %% All processes spawned from here on have 'low' priority to avoid starving + %% others (e.g. the rpc process) which could cause the test to time out. + [begin + churn_list_to_binary(), + churn_processes(), + churn_ets() + end || _ <- lists:seq(1, erlang:system_info(schedulers))], + ok. + +churn_processes() -> + Pid = spawn_opt(fun churn_processes/0, [{priority, low}]), + [Pid ! <<I, 0:128/unit:8>> || I <- lists:seq(1, 128)]. + +%% Nearly all types have a few allocations at all times but sl_alloc is +%% often empty. list_to_binary on large inputs will yield and spill the +%% state into an 'estack' which is allocated through sl_alloc. +%% +%% This is inherently unstable so we skip the verification step for this +%% type, but there's still a point to hammering it. +churn_list_to_binary() -> + List = binary_to_list(<<0:(1 bsl 20)/unit:8>>), + spawn_opt(fun() -> churn_list_to_binary_1(List) end, [{priority, low}]). + +churn_list_to_binary_1(List) -> + _ = id(list_to_binary(List)), + churn_list_to_binary_1(List). + +churn_ets() -> + spawn_opt(fun() -> churn_ets_1(ets:new(gurka, [])) end, [{priority, low}]). + +churn_ets_1(Tab) -> + ets:insert(Tab, {gaffel, lists:seq(1, 16)}), + ets:delete_all_objects(Tab), + churn_ets_1(Tab). + +id(I) -> + I. diff --git a/lib/wx/api_gen/wx_extra/wxGraphicsRenderer.c_src b/lib/wx/api_gen/wx_extra/wxGraphicsRenderer.c_src new file mode 100644 index 0000000000..4718525dd6 --- /dev/null +++ b/lib/wx/api_gen/wx_extra/wxGraphicsRenderer.c_src @@ -0,0 +1,58 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +<<CreatePen +case ~s: { // wxGraphicsRenderer::CreatePen taylormade + wxGraphicsRenderer *This = (wxGraphicsRenderer *) getPtr(bp,memenv); bp += 4; + wxPen *pen = (wxPen *) getPtr(bp,memenv); bp += 4; + if(!This) throw wxe_badarg(0); +#if !wxCHECK_VERSION(3,1,1) + wxGraphicsPen * Result = new wxGraphicsPen(This->CreatePen(*pen)); newPtr((void *) Result,4, memenv); + rt.addRef(getRef((void *)Result,memenv), "wxGraphicsPen"); + break; +#else + wxGraphicsPenInfo info = wxGraphicsPenInfo() + .Colour(pen->GetColour()) + .Width(pen->GetWidth()) + .Style(pen->GetStyle()) + .Join(pen->GetJoin()) + .Cap(pen->GetCap()) + ; + + if ( info.GetStyle() == wxPENSTYLE_USER_DASH ) + { + wxDash *dashes; + if ( int nb_dashes = pen->GetDashes(&dashes) ) + info.Dashes(nb_dashes, dashes); + } + + if ( info.GetStyle() == wxPENSTYLE_STIPPLE ) + { + if ( wxBitmap* const stipple = pen->GetStipple() ) + info.Stipple(*stipple); + } + wxGraphicsPen * Result = new wxGraphicsPen(This->CreatePen(info)); + newPtr((void *) Result,4, memenv); + rt.addRef(getRef((void *)Result,memenv), "wxGraphicsPen"); + break; +#endif +} +CreatePen>> diff --git a/lib/wx/api_gen/wx_gen.erl b/lib/wx/api_gen/wx_gen.erl index ab70a588ab..4ba57501a5 100644 --- a/lib/wx/api_gen/wx_gen.erl +++ b/lib/wx/api_gen/wx_gen.erl @@ -93,9 +93,10 @@ mangle_info(E={not_const,List}) -> put(not_const, [atom_to_list(M) || M <- List]), E; mangle_info(E={gvars,List}) -> - A2L = fun({N,{T,C}}) -> {atom_to_list(N), {T,atom_to_list(C)}}; + A2L = fun({N,{test_if,C}}) -> {atom_to_list(N), {test_if,C}}; + ({N,{T,C}}) -> {atom_to_list(N), {T,atom_to_list(C)}}; ({N,C}) -> {atom_to_list(N), atom_to_list(C)} - end, + end, put(gvars, map(A2L,List)), E; mangle_info({class,CN,P,O,FL}) -> diff --git a/lib/wx/api_gen/wx_gen_cpp.erl b/lib/wx/api_gen/wx_gen_cpp.erl index 573abfa9b8..cc4e1b5301 100644 --- a/lib/wx/api_gen/wx_gen_cpp.erl +++ b/lib/wx/api_gen/wx_gen_cpp.erl @@ -1127,6 +1127,15 @@ build_gvar({Name, {address,Class}, _Id}, Cnt) -> w(" rt.addAtom(\"~s\"); rt.addRef(getRef((void *)&~s,memenv), \"~s\");~n",[Name,Name,Class]), w(" rt.addTupleCount(2);~n"), Cnt+1; +build_gvar({Name, {test_if,Test}, _Id}, Cnt) -> + w("#if ~s~n", [Test]), + w(" rt.addAtom(\"~s\"); rt.addInt(~s);~n", [Name, Name]), + w(" rt.addTupleCount(2);~n"), + w("#else~n", []), + w(" rt.addAtom(\"~s\"); rt.addAtom(\"undefined\");~n", [Name]), + w(" rt.addTupleCount(2);~n"), + w("#endif~n", []), + Cnt+1; build_gvar({Name, Class, _Id}, Cnt) -> w(" rt.addAtom(\"~s\"); rt.addRef(getRef((void *)~s,memenv),\"~s\");~n",[Name,Name,Class]), w(" rt.addTupleCount(2);~n"), diff --git a/lib/wx/api_gen/wx_gen_erl.erl b/lib/wx/api_gen/wx_gen_erl.erl index e272c08d90..dfee7270b4 100644 --- a/lib/wx/api_gen/wx_gen_erl.erl +++ b/lib/wx/api_gen/wx_gen_erl.erl @@ -1106,7 +1106,7 @@ gen_enums_ints() -> w("-define(wxDefaultSize, {-1,-1}).~n", []), w("-define(wxDefaultPosition, {-1,-1}).~n", []), w("~n%% Global Variables~n", []), - [w("-define(~s, wxe_util:get_const(~s)).~n", [Gvar, Gvar]) || + [w("-define(~s, wxe_util:get_const(~s)).~n", [qoute_atom(Gvar), qoute_atom(Gvar)]) || {Gvar,_,_Id} <- get(gvars)], w("~n%% Enum and defines~n", []), foldl(fun(Enum= #enum{vals=Vals}, Done) when Vals =/= [] -> @@ -1115,6 +1115,11 @@ gen_enums_ints() -> end, gb_sets:empty(), lists:sort(Enums)), close(). +qoute_atom([Char|_]=Str) when Char < $a -> + "'" ++ Str ++ "'"; +qoute_atom(Str) -> + Str. + build_enum_ints(#enum{from=From, vals=Vals},Done) -> case From of {File, undefined, [$@|_]} -> diff --git a/lib/wx/api_gen/wxapi.conf b/lib/wx/api_gen/wxapi.conf index 146c9fecc7..e2ef2d890a 100644 --- a/lib/wx/api_gen/wxapi.conf +++ b/lib/wx/api_gen/wxapi.conf @@ -87,7 +87,27 @@ {wxNullPen, {address,wxPen}}, {wxNullBrush, {address,wxBrush}}, {wxNullPalette, {address,wxPalette}}, - {wxNullFont, {address,wxFont}}]}. + {wxNullFont, {address,wxFont}}, + + %% New enums needed for gl contexts not static numbers + {'WX_GL_SAMPLE_BUFFERS', {test_if, "wxCHECK_VERSION(3,0,0)"}}, + {'WX_GL_SAMPLES', {test_if, "wxCHECK_VERSION(3,0,0)"}}, + {'WX_GL_FRAMEBUFFER_SRGB', {test_if, "wxCHECK_VERSION(3,1,0)"}}, + {'WX_GL_CORE_PROFILE', {test_if, "wxCHECK_VERSION(3,0,3)"}}, + {'WX_GL_MAJOR_VERSION', {test_if, "wxCHECK_VERSION(3,0,3)"}}, + {'WX_GL_MINOR_VERSION', {test_if, "wxCHECK_VERSION(3,0,3)"}}, + {'wx_GL_COMPAT_PROFILE', {test_if, "wxCHECK_VERSION(3,1,0)"}}, + {'WX_GL_FORWARD_COMPAT', {test_if, "wxCHECK_VERSION(3,1,0)"}}, + {'WX_GL_ES2', {test_if, "wxCHECK_VERSION(3,1,0)"}}, + {'WX_GL_DEBUG', {test_if, "wxCHECK_VERSION(3,1,0)"}}, + {'WX_GL_ROBUST_ACCESS', {test_if, "wxCHECK_VERSION(3,1,0)"}}, + {'WX_GL_NO_RESET_NOTIFY', {test_if, "wxCHECK_VERSION(3,1,0)"}}, + {'WX_GL_LOSE_ON_RESET', {test_if, "wxCHECK_VERSION(3,1,0)"}}, + {'WX_GL_RESET_ISOLATION', {test_if, "wxCHECK_VERSION(3,1,0)"}}, + {'WX_GL_RELEASE_FLUSH', {test_if, "wxCHECK_VERSION(3,1,0)"}}, + {'WX_GL_RELEASE_NONE', {test_if, "wxCHECK_VERSION(3,1,0)"}} + ]}. + {enum, wxBackgroundStyle, "wxBG_STYLE_"}. {enum, wxWindowVariant, "wxWINDOW_VARIANT_"}. {enum, wxBitmapType, "wxBITMAP_TYPE_"}. @@ -433,7 +453,8 @@ {class, wxGraphicsRenderer, object, [{ifdef, wxUSE_GRAPHICS_CONTEXT}], ['GetDefaultRenderer','CreateContext', %%'CreateContextFromNativeContext', 'CreateContextFromNativeWindow', - 'CreatePen','CreateBrush', + {'CreatePen', [{where, taylormade}]}, + 'CreateBrush', {'CreateLinearGradientBrush', [{deprecated, "!wxCHECK_VERSION(2,9,0)"}]}, {'CreateRadialGradientBrush', [{deprecated, "!wxCHECK_VERSION(2,9,0)"}]}, 'CreateFont', diff --git a/lib/wx/c_src/gen/wxe_funcs.cpp b/lib/wx/c_src/gen/wxe_funcs.cpp index a47d602337..a7bac4cf9d 100644 --- a/lib/wx/c_src/gen/wxe_funcs.cpp +++ b/lib/wx/c_src/gen/wxe_funcs.cpp @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2017. All Rights Reserved. + * Copyright Ericsson AB 2008-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -7010,13 +7010,41 @@ case wxGraphicsRenderer_CreateContext_1_0: { // wxGraphicsRenderer::CreateContex rt.addRef(getRef((void *)Result,memenv), "wxGraphicsContext"); break; } -case wxGraphicsRenderer_CreatePen: { // wxGraphicsRenderer::CreatePen + +case wxGraphicsRenderer_CreatePen: { // wxGraphicsRenderer::CreatePen taylormade wxGraphicsRenderer *This = (wxGraphicsRenderer *) getPtr(bp,memenv); bp += 4; wxPen *pen = (wxPen *) getPtr(bp,memenv); bp += 4; if(!This) throw wxe_badarg(0); - wxGraphicsPen * Result = new wxGraphicsPen(This->CreatePen(*pen)); newPtr((void *) Result,4, memenv);; +#if !wxCHECK_VERSION(3,1,1) + wxGraphicsPen * Result = new wxGraphicsPen(This->CreatePen(*pen)); newPtr((void *) Result,4, memenv); rt.addRef(getRef((void *)Result,memenv), "wxGraphicsPen"); break; +#else + wxGraphicsPenInfo info = wxGraphicsPenInfo() + .Colour(pen->GetColour()) + .Width(pen->GetWidth()) + .Style(pen->GetStyle()) + .Join(pen->GetJoin()) + .Cap(pen->GetCap()) + ; + + if ( info.GetStyle() == wxPENSTYLE_USER_DASH ) + { + wxDash *dashes; + if ( int nb_dashes = pen->GetDashes(&dashes) ) + info.Dashes(nb_dashes, dashes); + } + + if ( info.GetStyle() == wxPENSTYLE_STIPPLE ) + { + if ( wxBitmap* const stipple = pen->GetStipple() ) + info.Stipple(*stipple); + } + wxGraphicsPen * Result = new wxGraphicsPen(This->CreatePen(info)); + newPtr((void *) Result,4, memenv); + rt.addRef(getRef((void *)Result,memenv), "wxGraphicsPen"); + break; +#endif } case wxGraphicsRenderer_CreateBrush: { // wxGraphicsRenderer::CreateBrush wxGraphicsRenderer *This = (wxGraphicsRenderer *) getPtr(bp,memenv); bp += 4; diff --git a/lib/wx/c_src/gen/wxe_init.cpp b/lib/wx/c_src/gen/wxe_init.cpp index 1e432e34ce..6ce33a5449 100644 --- a/lib/wx/c_src/gen/wxe_init.cpp +++ b/lib/wx/c_src/gen/wxe_init.cpp @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2015. All Rights Reserved. + * Copyright Ericsson AB 2008-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -529,6 +529,111 @@ void WxeApp::init_nonconsts(wxeMemEnv *memenv, ErlDrvTermData caller) { rt.addTupleCount(2); rt.addAtom("wxCURSOR_MAX"); rt.addInt(wxCURSOR_MAX); rt.addTupleCount(2); +#if wxCHECK_VERSION(3,0,3) + rt.addAtom("WX_GL_CORE_PROFILE"); rt.addInt(WX_GL_CORE_PROFILE); + rt.addTupleCount(2); +#else + rt.addAtom("WX_GL_CORE_PROFILE"); rt.addAtom("undefined"); + rt.addTupleCount(2); +#endif +#if wxCHECK_VERSION(3,1,0) + rt.addAtom("WX_GL_DEBUG"); rt.addInt(WX_GL_DEBUG); + rt.addTupleCount(2); +#else + rt.addAtom("WX_GL_DEBUG"); rt.addAtom("undefined"); + rt.addTupleCount(2); +#endif +#if wxCHECK_VERSION(3,1,0) + rt.addAtom("WX_GL_ES2"); rt.addInt(WX_GL_ES2); + rt.addTupleCount(2); +#else + rt.addAtom("WX_GL_ES2"); rt.addAtom("undefined"); + rt.addTupleCount(2); +#endif +#if wxCHECK_VERSION(3,1,0) + rt.addAtom("WX_GL_FORWARD_COMPAT"); rt.addInt(WX_GL_FORWARD_COMPAT); + rt.addTupleCount(2); +#else + rt.addAtom("WX_GL_FORWARD_COMPAT"); rt.addAtom("undefined"); + rt.addTupleCount(2); +#endif +#if wxCHECK_VERSION(3,1,0) + rt.addAtom("WX_GL_FRAMEBUFFER_SRGB"); rt.addInt(WX_GL_FRAMEBUFFER_SRGB); + rt.addTupleCount(2); +#else + rt.addAtom("WX_GL_FRAMEBUFFER_SRGB"); rt.addAtom("undefined"); + rt.addTupleCount(2); +#endif +#if wxCHECK_VERSION(3,1,0) + rt.addAtom("WX_GL_LOSE_ON_RESET"); rt.addInt(WX_GL_LOSE_ON_RESET); + rt.addTupleCount(2); +#else + rt.addAtom("WX_GL_LOSE_ON_RESET"); rt.addAtom("undefined"); + rt.addTupleCount(2); +#endif +#if wxCHECK_VERSION(3,0,3) + rt.addAtom("WX_GL_MAJOR_VERSION"); rt.addInt(WX_GL_MAJOR_VERSION); + rt.addTupleCount(2); +#else + rt.addAtom("WX_GL_MAJOR_VERSION"); rt.addAtom("undefined"); + rt.addTupleCount(2); +#endif +#if wxCHECK_VERSION(3,0,3) + rt.addAtom("WX_GL_MINOR_VERSION"); rt.addInt(WX_GL_MINOR_VERSION); + rt.addTupleCount(2); +#else + rt.addAtom("WX_GL_MINOR_VERSION"); rt.addAtom("undefined"); + rt.addTupleCount(2); +#endif +#if wxCHECK_VERSION(3,1,0) + rt.addAtom("WX_GL_NO_RESET_NOTIFY"); rt.addInt(WX_GL_NO_RESET_NOTIFY); + rt.addTupleCount(2); +#else + rt.addAtom("WX_GL_NO_RESET_NOTIFY"); rt.addAtom("undefined"); + rt.addTupleCount(2); +#endif +#if wxCHECK_VERSION(3,1,0) + rt.addAtom("WX_GL_RELEASE_FLUSH"); rt.addInt(WX_GL_RELEASE_FLUSH); + rt.addTupleCount(2); +#else + rt.addAtom("WX_GL_RELEASE_FLUSH"); rt.addAtom("undefined"); + rt.addTupleCount(2); +#endif +#if wxCHECK_VERSION(3,1,0) + rt.addAtom("WX_GL_RELEASE_NONE"); rt.addInt(WX_GL_RELEASE_NONE); + rt.addTupleCount(2); +#else + rt.addAtom("WX_GL_RELEASE_NONE"); rt.addAtom("undefined"); + rt.addTupleCount(2); +#endif +#if wxCHECK_VERSION(3,1,0) + rt.addAtom("WX_GL_RESET_ISOLATION"); rt.addInt(WX_GL_RESET_ISOLATION); + rt.addTupleCount(2); +#else + rt.addAtom("WX_GL_RESET_ISOLATION"); rt.addAtom("undefined"); + rt.addTupleCount(2); +#endif +#if wxCHECK_VERSION(3,1,0) + rt.addAtom("WX_GL_ROBUST_ACCESS"); rt.addInt(WX_GL_ROBUST_ACCESS); + rt.addTupleCount(2); +#else + rt.addAtom("WX_GL_ROBUST_ACCESS"); rt.addAtom("undefined"); + rt.addTupleCount(2); +#endif +#if wxCHECK_VERSION(3,0,0) + rt.addAtom("WX_GL_SAMPLES"); rt.addInt(WX_GL_SAMPLES); + rt.addTupleCount(2); +#else + rt.addAtom("WX_GL_SAMPLES"); rt.addAtom("undefined"); + rt.addTupleCount(2); +#endif +#if wxCHECK_VERSION(3,0,0) + rt.addAtom("WX_GL_SAMPLE_BUFFERS"); rt.addInt(WX_GL_SAMPLE_BUFFERS); + rt.addTupleCount(2); +#else + rt.addAtom("WX_GL_SAMPLE_BUFFERS"); rt.addAtom("undefined"); + rt.addTupleCount(2); +#endif rt.addAtom("wxBLACK"); rt.add(*(wxBLACK)); rt.addTupleCount(2); rt.addAtom("wxBLACK_BRUSH"); rt.addRef(getRef((void *)wxBLACK_BRUSH,memenv),"wxBrush"); @@ -611,7 +716,14 @@ void WxeApp::init_nonconsts(wxeMemEnv *memenv, ErlDrvTermData caller) { rt.addTupleCount(2); rt.addAtom("wxWHITE_PEN"); rt.addRef(getRef((void *)wxWHITE_PEN,memenv),"wxPen"); rt.addTupleCount(2); - rt.endList(293); +#if wxCHECK_VERSION(3,1,0) + rt.addAtom("wx_GL_COMPAT_PROFILE"); rt.addInt(wx_GL_COMPAT_PROFILE); + rt.addTupleCount(2); +#else + rt.addAtom("wx_GL_COMPAT_PROFILE"); rt.addAtom("undefined"); + rt.addTupleCount(2); +#endif + rt.endList(309); rt.addTupleCount(2); rt.send(); } diff --git a/lib/wx/doc/src/Makefile b/lib/wx/doc/src/Makefile index a76740adf1..c132c628f7 100644 --- a/lib/wx/doc/src/Makefile +++ b/lib/wx/doc/src/Makefile @@ -46,9 +46,13 @@ XML_NOTES_FILES = notes.xml BOOK_FILES = book.xml XML_FILES = \ - $(BOOK_FILES) $(XML_CHAPTER_FILES) \ - $(XML_PART_FILES) $(XML_REF3_FILES) \ - $(XML_NOTES_FILES) $(XML_APPLICATION_FILES) + $(BOOK_FILES) \ + $(XML_PART_FILES) $(XML_NOTES_FILES) + +XML_GEN_FILES = \ + $(XML_CHAPTER_FILES:%=$(XMLDIR)/%) \ + $(XML_REF3_FILES:%=$(XMLDIR)/%) \ + $(XML_APPLICATION_FILES:%=$(XMLDIR)/%) # ---------------------------------------------------- INFO_FILE = ../../info @@ -93,22 +97,26 @@ gifs: $(GIF_FILES:%=$(HTMLDIR)/%) xml: $(XML_REF3_FILES) $(XML_CHAPTER_FILES) ref_man.xml: ref_man.xml.src - @echo Preparing ref_man.xml - @cat ref_man.xml.src > ref_man.xml + @echo Preparing $@ + @cat ref_man.xml.src > $@ @for d in $(ModsNoExt); do \ - echo " <xi:include href=\"$$d.xml\"/>" >> ref_man.xml ; \ + echo " <xi:include href=\"$$d.xml\"/>" >> $@ ; \ done - @echo "</application>" >> ref_man.xml - @echo + @echo "</application>" >> $@ -$(ErlMods:%.erl=%.xml): ../../src/$(@:%.xml=%.erl) - escript $(DOCGEN)/priv/bin/xml_from_edoc.escript -def vsn $(VSN) -preprocess true -sort_functions false ../../src/$(@:%.xml=%.erl) +$(ErlMods:%.erl=$(XMLDIR)/%.xml): + $(gen_verbose)escript $(DOCGEN)/priv/bin/xml_from_edoc.escript \ + -def vsn $(VSN) -preprocess true -sort_functions false -dir $(XMLDIR) \ + ../../src/$(@:$(XMLDIR)/%.xml=%.erl) -$(GenMods:%.erl=%.xml): ../../src/gen/$(@:%.xml=%.erl) - escript $(DOCGEN)/priv/bin/xml_from_edoc.escript -def vsn $(VSN) -i ../../src -preprocess true -sort_functions false ../../src/gen/$(@:%.xml=%.erl) +$(GenMods:%.erl=$(XMLDIR)/%.xml): + $(gen_verbose)escript $(DOCGEN)/priv/bin/xml_from_edoc.escript \ + -def vsn $(VSN) -i ../../src -preprocess true -sort_functions false -dir $(XMLDIR) \ + ../../src/gen/$(@:$(XMLDIR)/%.xml=%.erl) -$(XML_CHAPTER_FILES): ../overview.edoc - escript $(DOCGEN)/priv/bin/xml_from_edoc.escript -def vsn $(VSN) -chapter ../overview.edoc +$(XML_CHAPTER_FILES:%=$(XMLDIR)/%): ../overview.edoc + $(gen_verbose)escript $(DOCGEN)/priv/bin/xml_from_edoc.escript \ + -def vsn $(VSN) -chapter -dir $(XMLDIR) $< debug opt: @@ -118,7 +126,7 @@ clean clean_docs: rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) rm -f $(SPECDIR)/* rm -f errs core *~ ../html/edoc-info - rm -f $(XML_REF3_FILES) $(XML_CHAPTER_FILES) *.html + rm -f $(XML_GEN_FILES) *.html # ---------------------------------------------------- # Release Target diff --git a/lib/wx/include/wx.hrl b/lib/wx/include/wx.hrl index a14cc89cee..23f3b95403 100644 --- a/lib/wx/include/wx.hrl +++ b/lib/wx/include/wx.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2016. All Rights Reserved. +%% Copyright Ericsson AB 2008-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -373,6 +373,21 @@ -define(wxDefaultPosition, {-1,-1}). %% Global Variables +-define('WX_GL_CORE_PROFILE', wxe_util:get_const('WX_GL_CORE_PROFILE')). +-define('WX_GL_DEBUG', wxe_util:get_const('WX_GL_DEBUG')). +-define('WX_GL_ES2', wxe_util:get_const('WX_GL_ES2')). +-define('WX_GL_FORWARD_COMPAT', wxe_util:get_const('WX_GL_FORWARD_COMPAT')). +-define('WX_GL_FRAMEBUFFER_SRGB', wxe_util:get_const('WX_GL_FRAMEBUFFER_SRGB')). +-define('WX_GL_LOSE_ON_RESET', wxe_util:get_const('WX_GL_LOSE_ON_RESET')). +-define('WX_GL_MAJOR_VERSION', wxe_util:get_const('WX_GL_MAJOR_VERSION')). +-define('WX_GL_MINOR_VERSION', wxe_util:get_const('WX_GL_MINOR_VERSION')). +-define('WX_GL_NO_RESET_NOTIFY', wxe_util:get_const('WX_GL_NO_RESET_NOTIFY')). +-define('WX_GL_RELEASE_FLUSH', wxe_util:get_const('WX_GL_RELEASE_FLUSH')). +-define('WX_GL_RELEASE_NONE', wxe_util:get_const('WX_GL_RELEASE_NONE')). +-define('WX_GL_RESET_ISOLATION', wxe_util:get_const('WX_GL_RESET_ISOLATION')). +-define('WX_GL_ROBUST_ACCESS', wxe_util:get_const('WX_GL_ROBUST_ACCESS')). +-define('WX_GL_SAMPLES', wxe_util:get_const('WX_GL_SAMPLES')). +-define('WX_GL_SAMPLE_BUFFERS', wxe_util:get_const('WX_GL_SAMPLE_BUFFERS')). -define(wxBLACK, wxe_util:get_const(wxBLACK)). -define(wxBLACK_BRUSH, wxe_util:get_const(wxBLACK_BRUSH)). -define(wxBLACK_DASHED_PEN, wxe_util:get_const(wxBLACK_DASHED_PEN)). @@ -414,6 +429,7 @@ -define(wxWHITE, wxe_util:get_const(wxWHITE)). -define(wxWHITE_BRUSH, wxe_util:get_const(wxWHITE_BRUSH)). -define(wxWHITE_PEN, wxe_util:get_const(wxWHITE_PEN)). +-define(wx_GL_COMPAT_PROFILE, wxe_util:get_const(wx_GL_COMPAT_PROFILE)). %% Enum and defines % From class wxAuiManager diff --git a/lib/wx/test/wx_class_SUITE.erl b/lib/wx/test/wx_class_SUITE.erl index 6d314ab8fc..c610b9c4f4 100644 --- a/lib/wx/test/wx_class_SUITE.erl +++ b/lib/wx/test/wx_class_SUITE.erl @@ -618,7 +618,7 @@ lang_env() -> Env0 = os:getenv(), Env = [[R,"\n"]||R <- Env0], %%io:format("~p~n",[lists:sort(Env)]), - Opts = [global, multiline, {capture, all, list}], + Opts = [global, multiline, {capture, all, list}, unicode], format_env(re:run(Env, "LC_ALL.*", Opts)), format_env(re:run(Env, "^LANG.*=.*$", Opts)), ok. diff --git a/lib/xmerl/doc/src/Makefile b/lib/xmerl/doc/src/Makefile index 7d0b0b2392..94100910ef 100644 --- a/lib/xmerl/doc/src/Makefile +++ b/lib/xmerl/doc/src/Makefile @@ -54,10 +54,9 @@ XMERL_MODULES = \ XML_APPLICATION_FILES = ref_man.xml -XMERL_XML_FILES = $(XMERL_MODULES:=.xml) +XMERL_XML_FILES = $(XMERL_MODULES:%=$(XMLDIR)/%.xml) -XML_REF3_FILES = $(XMERL_XML_FILES) \ - xmerl_sax_parser.xml +XML_REF3_FILES = xmerl_sax_parser.xml XML_PART_FILES = \ part.xml @@ -65,9 +64,10 @@ XML_PART_FILES = \ XML_REF6_FILES = XML_CHAPTER_FILES = \ - xmerl_ug.xml \ notes.xml +XML_CHAPTER_GEN_FILES = \ + $(XMLDIR)/xmerl_ug.xml HTML_EXAMPLE_FILES = \ xmerl_examples.html \ @@ -89,6 +89,8 @@ XML_FILES= \ $(BOOK_FILES) $(XML_CHAPTER_FILES) \ $(XML_PART_FILES) $(XML_REF3_FILES) $(XML_APPLICATION_FILES) +XML_GEN_FILES = $(XMERL_XML_FILES) $(XML_CHAPTER_GEN_FILES) + # ---------------------------------------------------- INFO_FILE = ../../info @@ -97,7 +99,7 @@ HTML_FILES = $(XML_REF_MAN:%.xml=$(HTMLDIR)/%.html) \ $(XML_PART_FILES:%.xml=$(HTMLDIR)/%.html) -MAN3_FILES = $(XML_REF3_FILES:%.xml=$(MAN3DIR)/%.3) +MAN3_FILES = $(XML_REF3_FILES:%.xml=$(MAN3DIR)/%.3) $(XMERL_MODULES:%=$(MAN3DIR)/%.3) MAN6_FILES = $(XML_REF6_FILES:%_app.xml=$(MAN6DIR)/%.6) HTML_REF_MAN_FILE = $(HTMLDIR)/index.html @@ -126,7 +128,7 @@ pdf: $(TOP_PDF_FILE) html: gifs $(HTML_REF_MAN_FILE) $(XMERL_XML_FILES): - escript $(DOCGEN)/priv/bin/xml_from_edoc.escript $(XMERL_DIR)/$(@:%.xml=%.erl) + $(gen_verbose)escript $(DOCGEN)/priv/bin/xml_from_edoc.escript -dir $(XMLDIR) $(XMERL_DIR)/$(@:$(XMLDIR)/%.xml=%.erl) man: $(MAN3_FILES) $(MAN6_FILES) diff --git a/lib/xmerl/src/xmerl_xsd.erl b/lib/xmerl/src/xmerl_xsd.erl index a89b3159ec..d727084175 100644 --- a/lib/xmerl/src/xmerl_xsd.erl +++ b/lib/xmerl/src/xmerl_xsd.erl @@ -18,7 +18,7 @@ %% %CopyrightEnd% %% -%% @doc Interface module for XML Schema vlidation. +%% @doc Interface module for XML Schema validation. %% It handles the W3.org %% <a href="http://www.w3.org/XML/Schema#dev">specifications</a> %% of XML Schema second edition 28 october 2004. For an introduction to diff --git a/lib/xmerl/test/xmerl_SUITE_data/eventp/CMOM.xml b/lib/xmerl/test/xmerl_SUITE_data/eventp/CMOM.xml index 7c64046897..c2533248d1 100644 --- a/lib/xmerl/test/xmerl_SUITE_data/eventp/CMOM.xml +++ b/lib/xmerl/test/xmerl_SUITE_data/eventp/CMOM.xml @@ -10264,7 +10264,7 @@ Note! This attribute cannot have a value larger than for 'egressAtmPcr'.</descri <attribute name="ingressAtmMcr"> <description>Ingress minimum desired cell rate (cells/s). -Only positive vaues allowed. This attribute is mandatory only when serviceCategory is UBR+. +Only positive values allowed. This attribute is mandatory only when serviceCategory is UBR+. Note! When 'serviceCategory' is set to CBR or UBR this attribute has no relevance and the value submitted is ignored by the system. diff --git a/lib/xmerl/test/xmerl_SUITE_data/eventp/CelloMOM.xml b/lib/xmerl/test/xmerl_SUITE_data/eventp/CelloMOM.xml index 8f8cf54505..3b5d8ae2ad 100644 --- a/lib/xmerl/test/xmerl_SUITE_data/eventp/CelloMOM.xml +++ b/lib/xmerl/test/xmerl_SUITE_data/eventp/CelloMOM.xml @@ -10264,7 +10264,7 @@ Note! This attribute cannot have a value larger than for 'egressAtmPcr'.</descri <attribute name="ingressAtmMcr">
<description>Ingress minimum desired cell rate (cells/s).
-Only positive vaues allowed. This attribute is mandatory only when serviceCategory is UBR+.
+Only positive values allowed. This attribute is mandatory only when serviceCategory is UBR+.
Note! When 'serviceCategory' is set to CBR or UBR this attribute has no relevance and the value submitted is ignored by the system.
|
