diff options
Diffstat (limited to 'lib')
50 files changed, 2050 insertions, 317 deletions
diff --git a/lib/asn1/c_src/asn1_erl_nif.c b/lib/asn1/c_src/asn1_erl_nif.c index 53e3aa1678..317a464060 100644 --- a/lib/asn1/c_src/asn1_erl_nif.c +++ b/lib/asn1/c_src/asn1_erl_nif.c @@ -949,7 +949,7 @@ static int ber_decode_value(ErlNifEnv* env, ERL_NIF_TERM *value, unsigned char * } else if (in_buf[*ib_index] == ASN1_INDEFINITE_LENGTH) { (*ib_index)++; curr_head = enif_make_list(env, 0); - if (*ib_index+1 >= in_buf_len) { + if (*ib_index+1 >= in_buf_len || form == ASN1_PRIMITIVE) { return ASN1_INDEF_LEN_ERROR; } while (!(in_buf[*ib_index] == 0 && in_buf[*ib_index + 1] == 0)) { diff --git a/lib/asn1/test/asn1_SUITE_data/Constructed.asn b/lib/asn1/test/asn1_SUITE_data/Constructed.asn index 09a66d0c0d..bd49741726 100644 --- a/lib/asn1/test/asn1_SUITE_data/Constructed.asn +++ b/lib/asn1/test/asn1_SUITE_data/Constructed.asn @@ -1,6 +1,3 @@ - - - Constructed DEFINITIONS ::= BEGIN @@ -20,4 +17,7 @@ C ::= CHOICE { S3 ::= SEQUENCE {i INTEGER} S3ext ::= SEQUENCE {i INTEGER, ...} + +OS ::= OCTET STRING + END diff --git a/lib/asn1/test/ber_decode_error.erl b/lib/asn1/test/ber_decode_error.erl index 6fd2450c62..ef11717c45 100644 --- a/lib/asn1/test/ber_decode_error.erl +++ b/lib/asn1/test/ber_decode_error.erl @@ -61,6 +61,10 @@ run([]) -> (catch 'Constructed':decode('S', sub(<<40,16#80,1,1,255,0,0>>, 6))), {error,{asn1,{invalid_length,_}}} = (catch 'Constructed':decode('S', sub(<<40,16#80,1,1,255,0,0>>, 5))), + + %% A primitive must not be encoded with an indefinite length. + {error,{asn1,{invalid_length,_}}} = + (catch 'Constructed':decode('OS', <<4,128,4,3,97,98,99,0,0>>)), ok. sub(Bin, Bytes) -> diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl index 43eabb18d5..7037cdca73 100644 --- a/lib/common_test/src/ct_logs.erl +++ b/lib/common_test/src/ct_logs.erl @@ -129,7 +129,13 @@ datestr_from_dirname([]) -> close(Info, StartDir) -> %% close executes on the ct_util process, not on the logger process %% so we need to use a local copy of the log cache data - LogCacheBin = make_last_run_index(), + LogCacheBin = + case make_last_run_index() of + {error,_} -> % log server not responding + undefined; + LCB -> + LCB + end, put(ct_log_cache,LogCacheBin), Cache2File = fun() -> case get(ct_log_cache) of @@ -710,6 +716,7 @@ logger_loop(State) -> end end, if Importance >= (100-VLvl) -> + CtLogFd = State#logger_state.ct_log_fd, case get_groupleader(Pid, GL, State) of {tc_log,TCGL,TCGLs} -> case erlang:is_process_alive(TCGL) of @@ -723,14 +730,15 @@ logger_loop(State) -> %% Group leader is dead, so write to the %% CtLog or unexpected_io log instead unexpected_io(Pid,Category,Importance, - List,State), + List,CtLogFd), + logger_loop(State) end; {ct_log,_Fd,TCGLs} -> %% If category is ct_internal then write %% to ct_log, else write to unexpected_io %% log - unexpected_io(Pid,Category,Importance,List,State), + unexpected_io(Pid,Category,Importance,List,CtLogFd), logger_loop(State#logger_state{ tc_groupleaders = TCGLs}) end; @@ -803,16 +811,15 @@ logger_loop(State) -> ok end. -create_io_fun(FromPid, State) -> +create_io_fun(FromPid, CtLogFd) -> %% we have to build one io-list of all strings %% before printing, or other io printouts (made in %% parallel) may get printed between this header %% and footer - Fd = State#logger_state.ct_log_fd, fun({Str,Args}, IoList) -> case catch io_lib:format(Str,Args) of {'EXIT',_Reason} -> - io:format(Fd, "Logging fails! Str: ~p, Args: ~p~n", + io:format(CtLogFd, "Logging fails! Str: ~p, Args: ~p~n", [Str,Args]), %% stop the testcase, we need to see the fault exit(FromPid, {log_printout_error,Str,Args}), @@ -827,28 +834,53 @@ create_io_fun(FromPid, State) -> print_to_log(sync, FromPid, Category, TCGL, List, State) -> %% in some situations (exceptions), the printout is made from the %% test server IO process and there's no valid group leader to send to + CtLogFd = State#logger_state.ct_log_fd, if FromPid /= TCGL -> - IoFun = create_io_fun(FromPid, State), + IoFun = create_io_fun(FromPid, CtLogFd), io:format(TCGL,"~ts", [lists:foldl(IoFun, [], List)]); true -> - unexpected_io(FromPid,Category,?MAX_IMPORTANCE,List,State) + unexpected_io(FromPid,Category,?MAX_IMPORTANCE,List,CtLogFd) end, State; print_to_log(async, FromPid, Category, TCGL, List, State) -> %% in some situations (exceptions), the printout is made from the %% test server IO process and there's no valid group leader to send to + CtLogFd = State#logger_state.ct_log_fd, Printer = if FromPid /= TCGL -> - IoFun = create_io_fun(FromPid, State), + IoFun = create_io_fun(FromPid, CtLogFd), fun() -> test_server:permit_io(TCGL, self()), - io:format(TCGL, "~ts", [lists:foldl(IoFun, [], List)]) + + %% Since asynchronous io gets can get buffered if + %% the file system is slow, there is also a risk that + %% the group leader has terminated before we get to + %% the io:format(GL, ...) call. We check this and + %% print "expired" messages to the unexpected io + %% log instead (best we can do). + + case erlang:is_process_alive(TCGL) of + true -> + try io:format(TCGL, "~ts", + [lists:foldl(IoFun,[],List)]) of + _ -> ok + catch + _:terminated -> + unexpected_io(FromPid, Category, + ?MAX_IMPORTANCE, + List, CtLogFd) + end; + false -> + unexpected_io(FromPid, Category, + ?MAX_IMPORTANCE, + List, CtLogFd) + end end; true -> fun() -> - unexpected_io(FromPid,Category,?MAX_IMPORTANCE, - List,State) + unexpected_io(FromPid, Category, ?MAX_IMPORTANCE, + List, CtLogFd) end end, case State#logger_state.async_print_jobs of @@ -3149,12 +3181,11 @@ html_encoding(latin1) -> html_encoding(utf8) -> "utf-8". -unexpected_io(Pid,ct_internal,_Importance,List,State) -> - IoFun = create_io_fun(Pid,State), - io:format(State#logger_state.ct_log_fd, "~ts", - [lists:foldl(IoFun, [], List)]); -unexpected_io(Pid,_Category,_Importance,List,State) -> - IoFun = create_io_fun(Pid,State), +unexpected_io(Pid,ct_internal,_Importance,List,CtLogFd) -> + IoFun = create_io_fun(Pid,CtLogFd), + io:format(CtLogFd, "~ts", [lists:foldl(IoFun, [], List)]); +unexpected_io(Pid,_Category,_Importance,List,CtLogFd) -> + IoFun = create_io_fun(Pid,CtLogFd), Data = io_lib:format("~ts", [lists:foldl(IoFun, [], List)]), test_server_io:print_unexpected(Data), ok. diff --git a/lib/debugger/src/dbg_icmd.erl b/lib/debugger/src/dbg_icmd.erl index b1bf4ebecc..ce12c1beb3 100644 --- a/lib/debugger/src/dbg_icmd.erl +++ b/lib/debugger/src/dbg_icmd.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2013. All Rights Reserved. +%% Copyright Ericsson AB 1998-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -49,10 +49,6 @@ %% specifies if the process should break. %%-------------------------------------------------------------------- -%% Common Test adaptation -cmd({call_remote,0,ct_line,line,_As}, Bs, _Ieval) -> - Bs; - cmd(Expr, Bs, Ieval) -> cmd(Expr, Bs, get(next_break), Ieval). diff --git a/lib/debugger/src/dbg_ieval.erl b/lib/debugger/src/dbg_ieval.erl index 77297de0f3..96f9f91808 100644 --- a/lib/debugger/src/dbg_ieval.erl +++ b/lib/debugger/src/dbg_ieval.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2013. All Rights Reserved. +%% Copyright Ericsson AB 1998-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -457,11 +457,6 @@ do_eval_function(Mod, Fun, As0, Bs0, _, Ieval0) when is_function(Fun); exception(error, Reason, Bs0, Ieval0) end; -%% Common Test adaptation -do_eval_function(ct_line, line, As, Bs, extern, #ieval{level=Le}=Ieval) -> - debugged_cmd({apply,ct_line,line,As}, Bs, Ieval#ieval{level=Le+1}), - {value, ignore, Bs}; - do_eval_function(Mod, Name, As0, Bs0, Called, Ieval0) -> #ieval{level=Le,line=Li,top=Top} = Ieval0, trace(call, {Called, {Le,Li,Mod,Name,As0}}), @@ -896,11 +891,6 @@ expr({make_ext_fun,Line,MFA0}, Bs0, Ieval0) -> exception(error, badarg, Bs, Ieval, true) end; -%% Common test adaptation -expr({call_remote,0,ct_line,line,As0,Lc}, Bs0, Ieval0) -> - {As,_Bs} = eval_list(As0, Bs0, Ieval0), - eval_function(ct_line, line, As, Bs0, extern, Ieval0, Lc); - %% Local function call expr({local_call,Line,F,As0,Lc}, Bs0, #ieval{module=M} = Ieval0) -> Ieval = Ieval0#ieval{line=Line}, diff --git a/lib/debugger/src/int.erl b/lib/debugger/src/int.erl index 2755db64b8..908390ce50 100644 --- a/lib/debugger/src/int.erl +++ b/lib/debugger/src/int.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2013. All Rights Reserved. +%% Copyright Ericsson AB 1998-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -265,9 +265,6 @@ first_lines(Clauses) -> first_line({clause,_L,_Vars,_,Exprs}) -> first_line(Exprs); -%% Common Test adaptation -first_line([{call_remote,0,ct_line,line,_As}|Exprs]) -> - first_line(Exprs); first_line([Expr|_Exprs]) -> % Expr = {Op, Line, ..varying no of args..} element(2, Expr). diff --git a/lib/dialyzer/src/dialyzer_utils.erl b/lib/dialyzer/src/dialyzer_utils.erl index 4e2ec67b35..5297a3a7b4 100644 --- a/lib/dialyzer/src/dialyzer_utils.erl +++ b/lib/dialyzer/src/dialyzer_utils.erl @@ -98,7 +98,7 @@ get_abstract_code_from_src(File) -> {'ok', abstract_code()} | {'error', [string()]}. get_abstract_code_from_src(File, Opts) -> - case compile:file(File, [to_pp, binary|Opts]) of + case compile:noenv_file(File, [to_pp, binary|Opts]) of error -> {error, []}; {error, Errors, _} -> {error, format_errors(Errors)}; {ok, _, AbstrCode} -> {ok, AbstrCode} @@ -173,7 +173,7 @@ get_core_from_abstract_code(AbstrCode, Opts) -> AbstrCode1 = cleanup_parse_transforms(AbstrCode), %% Remove parse_transforms (and other options) from compile options. Opts2 = cleanup_compile_options(Opts), - try compile:forms(AbstrCode1, Opts2 ++ src_compiler_opts()) of + try compile:noenv_forms(AbstrCode1, Opts2 ++ src_compiler_opts()) of {ok, _, Core} -> {ok, Core}; _What -> error catch @@ -444,21 +444,18 @@ cleanup_parse_transforms([]) -> -spec cleanup_compile_options([compile:option()]) -> [compile:option()]. +cleanup_compile_options(Opts) -> + lists:filter(fun keep_compile_option/1, Opts). + %% Using abstract, not asm or core. -cleanup_compile_options([from_asm|Opts]) -> - Opts; -cleanup_compile_options([asm|Opts]) -> - Opts; -cleanup_compile_options([from_core|Opts]) -> - Opts; -%% The parse transform will already have been applied, may cause problems if it -%% is re-applied. -cleanup_compile_options([{parse_transform, _}|Opts]) -> - Opts; -cleanup_compile_options([Other|Opts]) -> - [Other|cleanup_compile_options(Opts)]; -cleanup_compile_options([]) -> - []. +keep_compile_option(from_asm) -> false; +keep_compile_option(asm) -> false; +keep_compile_option(from_core) -> false; +%% The parse transform will already have been applied, may cause +%% problems if it is re-applied. +keep_compile_option({parse_transform, _}) -> false; +keep_compile_option(warnings_as_errors) -> false; +keep_compile_option(_) -> true. -spec format_errors([{module(), string()}]) -> [string()]. diff --git a/lib/eldap/test/eldap_connections_SUITE.erl b/lib/eldap/test/eldap_connections_SUITE.erl index 4c8aa9c2cf..c5460fef09 100644 --- a/lib/eldap/test/eldap_connections_SUITE.erl +++ b/lib/eldap/test/eldap_connections_SUITE.erl @@ -27,27 +27,59 @@ all() -> [ - tcp_connection, - tcp_inet6_connection, - tcp_connection_option, - tcp_inet6_connection_option + {group, v4}, + {group, v6} + ]. + + +init_per_group(v4, Config) -> + [{listen_opts, []}, + {listen_host, "localhost"}, + {connect_opts, []} + | Config]; +init_per_group(v6, Config) -> + {ok, Hostname} = inet:gethostname(), + case lists:member(list_to_atom(Hostname), ct:get_config(ipv6_hosts,[])) of + true -> + [{listen_opts, [inet6]}, + {listen_host, "::"}, + {connect_opts, [{tcpopts,[inet6]}]} + | Config]; + false -> + {skip, io_lib:format("~p is not an ipv6_host",[Hostname])} + end. + + +end_per_group(_GroupName, Config) -> + Config. + + +groups() -> + [{v4, [], [tcp_connection, tcp_connection_option]}, + {v6, [], [tcp_connection, tcp_connection_option]} ]. init_per_suite(Config) -> Config. + end_per_suite(_Config) -> ok. init_per_testcase(_TestCase, Config) -> - {ok,Sl} = gen_tcp:listen(0,[]), - {ok,Sl6} = gen_tcp:listen(0,[inet6]), - [{listen_socket,Sl}, {listen_socket6,Sl6} | Config]. + case gen_tcp:listen(0, proplists:get_value(listen_opts,Config)) of + {ok,LSock} -> + {ok,{_,Port}} = inet:sockname(LSock), + [{listen_socket,LSock}, + {listen_port,Port} + | Config]; + Other -> + {fail, Other} + end. + end_per_testcase(_TestCase, Config) -> - catch gen_tcp:close( proplists:get_value(listen_socket, Config) ), - catch gen_tcp:close( proplists:get_value(listen_socket6, Config) ), - ok. + catch gen_tcp:close( proplists:get_value(listen_socket, Config) ). %%%================================================================ %%% @@ -55,35 +87,26 @@ end_per_testcase(_TestCase, Config) -> %%% %%%---------------------------------------------------------------- tcp_connection(Config) -> - do_tcp_connection(Config, listen_socket, "localhost", []). - -tcp_inet6_connection(Config) -> - do_tcp_connection(Config, listen_socket6, "::", [{tcpopts,[inet6]}]). - - -do_tcp_connection(Config, SockKey, Host, Opts) -> - Sl = proplists:get_value(SockKey, Config), - {ok,{_,Port}} = inet:sockname(Sl), + Host = proplists:get_value(listen_host, Config), + Port = proplists:get_value(listen_port, Config), + Opts = proplists:get_value(connect_opts, Config), case eldap:open([Host], [{port,Port}|Opts]) of {ok,_H} -> + Sl = proplists:get_value(listen_socket, Config), case gen_tcp:accept(Sl,1000) of {ok,_S} -> ok; {error,timeout} -> ct:fail("server side accept timeout",[]) end; Other -> ct:fail("eldap:open failed: ~p",[Other]) end. - -%%%---------------------------------------------------------------- -tcp_connection_option(Config) -> - do_tcp_connection_option(Config, listen_socket, "localhost", []). -tcp_inet6_connection_option(Config) -> - do_tcp_connection_option(Config, listen_socket6, "::", [{tcpopts,[inet6]}]). - -do_tcp_connection_option(Config, SockKey, Host, Opts) -> - Sl = proplists:get_value(SockKey, Config), - {ok,{_,Port}} = inet:sockname(Sl), +%%%---------------------------------------------------------------- +tcp_connection_option(Config) -> + Host = proplists:get_value(listen_host, Config), + Port = proplists:get_value(listen_port, Config), + Opts = proplists:get_value(connect_opts, Config), + Sl = proplists:get_value(listen_socket, Config), %% Make an option value to test. The option must be implemented on all %% platforms that we test on. Must check what the default value is @@ -95,7 +118,7 @@ do_tcp_connection_option(Config, SockKey, Host, Opts) -> end, case catch eldap:open([Host], - [{port,Port},{tcpopts,[{linger,TestLinger}]}|Opts]) of + [{port,Port},{tcpopts,[{linger,TestLinger}]}|Opts]) of {ok,H} -> case gen_tcp:accept(Sl,1000) of {ok,_} -> @@ -122,5 +145,3 @@ do_tcp_connection_option(Config, SockKey, Host, Opts) -> Other -> ct:fail("eldap:open failed: ~p",[Other]) end. - -%%%---------------------------------------------------------------- diff --git a/lib/inets/src/ftp/ftp.erl b/lib/inets/src/ftp/ftp.erl index 5674599ac5..8e51b1be5a 100644 --- a/lib/inets/src/ftp/ftp.erl +++ b/lib/inets/src/ftp/ftp.erl @@ -60,6 +60,7 @@ -define(DATA_ACCEPT_TIMEOUT, infinity). -define(DEFAULT_MODE, passive). -define(PROGRESS_DEFAULT, ignore). +-define(FTP_EXT_DEFAULT, false). %% Internal Constants -define(FTP_PORT, 21). @@ -94,7 +95,8 @@ ipfamily, % inet | inet6 | inet6fb4 progress = ignore, % ignore | pid() dtimeout = ?DATA_ACCEPT_TIMEOUT, % non_neg_integer() | infinity - tls_upgrading_data_connection = false + tls_upgrading_data_connection = false, + ftp_extension = ?FTP_EXT_DEFAULT }). @@ -969,6 +971,8 @@ start_options(Options) -> %% timeout %% dtimeout %% progress +%% ftp_extension + open_options(Options) -> ?fcrt("open_options", [{options, Options}]), ValidateMode = @@ -1013,6 +1017,11 @@ open_options(Options) -> (_) -> false end, + ValidateFtpExtension = + fun(true) -> true; + (false) -> true; + (_) -> false + end, ValidOptions = [{mode, ValidateMode, false, ?DEFAULT_MODE}, {host, ValidateHost, true, ehost}, @@ -1020,7 +1029,8 @@ open_options(Options) -> {ipfamily, ValidateIpFamily, false, inet}, {timeout, ValidateTimeout, false, ?CONNECTION_TIMEOUT}, {dtimeout, ValidateDTimeout, false, ?DATA_ACCEPT_TIMEOUT}, - {progress, ValidateProgress, false, ?PROGRESS_DEFAULT}], + {progress, ValidateProgress, false, ?PROGRESS_DEFAULT}, + {ftp_extension, ValidateFtpExtension, false, ?FTP_EXT_DEFAULT}], validate_options(Options, ValidOptions, []). tls_options(Options) -> @@ -1174,12 +1184,14 @@ handle_call({_, {open, ip_comm, Opts}}, From, State) -> DTimeout = key_search(dtimeout, Opts, ?DATA_ACCEPT_TIMEOUT), Progress = key_search(progress, Opts, ignore), IpFamily = key_search(ipfamily, Opts, inet), + FtpExt = key_search(ftp_extension, Opts, ?FTP_EXT_DEFAULT), State2 = State#state{client = From, mode = Mode, progress = progress(Progress), ipfamily = IpFamily, - dtimeout = DTimeout}, + dtimeout = DTimeout, + ftp_extension = FtpExt}, ?fcrd("handle_call(open) -> setup ctrl connection with", [{host, Host}, {port, Port}, {timeout, Timeout}]), @@ -1202,11 +1214,13 @@ handle_call({_, {open, ip_comm, Host, Opts}}, From, State) -> Timeout = key_search(timeout, Opts, ?CONNECTION_TIMEOUT), DTimeout = key_search(dtimeout, Opts, ?DATA_ACCEPT_TIMEOUT), Progress = key_search(progress, Opts, ignore), + FtpExt = key_search(ftp_extension, Opts, ?FTP_EXT_DEFAULT), State2 = State#state{client = From, mode = Mode, progress = progress(Progress), - dtimeout = DTimeout}, + dtimeout = DTimeout, + ftp_extension = FtpExt}, case setup_ctrl_connection(Host, Port, Timeout, State2) of {ok, State3, WaitTimeout} -> @@ -1785,7 +1799,8 @@ handle_ctrl_result({pos_compl, Lines}, ipfamily = inet, client = From, caller = {setup_data_connection, Caller}, - timeout = Timeout} = State) -> + timeout = Timeout, + ftp_extension = false} = State) -> {_, [?LEFT_PAREN | Rest]} = lists:splitwith(fun(?LEFT_PAREN) -> false; (_) -> true end, Lines), @@ -1806,6 +1821,28 @@ handle_ctrl_result({pos_compl, Lines}, {noreply,State#state{client = undefined, caller = undefined}} end; +handle_ctrl_result({pos_compl, Lines}, + #state{mode = passive, + ipfamily = inet, + client = From, + caller = {setup_data_connection, Caller}, + csock = CSock, + timeout = Timeout, + ftp_extension = true} = State) -> + + [_, PortStr | _] = lists:reverse(string:tokens(Lines, "|")), + {ok, {IP, _}} = peername(CSock), + + ?DBG('<--data tcp connect to ~p:~p, Caller=~p~n',[IP,PortStr,Caller]), + case connect(IP, list_to_integer(PortStr), Timeout, State) of + {ok, _, Socket} -> + handle_caller(State#state{caller = Caller, dsock = {tcp, Socket}}); + {error, _Reason} = Error -> + gen_server:reply(From, Error), + {noreply, State#state{client = undefined, caller = undefined}} + end; + + %% FTP server does not support passive mode: try to fallback on active mode handle_ctrl_result(_, #state{mode = passive, @@ -2157,7 +2194,8 @@ setup_ctrl_connection(Host, Port, Timeout, State) -> setup_data_connection(#state{mode = active, caller = Caller, - csock = CSock} = State) -> + csock = CSock, + ftp_extension = FtpExt} = State) -> case (catch sockname(CSock)) of {ok, {{_, _, _, _, _, _, _, _} = IP, _}} -> {ok, LSock} = @@ -2174,11 +2212,18 @@ setup_data_connection(#state{mode = active, {ok, LSock} = gen_tcp:listen(0, [{ip, IP}, {active, false}, binary, {packet, 0}]), {ok, Port} = inet:port(LSock), - {IP1, IP2, IP3, IP4} = IP, - {Port1, Port2} = {Port div 256, Port rem 256}, - send_ctrl_message(State, - mk_cmd("PORT ~w,~w,~w,~w,~w,~w", - [IP1, IP2, IP3, IP4, Port1, Port2])), + case FtpExt of + false -> + {IP1, IP2, IP3, IP4} = IP, + {Port1, Port2} = {Port div 256, Port rem 256}, + send_ctrl_message(State, + mk_cmd("PORT ~w,~w,~w,~w,~w,~w", + [IP1, IP2, IP3, IP4, Port1, Port2])); + true -> + IpAddress = inet_parse:ntoa(IP), + Cmd = mk_cmd("EPRT |1|~s|~p|", [IpAddress, Port]), + send_ctrl_message(State, Cmd) + end, activate_ctrl_connection(State), {noreply, State#state{caller = {setup_data_connection, {LSock, Caller}}}} @@ -2191,9 +2236,17 @@ setup_data_connection(#state{mode = passive, ipfamily = inet6, {noreply, State#state{caller = {setup_data_connection, Caller}}}; setup_data_connection(#state{mode = passive, ipfamily = inet, - caller = Caller} = State) -> + caller = Caller, + ftp_extension = false} = State) -> send_ctrl_message(State, mk_cmd("PASV", [])), activate_ctrl_connection(State), + {noreply, State#state{caller = {setup_data_connection, Caller}}}; + +setup_data_connection(#state{mode = passive, ipfamily = inet, + caller = Caller, + ftp_extension = true} = State) -> + send_ctrl_message(State, mk_cmd("EPSV", [])), + activate_ctrl_connection(State), {noreply, State#state{caller = {setup_data_connection, Caller}}}. connect(Host, Port, Timeout, #state{ipfamily = inet = IpFam}) -> diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl index d152d9f0be..0bbd40d656 100644 --- a/lib/inets/src/http_client/httpc_handler.erl +++ b/lib/inets/src/http_client/httpc_handler.erl @@ -350,7 +350,7 @@ handle_call(#request{address = Addr} = Request, _, {reply, ok, State0#state{keep_alive = NewKeepAlive, session = NewSession}}; undefined -> - %% Note: tcp-message reciving has already been + %% Note: tcp-message receiving has already been %% activated by handle_pipeline/2. ?hcrd("no current request", []), cancel_timer(Timers#timers.queue_timer, @@ -632,7 +632,7 @@ handle_info({timeout, RequestId}, handle_info(timeout_queue, State = #state{request = undefined}) -> {stop, normal, State}; -%% Timing was such as the pipeline_timout was not canceled! +%% Timing was such as the queue_timeout was not canceled! handle_info(timeout_queue, #state{timers = Timers} = State) -> {noreply, State#state{timers = Timers#timers{queue_timer = undefined}}}; diff --git a/lib/kernel/src/application_master.erl b/lib/kernel/src/application_master.erl index bc15b5a7de..7cdbe31ab2 100644 --- a/lib/kernel/src/application_master.erl +++ b/lib/kernel/src/application_master.erl @@ -103,9 +103,9 @@ call(AppMaster, Req) -> %%% The reason for not using the logical structrure is that %%% the application start function is synchronous, and %%% that the AM is GL. This means that if AM executed the start -%%% function, and this function uses spawn_request/1 -%%% or io, deadlock would occur. Therefore, this function is -%%% executed by the process X. Also, AM needs three loops; +%%% function, and this function uses io, deadlock would occur. +%%% Therefore, this function is executed by the process X. +%%% Also, AM needs three loops; %%% init_loop (waiting for the start function to return) %%% main_loop %%% terminate_loop (waiting for the process to die) diff --git a/lib/observer/doc/src/observer_ug.xml b/lib/observer/doc/src/observer_ug.xml index 3aeaf1997a..62f99c5210 100644 --- a/lib/observer/doc/src/observer_ug.xml +++ b/lib/observer/doc/src/observer_ug.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2011</year><year>2013</year> + <year>2011</year><year>2014</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -54,9 +54,6 @@ impact only the active viewer is updated and the other views will be updated when activated. </p> - <note> - <p>Only R15B nodes can be observed.</p> - </note> <p> In general the mouse buttons behaves as expected, use left click to select objects, right click to pop up a menu with most used diff --git a/lib/odbc/configure.in b/lib/odbc/configure.in index ea5c51965f..0cfcb9964b 100644 --- a/lib/odbc/configure.in +++ b/lib/odbc/configure.in @@ -136,7 +136,7 @@ AC_SUBST(THR_LIBS) odbc_lib_link_success=no AC_SUBST(TARGET_FLAGS) case $host_os in - darwin1[[0-2]].*|darwin[[0-9]].*) + darwin1[[0-4]].*|darwin[[0-9]].*) TARGET_FLAGS="-DUNIX" if test ! -d "$with_odbc" || test "$with_odbc" = "yes"; then ODBC_LIB= -L"/usr/lib" diff --git a/lib/parsetools/include/leexinc.hrl b/lib/parsetools/include/leexinc.hrl index dbbb688d2d..938aef58f9 100644 --- a/lib/parsetools/include/leexinc.hrl +++ b/lib/parsetools/include/leexinc.hrl @@ -36,8 +36,8 @@ string(Ics0, L0, Tcs, Ts) -> string_cont(Ics1, L1, yyaction(A, Alen, Tcs, L0), Ts); {reject,_Alen,Tlen,_Ics1,L1,_S1} -> % After a non-accepting state {error,{L0,?MODULE,{illegal,yypre(Tcs, Tlen+1)}},L1}; - {A,Alen,_Tlen,_Ics1,L1,_S1} -> - string_cont(yysuf(Tcs, Alen), L1, yyaction(A, Alen, Tcs, L0), Ts) + {A,Alen,_Tlen,_Ics1,_L1,_S1} -> + string_cont(yysuf(Tcs, Alen), L0, yyaction(A, Alen, Tcs, L0), Ts) end. %% string_cont(RestChars, Line, Token, Tokens) @@ -105,8 +105,8 @@ token(S0, Ics0, L0, Tcs, Tlen0, Tline, A0, Alen0) -> {reject,_Alen1,Tlen1,Ics1,L1,_S1} -> % No token match Error = {Tline,?MODULE,{illegal,yypre(Tcs, Tlen1+1)}}, {done,{error,Error,L1},Ics1}; - {A1,Alen1,_Tlen1,_Ics1,L1,_S1} -> % Use last accept match - token_cont(yysuf(Tcs, Alen1), L1, yyaction(A1, Alen1, Tcs, Tline)) + {A1,Alen1,_Tlen1,_Ics1,_L1,_S1} -> % Use last accept match + token_cont(yysuf(Tcs, Alen1), L0, yyaction(A1, Alen1, Tcs, Tline)) end. %% token_cont(RestChars, Line, Token) @@ -177,9 +177,9 @@ tokens(S0, Ics0, L0, Tcs, Tlen0, Tline, Ts, A0, Alen0) -> %% Skip rest of tokens. Error = {L1,?MODULE,{illegal,yypre(Tcs, Tlen1+1)}}, skip_tokens(yysuf(Tcs, Tlen1+1), L1, Error); - {A1,Alen1,_Tlen1,_Ics1,L1,_S1} -> + {A1,Alen1,_Tlen1,_Ics1,_L1,_S1} -> Token = yyaction(A1, Alen1, Tcs, Tline), - tokens_cont(yysuf(Tcs, Alen1), L1, Token, Ts) + tokens_cont(yysuf(Tcs, Alen1), L0, Token, Ts) end. %% tokens_cont(RestChars, Line, Token, Tokens) diff --git a/lib/parsetools/test/leex_SUITE.erl b/lib/parsetools/test/leex_SUITE.erl index eb15bebf63..6d2afe061e 100644 --- a/lib/parsetools/test/leex_SUITE.erl +++ b/lib/parsetools/test/leex_SUITE.erl @@ -43,8 +43,8 @@ file/1, compile/1, syntax/1, pt/1, man/1, ex/1, ex2/1, not_yet/1, - - otp_10302/1, otp_11286/1, unicode/1]). + line_wrap/1, + otp_10302/1, otp_11286/1, unicode/1]). % Default timetrap timeout (set in init_per_testcase). -define(default_timeout, ?t:minutes(1)). @@ -61,12 +61,13 @@ end_per_testcase(_Case, Config) -> suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [{group, checks}, {group, examples}]. + [{group, checks}, {group, examples}, {group, bugs}]. groups() -> [{checks, [], [file, compile, syntax]}, {examples, [], [pt, man, ex, ex2, not_yet, unicode]}, - {tickets, [], [otp_10302, otp_11286]}]. + {tickets, [], [otp_10302, otp_11286]}, + {bugs, [], [line_wrap]}]. init_per_suite(Config) -> Config. @@ -871,6 +872,48 @@ scan_token_1({more, Cont}, [C | Cs], Fun, Loc, Rs) -> %% End of ex2 +line_wrap(doc) -> "Much more examples."; +line_wrap(suite) -> []; +line_wrap(Config) when is_list(Config) -> + Xrl = + <<" +Definitions. +Rules. +[a]+[\\n]*= : {token, {first, TokenLine}}. +[a]+ : {token, {second, TokenLine}}. +[\\s\\r\\n\\t]+ : skip_token. +Erlang code. + ">>, + Dir = ?privdir, + XrlFile = filename:join(Dir, "test_line_wrap.xrl"), + ?line ok = file:write_file(XrlFile, Xrl), + ErlFile = filename:join(Dir, "test_line_wrap.erl"), + {ok, _} = leex:file(XrlFile, []), + {ok, _} = compile:file(ErlFile, [{outdir,Dir}]), + code:purge(test_line_wrap), + AbsFile = filename:rootname(ErlFile, ".erl"), + code:load_abs(AbsFile, test_line_wrap), + fun() -> + S = "aaa\naaa", + {ok,[{second,1},{second,2}],2} = test_line_wrap:string(S) + end(), + fun() -> + S = "aaa\naaa", + {ok,[{second,3},{second,4}],4} = test_line_wrap:string(S, 3) + end(), + fun() -> + {done,{ok,{second,1},1},"\na"} = test_line_wrap:token([], "a\na"), + {more,Cont1} = test_line_wrap:token([], "\na"), + {done,{ok,{second,2},2},eof} = test_line_wrap:token(Cont1, eof) + end(), + fun() -> + {more,Cont1} = test_line_wrap:tokens([], "a\na"), + {done,{ok,[{second,1},{second,2}],2},eof} = test_line_wrap:tokens(Cont1, eof) + end(), + ok. + +%% End of line_wrap + not_yet(doc) -> "Not yet implemented."; not_yet(suite) -> []; diff --git a/lib/ssh/doc/src/notes.xml b/lib/ssh/doc/src/notes.xml index 467e2ab27e..f3db05192e 100644 --- a/lib/ssh/doc/src/notes.xml +++ b/lib/ssh/doc/src/notes.xml @@ -29,6 +29,53 @@ <file>notes.xml</file> </header> +<section><title>Ssh 3.0.8</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixes of login blocking after port scanning.</p> + <p> + Own Id: OTP-12247 Aux Id: seq12726 </p> + </item> + </list> + </section> + +</section> + +<section><title>Ssh 3.0.7</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Add option sftp_vsn to SFTP</p> + <p> + Own Id: OTP-12227</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Fix option user_interaction to work as expected. When + password authentication is implemented with ssh + keyboard-interactive method and the password is already + supplied, so that we do not need to query user, then + connections should succeed even though user_interaction + option is set to false.</p> + <p> + Own Id: OTP-11329 Aux Id: seq12420, seq12335 </p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 3.0.6</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssh/doc/src/ssh_sftp.xml b/lib/ssh/doc/src/ssh_sftp.xml index e55d092fe2..f1091e9eca 100644 --- a/lib/ssh/doc/src/ssh_sftp.xml +++ b/lib/ssh/doc/src/ssh_sftp.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2005</year><year>2013</year> + <year>2005</year><year>2014</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -81,6 +81,17 @@ <p>The timeout is passed to the ssh_channel start function, and defaults to infinity.</p> </item> + <tag> + <p><c><![CDATA[{sftp_vsn, integer()}]]></c></p> + </tag> + <item> + <p> + Desired SFTP protocol version. + The actual version will be the minimum of + the desired version and the maximum supported + versions by the SFTP server. + </p> + </item> </taglist> <p>All other options are directly passed to <seealso marker="ssh">ssh:connect/3</seealso> or ignored if a diff --git a/lib/ssh/src/Makefile b/lib/ssh/src/Makefile index 53c755d3cb..90d71107ad 100644 --- a/lib/ssh/src/Makefile +++ b/lib/ssh/src/Makefile @@ -65,6 +65,7 @@ MODULES= \ ssh_cli \ ssh_file \ ssh_io \ + ssh_info \ ssh_math \ ssh_message \ ssh_no_io \ diff --git a/lib/ssh/src/ssh.app.src b/lib/ssh/src/ssh.app.src index e0a51b3574..4ad55b34ca 100644 --- a/lib/ssh/src/ssh.app.src +++ b/lib/ssh/src/ssh.app.src @@ -23,6 +23,7 @@ sshd_sup, ssh_file, ssh_io, + ssh_info, ssh_math, ssh_no_io, ssh_server_key_api, diff --git a/lib/ssh/src/ssh.appup.src b/lib/ssh/src/ssh.appup.src index 1917c95f5a..600c01454c 100644 --- a/lib/ssh/src/ssh.appup.src +++ b/lib/ssh/src/ssh.appup.src @@ -19,9 +19,49 @@ {"%VSN%", [ + {"3.0.7", [{load_module, ssh_auth, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_acceptor, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_channel, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_connection, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_connection_handler, soft_purge, soft_purge, []}, + {load_module, ssh_info, soft_purge, soft_purge, []}, + {load_module, ssh_message, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_io, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_sftp, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_xfer, soft_purge, soft_purge, [ssh_connection_handler]}]}, + {"3.0.6", [{load_module, ssh_auth, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_acceptor, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_channel, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_connection, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_connection_handler, soft_purge, soft_purge, []}, + {load_module, ssh_info, soft_purge, soft_purge, []}, + {load_module, ssh_message, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_io, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_sftp, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_xfer, soft_purge, soft_purge, [ssh_connection_handler]}]}, {<<".*">>, [{restart_application, ssh}]} ], [ + {"3.0.7", [{load_module, ssh_auth, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_acceptor, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_channel, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_connection, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_connection_handler, soft_purge, soft_purge, []}, + {load_module, ssh_info, soft_purge, soft_purge, []}, + {load_module, ssh_message, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_io, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_sftp, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_xfer, soft_purge, soft_purge, [ssh_connection_handler]}]}, + {"3.0.6", [{load_module, ssh_auth, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_acceptor, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_channel, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_connection, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_connection_handler, soft_purge, soft_purge, []}, + {load_module, ssh_info, soft_purge, soft_purge, []}, + {load_module, ssh_message, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_io, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_sftp, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_xfer, soft_purge, soft_purge, [ssh_connection_handler]}]}, {<<".*">>, [{restart_application, ssh}]} ] }. diff --git a/lib/ssh/src/ssh_acceptor.erl b/lib/ssh/src/ssh_acceptor.erl index 7302196674..6c443eeb9c 100644 --- a/lib/ssh/src/ssh_acceptor.erl +++ b/lib/ssh/src/ssh_acceptor.erl @@ -22,7 +22,8 @@ -module(ssh_acceptor). %% Internal application API --export([start_link/5]). +-export([start_link/5, + number_of_connections/1]). %% spawn export -export([acceptor_init/6, acceptor_loop/6]). @@ -140,5 +141,6 @@ handle_error(Reason) -> number_of_connections(SystemSup) -> length([X || {R,X,supervisor,[ssh_subsystem_sup]} <- supervisor:which_children(SystemSup), + is_pid(X), is_reference(R) ]). diff --git a/lib/ssh/src/ssh_auth.erl b/lib/ssh/src/ssh_auth.erl index 45fd907383..45c4d52d7e 100644 --- a/lib/ssh/src/ssh_auth.erl +++ b/lib/ssh/src/ssh_auth.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2013. All Rights Reserved. +%% Copyright Ericsson AB 2008-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -119,8 +119,7 @@ init_userauth_request_msg(#ssh{opts = Opts} = Ssh) -> false -> FirstAlg = proplists:get_value(public_key_alg, Opts, ?PREFERRED_PK_ALG), SecondAlg = other_alg(FirstAlg), - AllowUserInt = proplists:get_value(user_interaction, Opts, true), - Prefs = method_preference(FirstAlg, SecondAlg, AllowUserInt), + Prefs = method_preference(FirstAlg, SecondAlg), ssh_transport:ssh_packet(Msg, Ssh#ssh{user = User, userauth_preference = Prefs, userauth_methods = none, @@ -130,15 +129,13 @@ init_userauth_request_msg(#ssh{opts = Opts} = Ssh) -> case length(Algs) =:= 2 of true -> SecondAlg = other_alg(FirstAlg), - AllowUserInt = proplists:get_value(user_interaction, Opts, true), - Prefs = method_preference(FirstAlg, SecondAlg, AllowUserInt), + Prefs = method_preference(FirstAlg, SecondAlg), ssh_transport:ssh_packet(Msg, Ssh#ssh{user = User, userauth_preference = Prefs, userauth_methods = none, service = "ssh-connection"}); _ -> - AllowUserInt = proplists:get_value(user_interaction, Opts, true), - Prefs = method_preference(FirstAlg, AllowUserInt), + Prefs = method_preference(FirstAlg), ssh_transport:ssh_packet(Msg, Ssh#ssh{user = User, userauth_preference = Prefs, userauth_methods = none, @@ -187,9 +184,8 @@ handle_userauth_request(#ssh_msg_service_request{name = handle_userauth_request(#ssh_msg_userauth_request{user = User, service = "ssh-connection", method = "password", - data = Data}, _, + data = <<?FALSE, ?UINT32(Sz), BinPwd:Sz/binary>>}, _, #ssh{opts = Opts} = Ssh) -> - <<_:8, ?UINT32(Sz), BinPwd:Sz/binary>> = Data, Password = unicode:characters_to_list(BinPwd), case check_password(User, Password, Opts) of true -> @@ -204,6 +200,27 @@ handle_userauth_request(#ssh_msg_userauth_request{user = User, handle_userauth_request(#ssh_msg_userauth_request{user = User, service = "ssh-connection", + method = "password", + data = <<?TRUE, + _/binary + %% ?UINT32(Sz1), OldBinPwd:Sz1/binary, + %% ?UINT32(Sz2), NewBinPwd:Sz2/binary + >> + }, _, + Ssh) -> + %% Password change without us having sent SSH_MSG_USERAUTH_PASSWD_CHANGEREQ (because we never do) + %% RFC 4252 says: + %% SSH_MSG_USERAUTH_FAILURE without partial success - The password + %% has not been changed. Either password changing was not supported, + %% or the old password was bad. + + {not_authorized, {User, {error,"Password change not supported"}}, + ssh_transport:ssh_packet(#ssh_msg_userauth_failure{ + authentications = "", + partial_success = false}, Ssh)}; + +handle_userauth_request(#ssh_msg_userauth_request{user = User, + service = "ssh-connection", method = "none"}, _, #ssh{userauth_supported_methods = Methods} = Ssh) -> {not_authorized, {User, undefined}, @@ -256,15 +273,12 @@ handle_userauth_info_request( data = Data}, IoCb, #ssh{opts = Opts} = Ssh) -> PromptInfos = decode_keyboard_interactive_prompts(NumPrompts,Data), - Resps = keyboard_interact_get_responses(IoCb, Opts, + Responses = keyboard_interact_get_responses(IoCb, Opts, Name, Instr, PromptInfos), - RespBin = list_to_binary( - lists:map(fun(S) -> <<?STRING(list_to_binary(S))>> end, - Resps)), {ok, ssh_transport:ssh_packet( #ssh_msg_userauth_info_response{num_responses = NumPrompts, - data = RespBin}, Ssh)}. + data = Responses}, Ssh)}. handle_userauth_info_response(#ssh_msg_userauth_info_response{}, _Auth) -> @@ -276,25 +290,16 @@ handle_userauth_info_response(#ssh_msg_userauth_info_response{}, %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- -method_preference(Alg1, Alg2, true) -> +method_preference(Alg1, Alg2) -> [{"publickey", ?MODULE, publickey_msg, [Alg1]}, {"publickey", ?MODULE, publickey_msg,[Alg2]}, {"password", ?MODULE, password_msg, []}, {"keyboard-interactive", ?MODULE, keyboard_interactive_msg, []} - ]; -method_preference(Alg1, Alg2, false) -> - [{"publickey", ?MODULE, publickey_msg, [Alg1]}, - {"publickey", ?MODULE, publickey_msg,[Alg2]}, - {"password", ?MODULE, password_msg, []} ]. -method_preference(Alg1, true) -> +method_preference(Alg1) -> [{"publickey", ?MODULE, publickey_msg, [Alg1]}, {"password", ?MODULE, password_msg, []}, {"keyboard-interactive", ?MODULE, keyboard_interactive_msg, []} - ]; -method_preference(Alg1, false) -> - [{"publickey", ?MODULE, publickey_msg, [Alg1]}, - {"password", ?MODULE, password_msg, []} ]. user_name(Opts) -> @@ -362,35 +367,29 @@ build_sig_data(SessionId, User, Service, KeyBlob, Alg) -> algorithm_string('ssh-rsa') -> "ssh-rsa"; algorithm_string('ssh-dss') -> - "ssh-dss". + "ssh-dss". decode_keyboard_interactive_prompts(_NumPrompts, Data) -> ssh_message:decode_keyboard_interactive_prompts(Data, []). keyboard_interact_get_responses(IoCb, Opts, Name, Instr, PromptInfos) -> NumPrompts = length(PromptInfos), - case proplists:get_value(keyboard_interact_fun, Opts) of - undefined when NumPrompts == 1 -> - %% Special case/fallback for just one prompt - %% (assumed to be the password prompt) - case proplists:get_value(password, Opts) of - undefined -> keyboard_interact(IoCb, Name, Instr, PromptInfos, Opts); - PW -> [PW] - end; - undefined -> - keyboard_interact(IoCb, Name, Instr, PromptInfos, Opts); - KbdInteractFun -> - Prompts = lists:map(fun({Prompt, _Echo}) -> Prompt end, - PromptInfos), - case KbdInteractFun(Name, Instr, Prompts) of - Rs when length(Rs) == NumPrompts -> - Rs; - Rs -> - erlang:error({mismatching_number_of_responses, - {got,Rs}, - {expected,NumPrompts}}) - end - end. + keyboard_interact_get_responses(proplists:get_value(user_interaction, Opts, true), + proplists:get_value(keyboard_interact_fun, Opts), + proplists:get_value(password, Opts, undefined), IoCb, Name, + Instr, PromptInfos, Opts, NumPrompts). + +keyboard_interact_get_responses(_, undefined, Password, _, _, _, _, _, + 1) when Password =/= undefined -> + [Password]; %% Password auth implemented with keyboard-interaction and passwd is known +keyboard_interact_get_responses(_, _, _, _, _, _, _, _, 0) -> + [""]; +keyboard_interact_get_responses(false, undefined, undefined, _, _, _, [Prompt|_], Opts, _) -> + ssh_no_io:read_line(Prompt, Opts); %% Throws error as keyboard interaction is not allowed +keyboard_interact_get_responses(true, undefined, _,IoCb, Name, Instr, PromptInfos, Opts, _) -> + keyboard_interact(IoCb, Name, Instr, PromptInfos, Opts); +keyboard_interact_get_responses(true, Fun, _, Name, Instr, PromptInfos, _, _, NumPrompts) -> + keyboard_interact_fun(Fun, Name, Instr, PromptInfos, NumPrompts). keyboard_interact(IoCb, Name, Instr, Prompts, Opts) -> if Name /= "" -> IoCb:format("~s", [Name]); @@ -404,6 +403,21 @@ keyboard_interact(IoCb, Name, Instr, Prompts, Opts) -> end, Prompts). +keyboard_interact_fun(KbdInteractFun, Name, Instr, PromptInfos, NumPrompts) -> + Prompts = lists:map(fun({Prompt, _Echo}) -> Prompt end, + PromptInfos), + case KbdInteractFun(Name, Instr, Prompts) of + Rs when length(Rs) == NumPrompts -> + Rs; + Rs -> + throw({mismatching_number_of_responses, + {got,Rs}, + {expected, NumPrompts}, + #ssh_msg_disconnect{code = ?SSH_DISCONNECT_SERVICE_NOT_AVAILABLE, + description = "User interaction failed", + language = "en"}}) + end. + other_alg('ssh-rsa') -> 'ssh-dss'; other_alg('ssh-dss') -> diff --git a/lib/ssh/src/ssh_channel.erl b/lib/ssh/src/ssh_channel.erl index 508ae637cf..5c24f362b1 100644 --- a/lib/ssh/src/ssh_channel.erl +++ b/lib/ssh/src/ssh_channel.erl @@ -67,7 +67,8 @@ %% Internal application API -export([cache_create/0, cache_lookup/2, cache_update/2, cache_delete/1, cache_delete/2, cache_foldl/3, - cache_find/2]). + cache_find/2, + get_print_info/1]). -record(state, { cm, @@ -190,6 +191,14 @@ init([Options]) -> %% {stop, Reason, State} %% Description: Handling call messages %%-------------------------------------------------------------------- +handle_call(get_print_info, _From, State) -> + Reply = + {{State#state.cm, + State#state.channel_id}, + io_lib:format('CB=~p',[State#state.channel_cb]) + }, + {reply, Reply, State}; + handle_call(Request, From, #state{channel_cb = Module, channel_state = ChannelState} = State) -> try Module:handle_call(Request, From, ChannelState) of @@ -333,6 +342,9 @@ cache_find(ChannelPid, Cache) -> Channel end. +get_print_info(Pid) -> + call(Pid, get_print_info, 1000). + %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- diff --git a/lib/ssh/src/ssh_connection.erl b/lib/ssh/src/ssh_connection.erl index 33849f4527..f3ff9ae67a 100644 --- a/lib/ssh/src/ssh_connection.erl +++ b/lib/ssh/src/ssh_connection.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2013. All Rights Reserved. +%% Copyright Ericsson AB 2008-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -107,9 +107,15 @@ shell(ConnectionHandler, ChannelId) -> %% Description: Executes a predefined subsystem. %%-------------------------------------------------------------------- subsystem(ConnectionHandler, ChannelId, SubSystem, TimeOut) -> - ssh_connection_handler:request(ConnectionHandler, self(), - ChannelId, "subsystem", - true, [?string(SubSystem)], TimeOut). + case ssh_connection_handler:request(ConnectionHandler, self(), + ChannelId, "subsystem", + true, [?string(SubSystem)], TimeOut) of + success -> success; + failure -> failure; + {error,timeout} -> {error,timeout}; + _ -> failure + end. + %%-------------------------------------------------------------------- -spec send(pid(), channel_id(), iodata()) -> ok | {error, closed}. diff --git a/lib/ssh/src/ssh_connection_handler.erl b/lib/ssh/src/ssh_connection_handler.erl index 4fbc5d0ae2..fa107be1b1 100644 --- a/lib/ssh/src/ssh_connection_handler.erl +++ b/lib/ssh/src/ssh_connection_handler.erl @@ -41,11 +41,13 @@ global_request/4, send/5, send_eof/2, info/1, info/2, connection_info/2, channel_info/3, adjust_window/3, close/2, stop/1, renegotiate/1, renegotiate_data/1, - start_connection/4]). + start_connection/4, + get_print_info/1]). %% gen_fsm callbacks -export([hello/2, kexinit/2, key_exchange/2, new_keys/2, - userauth/2, connected/2]). + userauth/2, connected/2, + error/2]). -export([init/1, handle_event/3, handle_sync_event/4, handle_info/3, terminate/3, format_status/2, code_change/4]). @@ -171,9 +173,23 @@ init([Role, Socket, SshOpts]) -> State#state{ssh_params = Ssh}) catch _:Error -> - gen_fsm:enter_loop(?MODULE, [], error, {Error, State0}) + gen_fsm:enter_loop(?MODULE, [], error, {Error, State}) end. +%% Temporary fix for the Nessus error. SYN-> <-SYNACK ACK-> RST-> ? +error(_Event, {Error,State=#state{}}) -> + case Error of + {badmatch,{error,enotconn}} -> + %% {error,enotconn} probably from inet:peername in + %% init_ssh(server,..)/5 called from init/1 + {stop, {shutdown,"TCP connenction to server was prematurely closed by the client"}, State}; + _ -> + {stop, {shutdown,{init,Error}}, State} + end; +error(Event, State) -> + %% State deliberately not checked beeing #state. This is a panic-clause... + {stop, {shutdown,{init,{spurious_error,Event}}}, State}. + %%-------------------------------------------------------------------- -spec open_channel(pid(), string(), iodata(), integer(), integer(), timeout()) -> {open, channel_id()} | {error, term()}. @@ -240,6 +256,9 @@ send_eof(ConnectionHandler, ChannelId) -> %%-------------------------------------------------------------------- -spec connection_info(pid(), [atom()]) -> proplists:proplist(). %%-------------------------------------------------------------------- +get_print_info(ConnectionHandler) -> + sync_send_all_state_event(ConnectionHandler, get_print_info, 1000). + connection_info(ConnectionHandler, Options) -> sync_send_all_state_event(ConnectionHandler, {connection_info, Options}). @@ -550,7 +569,7 @@ connected({#ssh_msg_kexinit{}, _Payload} = Event, State) -> %%-------------------------------------------------------------------- handle_event(#ssh_msg_disconnect{description = Desc} = DisconnectMsg, _StateName, #state{} = State) -> - handle_disconnect(DisconnectMsg, State), + handle_disconnect(peer, DisconnectMsg, State), {stop, {shutdown, Desc}, State}; handle_event(#ssh_msg_ignore{}, StateName, State) -> @@ -758,6 +777,20 @@ handle_sync_event({recv_window, ChannelId}, _From, StateName, end, {reply, Reply, StateName, next_packet(State)}; +handle_sync_event(get_print_info, _From, StateName, State) -> + Reply = + try + {inet:sockname(State#state.socket), + inet:peername(State#state.socket) + } + of + {{ok,Local}, {ok,Remote}} -> {{Local,Remote},io_lib:format("statename=~p",[StateName])}; + _ -> {{"-",0},"-"} + catch + _:_ -> {{"?",0},"?"} + end, + {reply, Reply, StateName, State}; + handle_sync_event({connection_info, Options}, _From, StateName, State) -> Info = ssh_info(Options, State, []), {reply, Info, StateName, State}; @@ -936,6 +969,10 @@ terminate(normal, _, #state{transport_cb = Transport, (catch Transport:close(Socket)), ok; +terminate({shutdown,{init,Reason}}, StateName, State) -> + error_logger:info_report(io_lib:format("Erlang ssh in connection handler init: ~p~n",[Reason])), + terminate(normal, StateName, State); + %% Terminated by supervisor terminate(shutdown, StateName, #state{ssh_params = Ssh0} = State) -> DisconnectMsg = @@ -951,8 +988,10 @@ terminate({shutdown, #ssh_msg_disconnect{} = Msg}, StateName, {SshPacket, Ssh} = ssh_transport:ssh_packet(Msg, Ssh0), send_msg(SshPacket, State), terminate(normal, StateName, State#state{ssh_params = Ssh}); + terminate({shutdown, _}, StateName, State) -> terminate(normal, StateName, State); + terminate(Reason, StateName, #state{ssh_params = Ssh0, starter = _Pid, connection_state = Connection} = State) -> terminate_subsytem(Connection), @@ -965,6 +1004,7 @@ terminate(Reason, StateName, #state{ssh_params = Ssh0, starter = _Pid, send_msg(SshPacket, State), terminate(normal, StateName, State#state{ssh_params = Ssh}). + terminate_subsytem(#connection{system_supervisor = SysSup, sub_system_supervisor = SubSysSup}) when is_pid(SubSysSup) -> ssh_system_sup:stop_subsystem(SysSup, SubSysSup); @@ -1161,7 +1201,10 @@ send_all_state_event(FsmPid, Event) -> gen_fsm:send_all_state_event(FsmPid, Event). sync_send_all_state_event(FsmPid, Event) -> - try gen_fsm:sync_send_all_state_event(FsmPid, Event, infinity) + sync_send_all_state_event(FsmPid, Event, infinity). + +sync_send_all_state_event(FsmPid, Event, Timeout) -> + try gen_fsm:sync_send_all_state_event(FsmPid, Event, Timeout) catch exit:{noproc, _} -> {error, closed}; @@ -1258,13 +1301,23 @@ generate_event(<<?BYTE(Byte), _/binary>> = Msg, StateName, generate_event(Msg, StateName, State0, EncData) -> Event = ssh_message:decode(Msg), State = generate_event_new_state(State0, EncData), - case Event of - #ssh_msg_kexinit{} -> - %% We need payload for verification later. - event({Event, Msg}, StateName, State); - _ -> - event(Event, StateName, State) - end. + try + case Event of + #ssh_msg_kexinit{} -> + %% We need payload for verification later. + event({Event, Msg}, StateName, State); + _ -> + event(Event, StateName, State) + end + catch + _:_ -> + DisconnectMsg = + #ssh_msg_disconnect{code = ?SSH_DISCONNECT_PROTOCOL_ERROR, + description = "Encountered unexpected input", + language = "en"}, + handle_disconnect(DisconnectMsg, State) + end. + handle_request(ChannelPid, ChannelId, Type, Data, WantReply, From, @@ -1442,17 +1495,27 @@ handle_ssh_packet(Length, StateName, #state{decoded_data_buffer = DecData0, handle_disconnect(DisconnectMsg, State0) end. -handle_disconnect(#ssh_msg_disconnect{description = Desc} = Msg, #state{connection_state = Connection0, - role = Role} = State0) -> +handle_disconnect(DisconnectMsg, State) -> + handle_disconnect(own, DisconnectMsg, State). + +handle_disconnect(#ssh_msg_disconnect{} = DisconnectMsg, State, Error) -> + handle_disconnect(own, DisconnectMsg, State, Error); +handle_disconnect(Type, #ssh_msg_disconnect{description = Desc} = Msg, #state{connection_state = Connection0, role = Role} = State0) -> {disconnect, _, {{replies, Replies}, Connection}} = ssh_connection:handle_msg(Msg, Connection0, Role), - State = send_replies(Replies, State0), + State = send_replies(disconnect_replies(Type, Msg, Replies), State0), {stop, {shutdown, Desc}, State#state{connection_state = Connection}}. -handle_disconnect(#ssh_msg_disconnect{description = Desc} = Msg, #state{connection_state = Connection0, - role = Role} = State0, ErrorMsg) -> + +handle_disconnect(Type, #ssh_msg_disconnect{description = Desc} = Msg, #state{connection_state = Connection0, + role = Role} = State0, ErrorMsg) -> {disconnect, _, {{replies, Replies}, Connection}} = ssh_connection:handle_msg(Msg, Connection0, Role), - State = send_replies(Replies, State0), + State = send_replies(disconnect_replies(Type, Msg, Replies), State0), {stop, {shutdown, {Desc, ErrorMsg}}, State#state{connection_state = Connection}}. +disconnect_replies(own, Msg, Replies) -> + [{connection_reply, Msg} | Replies]; +disconnect_replies(peer, _, Replies) -> + Replies. + counterpart_versions(NumVsn, StrVsn, #ssh{role = server} = Ssh) -> Ssh#ssh{c_vsn = NumVsn , c_version = StrVsn}; counterpart_versions(NumVsn, StrVsn, #ssh{role = client} = Ssh) -> diff --git a/lib/ssh/src/ssh_info.erl b/lib/ssh/src/ssh_info.erl new file mode 100644 index 0000000000..9ed598b3ab --- /dev/null +++ b/lib/ssh/src/ssh_info.erl @@ -0,0 +1,193 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2008-2014. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%% +%%---------------------------------------------------------------------- +%% Purpose: Print some info of a running ssh aplication. +%%---------------------------------------------------------------------- + +-module(ssh_info). + +-compile(export_all). + +print() -> + try supervisor:which_children(ssh_sup) + of + _ -> + io:nl(), + print_general(), + io:nl(), + underline("Client part", $=), + print_clients(), + io:nl(), + underline("Server part", $=), + print_servers(), + io:nl(), + %% case os:type() of + %% {unix,_} -> + %% io:nl(), + %% underline("Linux part", $=), + %% underline("Listening"), + %% catch io:format(os:cmd("netstat -tpln")), + %% io:nl(), + %% underline("Other"), + %% catch io:format(os:cmd("netstat -tpn")); + %% _ -> ok + %% end, + underline("Supervisors", $=), + walk_sups(ssh_sup), + io:nl() + catch + _:_ -> + io:format("Ssh not found~n",[]) + end. + +%%%================================================================ +print_general() -> + {_Name, Slogan, Ver} = lists:keyfind(ssh,1,application:which_applications()), + underline(io_lib:format("~s ~s", [Slogan, Ver]), $=), + io:format('This printout is generated ~s. ~n',[datetime()]). + +%%%================================================================ +print_clients() -> + try + lists:foreach(fun print_client/1, supervisor:which_children(sshc_sup)) + catch + C:E -> + io:format('***FAILED: ~p:~p~n',[C,E]) + end. + +print_client({undefined,Pid,supervisor,[ssh_connection_handler]}) -> + {{Local,Remote},_Str} = ssh_connection_handler:get_print_info(Pid), + io:format(" Local=~s Remote=~s~n",[fmt_host_port(Local),fmt_host_port(Remote)]); +print_client(Other) -> + io:format(" [[Other 1: ~p]]~n",[Other]). + + +%%%================================================================ +print_servers() -> + try + lists:foreach(fun print_server/1, supervisor:which_children(sshd_sup)) + catch + C:E -> + io:format('***FAILED: ~p:~p~n',[C,E]) + end. + +print_server({{server,ssh_system_sup,LocalHost,LocalPort},Pid,supervisor,[ssh_system_sup]}) when is_pid(Pid) -> + io:format('Local=~s (~p children)~n',[fmt_host_port({LocalHost,LocalPort}), + ssh_acceptor:number_of_connections(Pid)]), + lists:foreach(fun print_system_sup/1, supervisor:which_children(Pid)); +print_server(Other) -> + io:format(" [[Other 2: ~p]]~n",[Other]). + +print_system_sup({Ref,Pid,supervisor,[ssh_subsystem_sup]}) when is_reference(Ref), + is_pid(Pid) -> + lists:foreach(fun print_channels/1, supervisor:which_children(Pid)); +print_system_sup({{ssh_acceptor_sup,LocalHost,LocalPort}, Pid,supervisor, [ssh_acceptor_sup]}) when is_pid(Pid) -> + io:format(" [Acceptor for ~s]~n",[fmt_host_port({LocalHost,LocalPort})]); +print_system_sup(Other) -> + io:format(" [[Other 3: ~p]]~n",[Other]). + +print_channels({{server,ssh_channel_sup,_,_},Pid,supervisor,[ssh_channel_sup]}) when is_pid(Pid) -> + lists:foreach(fun print_channel/1, supervisor:which_children(Pid)); +print_channels(Other) -> + io:format(" [[Other 4: ~p]]~n",[Other]). + + +print_channel({Ref,Pid,worker,[ssh_channel]}) when is_reference(Ref), + is_pid(Pid) -> + {{ConnManager,ChannelID}, Str} = ssh_channel:get_print_info(Pid), + {{Local,Remote},StrM} = ssh_connection_handler:get_print_info(ConnManager), + io:format(' ch ~p: ~s ~s',[ChannelID, StrM, Str]), + io:format(" Local=~s Remote=~s~n",[fmt_host_port(Local),fmt_host_port(Remote)]); +print_channel(Other) -> + io:format(" [[Other 5: ~p]]~n",[Other]). + +%%%================================================================ +-define(inc(N), (N+4)). + +walk_sups(StartPid) -> + io:format("Start at ~p, ~s.~n",[StartPid,dead_or_alive(StartPid)]), + walk_sups(children(StartPid), _Indent=?inc(0)). + +walk_sups([H={_,Pid,SupOrWorker,_}|T], Indent) -> + indent(Indent), io:format('~200p ~p is ~s~n',[H,Pid,dead_or_alive(Pid)]), + case SupOrWorker of + supervisor -> walk_sups(children(Pid), ?inc(Indent)); + _ -> ok + end, + walk_sups(T, Indent); +walk_sups([], _) -> + ok. + +dead_or_alive(Name) when is_atom(Name) -> + case whereis(Name) of + undefined -> + "**UNDEFINED**"; + Pid -> + dead_or_alive(Pid) + end; +dead_or_alive(Pid) when is_pid(Pid) -> + case process_info(Pid) of + undefined -> "**DEAD**"; + _ -> "alive" + end. + +indent(I) -> io:format('~*c',[I,$ ]). + +children(Pid) -> + Parent = self(), + Helper = spawn(fun() -> + Parent ! {self(),supervisor:which_children(Pid)} + end), + receive + {Helper,L} when is_list(L) -> + L + after + 2000 -> + catch exit(Helper, kill), + [] + end. + +%%%================================================================ +underline(Str) -> + underline(Str, $-). + +underline(Str, LineChar) -> + Len = lists:flatlength(Str), + io:format('~s~n',[Str]), + line(Len,LineChar). + +line(Len, Char) -> + io:format('~*c~n', [Len,Char]). + + +datetime() -> + {{YYYY,MM,DD}, {H,M,S}} = calendar:now_to_universal_time(now()), + lists:flatten(io_lib:format('~4w-~2..0w-~2..0w ~2..0w:~2..0w:~2..0w UTC',[YYYY,MM,DD, H,M,S])). + + +fmt_host_port({{A,B,C,D},Port}) -> io_lib:format('~p.~p.~p.~p:~p',[A,B,C,D,Port]); +fmt_host_port({Host,Port}) -> io_lib:format('~s:~p',[Host,Port]). + + + +nyi() -> + io:format('Not yet implemented~n',[]), + nyi. diff --git a/lib/ssh/src/ssh_io.erl b/lib/ssh/src/ssh_io.erl index 35336bce8b..97e2dee27a 100644 --- a/lib/ssh/src/ssh_io.erl +++ b/lib/ssh/src/ssh_io.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2013. All Rights Reserved. +%% Copyright Ericsson AB 2005-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -73,7 +73,9 @@ read_password(Prompt, Ssh) -> listify(A) when is_atom(A) -> atom_to_list(A); listify(L) when is_list(L) -> - L. + L; +listify(B) when is_binary(B) -> + binary_to_list(B). format(Fmt, Args) -> io:format(Fmt, Args). diff --git a/lib/ssh/src/ssh_message.erl b/lib/ssh/src/ssh_message.erl index 76b57cb995..66e7717095 100644 --- a/lib/ssh/src/ssh_message.erl +++ b/lib/ssh/src/ssh_message.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2013. All Rights Reserved. +%% Copyright Ericsson AB 2013-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -162,8 +162,15 @@ encode(#ssh_msg_userauth_info_request{ encode(#ssh_msg_userauth_info_response{ num_responses = Num, data = Data}) -> - ssh_bits:encode([?SSH_MSG_USERAUTH_INFO_RESPONSE, Num, Data], - [byte, uint32, '...']); + Responses = lists:map(fun("") -> + <<>>; + (Response) -> + ssh_bits:encode([Response], [string]) + end, Data), + Start = ssh_bits:encode([?SSH_MSG_USERAUTH_INFO_RESPONSE, Num], + [byte, uint32]), + iolist_to_binary([Start, Responses]); + encode(#ssh_msg_disconnect{ code = Code, description = Desc, @@ -498,6 +505,11 @@ erl_boolean(1) -> decode_kex_init(<<?BYTE(Bool), ?UINT32(X)>>, Acc, 0) -> list_to_tuple(lists:reverse([X, erl_boolean(Bool) | Acc])); +decode_kex_init(<<?BYTE(Bool)>>, Acc, 0) -> + %% The mandatory trailing UINT32 is missing. Assume the value it anyhow must have + %% See rfc 4253 7.1 + X = 0, + list_to_tuple(lists:reverse([X, erl_boolean(Bool) | Acc])); decode_kex_init(<<?UINT32(Len), Data:Len/binary, Rest/binary>>, Acc, N) -> Names = string:tokens(unicode:characters_to_list(Data), ","), decode_kex_init(Rest, [Names | Acc], N -1). diff --git a/lib/ssh/src/ssh_sftp.erl b/lib/ssh/src/ssh_sftp.erl index 0ea2366ac7..721146c509 100644 --- a/lib/ssh/src/ssh_sftp.erl +++ b/lib/ssh/src/ssh_sftp.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2013. All Rights Reserved. +%% Copyright Ericsson AB 2005-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -57,7 +57,8 @@ rep_buf = <<>>, req_id, req_list = [], %% {ReqId, Fun} - inf %% list of fileinf + inf, %% list of fileinf, + opts }). -record(fileinf, @@ -85,10 +86,11 @@ start_channel(Host) when is_list(Host) -> start_channel(Host, []). start_channel(Cm, Opts) when is_pid(Cm) -> Timeout = proplists:get_value(timeout, Opts, infinity), + {_, SftpOpts} = handle_options(Opts, [], []), case ssh_xfer:attach(Cm, []) of {ok, ChannelId, Cm} -> case ssh_channel:start(Cm, ChannelId, - ?MODULE, [Cm, ChannelId, Timeout]) of + ?MODULE, [Cm, ChannelId, SftpOpts]) of {ok, Pid} -> case wait_for_version_negotiation(Pid, Timeout) of ok -> @@ -108,11 +110,12 @@ start_channel(Cm, Opts) when is_pid(Cm) -> start_channel(Host, Opts) -> start_channel(Host, 22, Opts). start_channel(Host, Port, Opts) -> - Timeout = proplists:get_value(timeout, Opts, infinity), - case ssh_xfer:connect(Host, Port, proplists:delete(timeout, Opts)) of + {SshOpts, SftpOpts} = handle_options(Opts, [], []), + Timeout = proplists:get_value(timeout, SftpOpts, infinity), + case ssh_xfer:connect(Host, Port, SshOpts) of {ok, ChannelId, Cm} -> case ssh_channel:start(Cm, ChannelId, ?MODULE, [Cm, - ChannelId, Timeout]) of + ChannelId, SftpOpts]) of {ok, Pid} -> case wait_for_version_negotiation(Pid, Timeout) of ok -> @@ -392,7 +395,8 @@ write_file_loop(Pid, Handle, Pos, Bin, Remain, PacketSz, FileOpTimeout) -> %% %% Description: %%-------------------------------------------------------------------- -init([Cm, ChannelId, Timeout]) -> +init([Cm, ChannelId, Options]) -> + Timeout = proplists:get_value(timeout, Options, infinity), erlang:monitor(process, Cm), case ssh_connection:subsystem(Cm, ChannelId, "sftp", Timeout) of success -> @@ -401,7 +405,8 @@ init([Cm, ChannelId, Timeout]) -> {ok, #state{xf = Xf, req_id = 0, rep_buf = <<>>, - inf = new_inf()}}; + inf = new_inf(), + opts = Options}}; failure -> {stop, "server failed to start sftp subsystem"}; Error -> @@ -707,8 +712,9 @@ handle_ssh_msg({ssh_cm, _, {exit_status, ChannelId, Status}}, State0) -> %% %% Description: Handles channel messages %%-------------------------------------------------------------------- -handle_msg({ssh_channel_up, _, _}, #state{xf = Xf} = State) -> - ssh_xfer:protocol_version_request(Xf), +handle_msg({ssh_channel_up, _, _}, #state{opts = Options, xf = Xf} = State) -> + Version = proplists:get_value(sftp_vsn, Options, ?SSH_SFTP_PROTOCOL_VERSION), + ssh_xfer:protocol_version_request(Xf, Version), {ok, State}; %% Version negotiation timed out @@ -754,6 +760,15 @@ terminate(_Reason, State) -> %%==================================================================== %% Internal functions %%==================================================================== +handle_options([], Sftp, Ssh) -> + {Ssh, Sftp}; +handle_options([{timeout, _} = Opt | Rest], Sftp, Ssh) -> + handle_options(Rest, [Opt | Sftp], Ssh); +handle_options([{sftp_vsn, _} = Opt| Rest], Sftp, Ssh) -> + handle_options(Rest, [Opt | Sftp], Ssh); +handle_options([Opt | Rest], Sftp, Ssh) -> + handle_options(Rest, Sftp, [Opt | Ssh]). + call(Pid, Msg, TimeOut) -> ssh_channel:call(Pid, {{timeout, TimeOut}, Msg}, infinity). diff --git a/lib/ssh/src/ssh_xfer.erl b/lib/ssh/src/ssh_xfer.erl index 63d01fd9de..1881392db8 100644 --- a/lib/ssh/src/ssh_xfer.erl +++ b/lib/ssh/src/ssh_xfer.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2013. All Rights Reserved. +%% Copyright Ericsson AB 2005-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -28,7 +28,7 @@ rename/5, remove/3, mkdir/4, rmdir/3, realpath/3, extended/4, stat/4, fstat/4, lstat/4, setstat/4, readlink/3, fsetstat/4, symlink/4, - protocol_version_request/1, + protocol_version_request/2, xf_reply/2, xf_send_reply/3, xf_send_names/3, xf_send_name/4, xf_send_status/3, xf_send_status/4, xf_send_status/5, @@ -67,8 +67,8 @@ open_xfer(CM, Opts) -> Error end. -protocol_version_request(XF) -> - xf_request(XF, ?SSH_FXP_INIT, <<?UINT32(?SSH_SFTP_PROTOCOL_VERSION)>>). +protocol_version_request(XF, Version) -> + xf_request(XF, ?SSH_FXP_INIT, <<?UINT32(Version)>>). open(XF, ReqID, FileName, Access, Flags, Attrs) -> Vsn = XF#ssh_xfer.vsn, diff --git a/lib/ssh/test/property_test/ssh_eqc_client_server.erl b/lib/ssh/test/property_test/ssh_eqc_client_server.erl index cf895ae85e..123b48412b 100644 --- a/lib/ssh/test/property_test/ssh_eqc_client_server.erl +++ b/lib/ssh/test/property_test/ssh_eqc_client_server.erl @@ -32,6 +32,10 @@ -else. +%% Limit the testing time on CI server... this needs to be improved in % from total budget. +-define(TESTINGTIME(Prop), eqc:testing_time(30,Prop)). + + -include_lib("eqc/include/eqc.hrl"). -include_lib("eqc/include/eqc_statem.hrl"). -eqc_group_commands(true). @@ -75,7 +79,9 @@ -define(SUBSYSTEMS, ["echo1", "echo2", "echo3", "echo4"]). --define(SERVER_ADDRESS, { {127,1,1,1}, inet_port({127,1,1,1}) }). +-define(SERVER_ADDRESS, { {127,1,0,choose(1,254)}, % IP + choose(1024,65535) % Port + }). -define(SERVER_EXTRA_OPTIONS, [{parallel_login,bool()}] ). @@ -97,7 +103,7 @@ %% To be called as eqc:quickcheck( ssh_eqc_client_server:prop_seq() ). prop_seq() -> - do_prop_seq(?SSH_DIR). + ?TESTINGTIME(do_prop_seq(?SSH_DIR)). %% To be called from a common_test test suite prop_seq(CT_Config) -> @@ -105,9 +111,10 @@ prop_seq(CT_Config) -> do_prop_seq(DataDir) -> - ?FORALL(Cmds,commands(?MODULE, #state{data_dir=DataDir}), + setup_rsa(DataDir), + ?FORALL(Cmds,commands(?MODULE), begin - {H,Sf,Result} = run_commands(?MODULE,Cmds), + {H,Sf,Result} = run_commands(?MODULE,Cmds,[{data_dir,DataDir}]), present_result(?MODULE, Cmds, {H,Sf,Result}, Result==ok) end). @@ -116,33 +123,35 @@ full_path(SSHdir, CT_Config) -> SSHdir). %%%---- prop_parallel() -> - do_prop_parallel(?SSH_DIR). + ?TESTINGTIME(do_prop_parallel(?SSH_DIR)). %% To be called from a common_test test suite prop_parallel(CT_Config) -> do_prop_parallel(full_path(?SSH_DIR, CT_Config)). do_prop_parallel(DataDir) -> - ?FORALL(Cmds,parallel_commands(?MODULE, #state{data_dir=DataDir}), + setup_rsa(DataDir), + ?FORALL(Cmds,parallel_commands(?MODULE), begin - {H,Sf,Result} = run_parallel_commands(?MODULE,Cmds), + {H,Sf,Result} = run_parallel_commands(?MODULE,Cmds,[{data_dir,DataDir}]), present_result(?MODULE, Cmds, {H,Sf,Result}, Result==ok) end). %%%---- prop_parallel_multi() -> - do_prop_parallel_multi(?SSH_DIR). + ?TESTINGTIME(do_prop_parallel_multi(?SSH_DIR)). %% To be called from a common_test test suite prop_parallel_multi(CT_Config) -> do_prop_parallel_multi(full_path(?SSH_DIR, CT_Config)). do_prop_parallel_multi(DataDir) -> + setup_rsa(DataDir), ?FORALL(Repetitions,?SHRINK(1,[10]), - ?FORALL(Cmds,parallel_commands(?MODULE, #state{data_dir=DataDir}), + ?FORALL(Cmds,parallel_commands(?MODULE), ?ALWAYS(Repetitions, begin - {H,Sf,Result} = run_parallel_commands(?MODULE,Cmds), + {H,Sf,Result} = run_parallel_commands(?MODULE,Cmds,[{data_dir,DataDir}]), present_result(?MODULE, Cmds, {H,Sf,Result}, Result==ok) end))). @@ -151,14 +160,12 @@ do_prop_parallel_multi(DataDir) -> %%% called when using commands/1 initial_state() -> - S = initial_state(#state{}), - S#state{initialized=true}. + #state{}. %%% called when using commands/2 -initial_state(S) -> +initial_state(DataDir) -> application:stop(ssh), - ssh:start(), - setup_rsa(S#state.data_dir). + ssh:start(). %%%---------------- weight(S, ssh_send) -> 5*length([C || C<-S#state.channels, has_subsyst(C)]); @@ -172,7 +179,7 @@ weight(_S, _) -> 1. initial_state_pre(S) -> not S#state.initialized. -initial_state_args(S) -> [S]. +initial_state_args(_) -> [{var,data_dir}]. initial_state_next(S, _, _) -> S#state{initialized=true}. @@ -180,10 +187,17 @@ initial_state_next(S, _, _) -> S#state{initialized=true}. %%% Start a new daemon %%% Precondition: not more than ?MAX_NUM_SERVERS started +%%% This is a bit funny because we need to pick an IP address and Port to +%%% run the server on, but there is no way to atomically select a free Port! +%%% +%%% Therefore we just grab one IP-Port pair randomly and try to start the ssh server +%%% on that pair. If it fails, we just forget about it and goes on. Yes, it +%%% is a waste of cpu cycles, but at least it works! + ssh_server_pre(S) -> S#state.initialized andalso length(S#state.servers) < ?MAX_NUM_SERVERS. -ssh_server_args(S) -> [?SERVER_ADDRESS, S#state.data_dir, ?SERVER_EXTRA_OPTIONS]. +ssh_server_args(_) -> [?SERVER_ADDRESS, {var,data_dir}, ?SERVER_EXTRA_OPTIONS]. ssh_server({IP,Port}, DataDir, ExtraOptions) -> ok(ssh:daemon(IP, Port, @@ -194,8 +208,10 @@ ssh_server({IP,Port}, DataDir, ExtraOptions) -> | ExtraOptions ])). +ssh_server_post(_S, _Args, {error,eaddrinuse}) -> true; ssh_server_post(_S, _Args, Result) -> is_ok(Result). +ssh_server_next(S, {error,eaddrinuse}, _) -> S; ssh_server_next(S, Result, [{IP,Port},_,_]) -> S#state{servers=[#srvr{ref = Result, address = IP, @@ -241,15 +257,16 @@ do(Pid, Fun, Timeout) when is_function(Fun,0) -> ssh_open_connection_pre(S) -> S#state.servers /= []. -ssh_open_connection_args(S) -> [oneof(S#state.servers), S#state.data_dir]. +ssh_open_connection_args(S) -> [oneof(S#state.servers), {var,data_dir}]. ssh_open_connection(#srvr{address=Ip, port=Port}, DataDir) -> ok(ssh:connect(ensure_string(Ip), Port, [ {silently_accept_hosts, true}, {user_dir, user_dir(DataDir)}, - {user_interaction, false} - ])). + {user_interaction, false}, + {connect_timeout, 2000} + ], 2000)). ssh_open_connection_post(_S, _Args, Result) -> is_ok(Result). @@ -569,12 +586,6 @@ median(_) -> %%%================================================================ %%% The rest is taken and modified from ssh_test_lib.erl -inet_port(IpAddress)-> - {ok, Socket} = gen_tcp:listen(0, [{ip,IpAddress},{reuseaddr,true}]), - {ok, Port} = inet:port(Socket), - gen_tcp:close(Socket), - Port. - setup_rsa(Dir) -> erase_dir(system_dir(Dir)), erase_dir(user_dir(Dir)), diff --git a/lib/ssh/test/property_test/ssh_eqc_encode_decode.erl b/lib/ssh/test/property_test/ssh_eqc_encode_decode.erl index 34630bdc91..57ea2012c1 100644 --- a/lib/ssh/test/property_test/ssh_eqc_encode_decode.erl +++ b/lib/ssh/test/property_test/ssh_eqc_encode_decode.erl @@ -25,8 +25,6 @@ -proptest(eqc). -proptest([triq,proper]). --include_lib("ct_property_test.hrl"). - -ifndef(EQC). -ifndef(PROPER). -ifndef(TRIQ). diff --git a/lib/ssh/test/ssh_connection_SUITE.erl b/lib/ssh/test/ssh_connection_SUITE.erl index d226e5ba03..3c537d719c 100644 --- a/lib/ssh/test/ssh_connection_SUITE.erl +++ b/lib/ssh/test/ssh_connection_SUITE.erl @@ -37,6 +37,7 @@ all() -> [ {group, openssh_payload}, + start_subsystem_on_closed_channel, interrupted_send, start_shell, start_shell_exec, @@ -241,6 +242,32 @@ send_after_exit(Config) when is_list(Config) -> end. %%-------------------------------------------------------------------- +start_subsystem_on_closed_channel(Config) -> + PrivDir = ?config(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = ?config(data_dir, Config), + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"}, + {subsystems, [{"echo_n", {ssh_echo_server, [4000000]}}]}]), + + ConnectionRef = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_interaction, false}, + {user_dir, UserDir}]), + + {ok, ChannelId} = ssh_connection:session_channel(ConnectionRef, infinity), + + ok = ssh_connection:close(ConnectionRef, ChannelId), + + failure = ssh_connection:subsystem(ConnectionRef, ChannelId, "echo_n", infinity), + + ssh:close(ConnectionRef), + ssh:stop_daemon(Pid). + +%%-------------------------------------------------------------------- interrupted_send() -> [{doc, "Use a subsystem that echos n char and then sends eof to cause a channel exit partway through a large send."}]. diff --git a/lib/ssh/test/ssh_sftp_SUITE.erl b/lib/ssh/test/ssh_sftp_SUITE.erl index 56b1363b7a..4c46a1b1a8 100644 --- a/lib/ssh/test/ssh_sftp_SUITE.erl +++ b/lib/ssh/test/ssh_sftp_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2013. All Rights Reserved. +%% Copyright Ericsson AB 2005-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -65,7 +65,7 @@ groups() -> [{erlang_server, [], [open_close_file, open_close_dir, read_file, read_dir, write_file, rename_file, mk_rm_dir, remove_file, links, retrieve_attributes, set_attributes, async_read, - async_write, position, pos_read, pos_write]}, + async_write, position, pos_read, pos_write, version_option]}, {openssh_server, [], [open_close_file, open_close_dir, read_file, read_dir, write_file, rename_file, mk_rm_dir, remove_file, links, retrieve_attributes, set_attributes, async_read, @@ -111,6 +111,21 @@ init_per_testcase(sftp_nonexistent_subsystem, Config) -> ]), [{sftpd, Sftpd} | Config]; +init_per_testcase(version_option, Config) -> + prep(Config), + TmpConfig0 = lists:keydelete(watchdog, 1, Config), + TmpConfig = lists:keydelete(sftp, 1, TmpConfig0), + Dog = ct:timetrap(?default_timeout), + {_,Host, Port} = ?config(sftpd, Config), + {ok, ChannelPid, Connection} = + ssh_sftp:start_channel(Host, Port, + [{sftp_vsn, 3}, + {user, ?USER}, + {password, ?PASSWD}, + {user_interaction, false}, + {silently_accept_hosts, true}]), + Sftp = {ChannelPid, Connection}, + [{sftp, Sftp}, {watchdog, Dog} | TmpConfig]; init_per_testcase(Case, Config) -> prep(Config), TmpConfig0 = lists:keydelete(watchdog, 1, Config), @@ -447,6 +462,11 @@ sftp_nonexistent_subsystem(Config) when is_list(Config) -> {silently_accept_hosts, true}]). %%-------------------------------------------------------------------- +version_option() -> + [{doc, "Test API option sftp_vsn"}]. +version_option(Config) when is_list(Config) -> + open_close_dir(Config). +%%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- prep(Config) -> diff --git a/lib/ssh/vsn.mk b/lib/ssh/vsn.mk index 11f30e8d04..68544c1d0e 100644 --- a/lib/ssh/vsn.mk +++ b/lib/ssh/vsn.mk @@ -1,5 +1,5 @@ #-*-makefile-*- ; force emacs to enter makefile-mode -SSH_VSN = 3.0.6 +SSH_VSN = 3.0.8 APP_VSN = "ssh-$(SSH_VSN)" diff --git a/lib/ssl/doc/src/notes.xml b/lib/ssl/doc/src/notes.xml index 8643cd3745..62e9bd0165 100644 --- a/lib/ssl/doc/src/notes.xml +++ b/lib/ssl/doc/src/notes.xml @@ -25,7 +25,23 @@ <file>notes.xml</file> </header> <p>This document describes the changes made to the SSL application.</p> - <section><title>SSL 5.3.6</title> + <section><title>SSL 5.3.7</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Handle the fact that servers may send an empty SNI + extension to the client.</p> + <p> + Own Id: OTP-12198</p> + </item> + </list> + </section> + +</section> + +<section><title>SSL 5.3.6</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/ssl/src/ssl.appup.src b/lib/ssl/src/ssl.appup.src index 650901ef54..9d692379b4 100644 --- a/lib/ssl/src/ssl.appup.src +++ b/lib/ssl/src/ssl.appup.src @@ -1,6 +1,7 @@ %% -*- erlang -*- {"%VSN%", [ + {"5.3.6", [{load_module, ssl_handshake, soft_purge, soft_purge, [ssl_connection]}]}, {"5.3.5", [{load_module, ssl, soft_purge, soft_purge, [ssl_connection]}, {load_module, ssl_handshake, soft_purge, soft_purge, [ssl_certificate]}, {load_module, ssl_certificate, soft_purge, soft_purge, []}, @@ -12,6 +13,7 @@ {<<"3\\..*">>, [{restart_application, ssl}]} ], [ + {"5.3.6", [{load_module, ssl_handshake, soft_purge, soft_purge, [ssl_connection]}]}, {"5.3.5", [{load_module, ssl, soft_purge, soft_purge,[ssl_certificate]}, {load_module, ssl_handshake, soft_purge, soft_purge,[ssl_certificate]}, {load_module, ssl_certificate, soft_purge, soft_purge,[]}, diff --git a/lib/ssl/vsn.mk b/lib/ssl/vsn.mk index 404b71374f..da20ed8593 100644 --- a/lib/ssl/vsn.mk +++ b/lib/ssl/vsn.mk @@ -1 +1 @@ -SSL_VSN = 5.3.6 +SSL_VSN = 5.3.7 diff --git a/lib/stdlib/src/dets.erl b/lib/stdlib/src/dets.erl index 76e03bbfaa..a4bd45ea19 100644 --- a/lib/stdlib/src/dets.erl +++ b/lib/stdlib/src/dets.erl @@ -2839,17 +2839,22 @@ fsck_try(Fd, Tab, FH, Fname, SlotNumbers, Version) -> tempfile(Fname) -> Tmp = lists:concat([Fname, ".TMP"]), - tempfile(Tmp, 10). - -tempfile(Tmp, 0) -> - Tmp; -tempfile(Tmp, N) -> case file:delete(Tmp) of - {error, eacces} -> % 'dets_process_died' happened anyway... (W-nd-ws) - timer:sleep(1000), - tempfile(Tmp, N-1); - _ -> - Tmp + {error, _Reason} -> % typically enoent + ok; + ok -> + assure_no_file(Tmp) + end, + Tmp. + +assure_no_file(File) -> + case file:read_file_info(File) of + {ok, _FileInfo} -> + %% Wait for some other process to close the file: + timer:sleep(100), + assure_no_file(File); + {error, _} -> + ok end. %% -> {ok, NewHead} | {try_again, integer()} | Error diff --git a/lib/stdlib/src/dets_server.erl b/lib/stdlib/src/dets_server.erl index 268c201047..3164d40f35 100644 --- a/lib/stdlib/src/dets_server.erl +++ b/lib/stdlib/src/dets_server.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2013. All Rights Reserved. +%% Copyright Ericsson AB 2001-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -171,9 +171,15 @@ handle_info({pending_reply, {Ref, Result0}}, State) -> link(Pid), do_link(Store, FromPid), true = ets:insert(Store, {FromPid, Tab}), - true = ets:insert(?REGISTRY, {Tab, 1, Pid}), - true = ets:insert(?OWNERS, {Pid, Tab}), + %% do_internal_open() has already done the following: + %% true = ets:insert(?REGISTRY, {Tab, 1, Pid}), + %% true = ets:insert(?OWNERS, {Pid, Tab}), {ok, Tab}; + {Reply, internal_open} -> + %% Clean up what do_internal_open() did: + true = ets:delete(?REGISTRY, Tab), + true = ets:delete(?OWNERS, Pid), + Reply; {Reply, _} -> % ok or Error Reply end, @@ -309,6 +315,12 @@ do_internal_open(State, From, Args) -> [T, _, _] -> T; [_, _] -> Ref end, + %% Pretend the table is open. If someone else tries to + %% open the file it will always become a pending + %% 'add_user' request. If someone tries to use the table + %% there will be a delay, but that is OK. + true = ets:insert(?REGISTRY, {Tab, 1, Pid}), + true = ets:insert(?OWNERS, {Pid, Tab}), pending_call(Tab, Pid, Ref, From, Args, internal_open, State); Error -> {Error, State} diff --git a/lib/stdlib/test/dets_SUITE.erl b/lib/stdlib/test/dets_SUITE.erl index 119b4dc7cb..3b08ac165e 100644 --- a/lib/stdlib/test/dets_SUITE.erl +++ b/lib/stdlib/test/dets_SUITE.erl @@ -223,8 +223,7 @@ open(Config, Version) -> ?format("Crashing dets server \n", []), process_flag(trap_exit, true), - Procs = [whereis(?DETS_SERVER) | map(fun(Tab) -> dets:info(Tab, pid) end, - Tabs)], + Procs = [whereis(?DETS_SERVER) | [dets:info(Tab, pid) || Tab <- Tabs]], foreach(fun(Pid) -> exit(Pid, kill) end, Procs), timer:sleep(100), c:flush(), %% flush all the EXIT sigs @@ -235,18 +234,32 @@ open(Config, Version) -> open_files(1, All, Version), ?format("Checking contents of repaired files \n", []), check(Tabs, Data), - - close_all(Tabs), + close_all(Tabs), delete_files(All), - P1 = pps(), + {Ports0, Procs0} = P0, - {Ports1, Procs1} = P1, - true = Ports1 =:= Ports0, - %% The dets_server process has been restarted: - [_] = Procs0 -- Procs1, - [_] = Procs1 -- Procs0, - ok. + Test = fun() -> + P1 = pps(), + {Ports1, Procs1} = P1, + show("Old port", Ports0 -- Ports1), + show("New port", Ports1 -- Ports0), + show("Old procs", Procs0 -- Procs1), + show("New procs", Procs1 -- Procs0), + io:format("Remaining Dets-pids (should be nil): ~p~n", + [find_dets_pids()]), + true = Ports1 =:= Ports0, + %% The dets_server process has been restarted: + [_] = Procs0 -- Procs1, + [_] = Procs1 -- Procs0, + ok + end, + case catch Test() of + ok -> ok; + _ -> + timer:sleep(500), + ok = Test() + end. check(Tabs, Data) -> foreach(fun(Tab) -> @@ -3275,12 +3288,22 @@ simultaneous_open(Config) -> File = filename(Tab, Config), ok = monit(Tab, File), - ok = kill_while_repairing(Tab, File), - ok = kill_while_init(Tab, File), - ok = open_ro(Tab, File), - ok = open_w(Tab, File, 0, Config), - ok = open_w(Tab, File, 100, Config), - ok. + case feasible() of + false -> {comment, "OK, but did not run all of the test"}; + true -> + ok = kill_while_repairing(Tab, File), + ok = kill_while_init(Tab, File), + ok = open_ro(Tab, File), + ok = open_w(Tab, File, 0, Config), + ok = open_w(Tab, File, 100, Config) + end. + +feasible() -> + LP = erlang:system_info(logical_processors), + (is_integer(LP) + andalso LP >= erlang:system_info(schedulers_online) + andalso not erlang:system_info(debug_compiled) + andalso not erlang:system_info(lock_checking)). %% One process logs and another process closes the log. Before %% monitors were used, this would make the client never return. @@ -3307,7 +3330,6 @@ kill_while_repairing(Tab, File) -> Delay = 1000, dets:start(), Parent = self(), - Ps = processes(), F = fun() -> R = (catch dets:open_file(Tab, [{file,File}])), timer:sleep(Delay), @@ -3318,7 +3340,7 @@ kill_while_repairing(Tab, File) -> P1 = spawn(F), P2 = spawn(F), P3 = spawn(F), - DetsPid = find_dets_pid([P1, P2, P3 | Ps]), + DetsPid = find_dets_pid(), exit(DetsPid, kill), receive {P1,R1} -> R1 end, @@ -3342,12 +3364,6 @@ kill_while_repairing(Tab, File) -> file:delete(File), ok. -find_dets_pid(P0) -> - case lists:sort(processes() -- P0) of - [P, _] -> P; - _ -> timer:sleep(100), find_dets_pid(P0) - end. - find_dets_pid() -> case find_dets_pids() of [] -> @@ -3421,6 +3437,13 @@ open_ro(Tab, File) -> open_w(Tab, File, Delay, Config) -> create_opened_log(File), + + Tab2 = t2, + File2 = filename(Tab2, Config), + file:delete(File2), + {ok,Tab2} = dets:open_file(Tab2, [{file,File2}]), + ok = dets:close(Tab2), + Parent = self(), F = fun() -> R = dets:open_file(Tab, [{file,File}]), @@ -3430,16 +3453,16 @@ open_w(Tab, File, Delay, Config) -> Pid1 = spawn(F), Pid2 = spawn(F), Pid3 = spawn(F), - undefined = dets:info(Tab), % is repairing now - 0 = qlen(), - Tab2 = t2, - File2 = filename(Tab2, Config), - file:delete(File2), + ok = wait_for_repair_to_start(Tab), + + %% It is assumed that it takes some time to repair the file. {ok,Tab2} = dets:open_file(Tab2, [{file,File2}]), + %% The Dets server managed to handle to open_file request. + 0 = qlen(), % still repairing + ok = dets:close(Tab2), file:delete(File2), - 0 = qlen(), % still repairing receive {Pid1,R1} -> {ok, Tab} = R1 end, receive {Pid2,R2} -> {ok, Tab} = R2 end, @@ -3456,6 +3479,15 @@ open_w(Tab, File, Delay, Config) -> file:delete(File), ok. +wait_for_repair_to_start(Tab) -> + case catch dets_server:get_pid(Tab) of + {'EXIT', _} -> + timer:sleep(1), + wait_for_repair_to_start(Tab); + Pid when is_pid(Pid) -> + ok + end. + qlen() -> {_, {_, N}} = lists:keysearch(message_queue_len, 1, process_info(self())), N. @@ -4350,6 +4382,7 @@ check_badarg({'EXIT', {badarg, [{M,F,A,_} | _]}}, M, F, Args) -> true = test_server:is_native(M) andalso length(Args) =:= A. check_pps({Ports0,Procs0} = P0) -> + ok = check_dets_tables(), case pps() of P0 -> ok; @@ -4375,13 +4408,45 @@ check_pps({Ports0,Procs0} = P0) -> end end. +%% Copied from dets_server.erl: +-define(REGISTRY, dets_registry). +-define(OWNERS, dets_owners). +-define(STORE, dets). + +check_dets_tables() -> + Store = [T || + T <- ets:all(), + ets:info(T, name) =:= ?STORE, + owner(T) =:= dets], + S = case Store of + [Tab] -> ets:tab2list(Tab); + [] -> [] + end, + case {ets:tab2list(?REGISTRY), ets:tab2list(?OWNERS), S} of + {[], [], []} -> ok; + {R, O, _} -> + io:format("Registry: ~p~n", [R]), + io:format("Owners: ~p~n", [O]), + io:format("Store: ~p~n", [S]), + not_ok + end. + +owner(Tab) -> + Owner = ets:info(Tab, owner), + case process_info(Owner, registered_name) of + {registered_name, Name} -> Name; + _ -> Owner + end. + show(_S, []) -> ok; -show(S, [Pid|Pids]) when is_pid(Pid) -> - io:format("~s: ~p~n", [S, erlang:process_info(Pid)]), +show(S, [{Pid, Name, InitCall}|Pids]) when is_pid(Pid) -> + io:format("~s: ~w (~w), ~w: ~p~n", + [S, Pid, proc_reg_name(Name), InitCall, + erlang:process_info(Pid)]), show(S, Pids); -show(S, [Port|Ports]) when is_port(Port)-> - io:format("~s: ~p~n", [S, erlang:port_info(Port)]), +show(S, [{Port, _}|Ports]) when is_port(Port)-> + io:format("~s: ~w: ~p~n", [S, Port, erlang:port_info(Port)]), show(S, Ports). pps() -> @@ -4397,5 +4462,8 @@ process_list() -> safe_second_element(process_info(P, initial_call))} || P <- processes()]. +proc_reg_name({registered_name, Name}) -> Name; +proc_reg_name([]) -> no_reg_name. + safe_second_element({_,Info}) -> Info; safe_second_element(Other) -> Other. diff --git a/lib/syntax_tools/src/erl_syntax.erl b/lib/syntax_tools/src/erl_syntax.erl index 46a5ca48df..40372a2106 100644 --- a/lib/syntax_tools/src/erl_syntax.erl +++ b/lib/syntax_tools/src/erl_syntax.erl @@ -669,6 +669,9 @@ is_leaf(Node) -> operator -> true; % nonstandard type string -> true; text -> true; % nonstandard type + map_expr -> + map_expr_fields(Node) =:= [] andalso + map_expr_argument(Node) =:= none; tuple -> tuple_elements(Node) =:= []; underscore -> true; variable -> true; @@ -6093,6 +6096,9 @@ abstract([]) -> nil(); abstract(T) when is_tuple(T) -> tuple(abstract_list(tuple_to_list(T))); +abstract(T) when is_map(T) -> + map_expr([map_field_assoc(abstract(Key),abstract(Value)) + || {Key,Value} <- maps:to_list(T)]); abstract(T) when is_binary(T) -> binary([binary_field(integer(B)) || B <- binary_to_list(T)]); abstract(T) -> @@ -6154,6 +6160,14 @@ concrete(Node) -> | concrete(list_tail(Node))]; tuple -> list_to_tuple(concrete_list(tuple_elements(Node))); + map_expr -> + As = [tuple([map_field_assoc_name(F), + map_field_assoc_value(F)]) || F <- map_expr_fields(Node)], + M0 = maps:from_list(concrete_list(As)), + case map_expr_argument(Node) of + none -> M0; + Node0 -> maps:merge(concrete(Node0),M0) + end; binary -> Fs = [revert_binary_field( binary_field(binary_field_body(F), @@ -6209,10 +6223,31 @@ is_literal(T) -> is_literal(list_head(T)) andalso is_literal(list_tail(T)); tuple -> lists:all(fun is_literal/1, tuple_elements(T)); + map_expr -> + case map_expr_argument(T) of + none -> true; + Arg -> is_literal(Arg) + end andalso lists:all(fun is_literal_map_field/1, map_expr_fields(T)); + binary -> + lists:all(fun is_literal_binary_field/1, binary_fields(T)); _ -> false end. +is_literal_binary_field(F) -> + case binary_field_types(F) of + [] -> is_literal(binary_field_body(F)); + _ -> false + end. + +is_literal_map_field(F) -> + case type(F) of + map_field_assoc -> + is_literal(map_field_assoc_name(F)) andalso + is_literal(map_field_assoc_value(F)); + map_field_exact -> + false + end. %% ===================================================================== %% @doc Returns an `erl_parse'-compatible representation of a diff --git a/lib/syntax_tools/test/Makefile b/lib/syntax_tools/test/Makefile index d4733b9a42..f67e3f8984 100644 --- a/lib/syntax_tools/test/Makefile +++ b/lib/syntax_tools/test/Makefile @@ -61,5 +61,6 @@ release_tests_spec: make_emakefile $(INSTALL_DATA) $(EMAKEFILE) $(ERL_FILES) "$(RELSYSDIR)" $(INSTALL_DATA) syntax_tools.spec syntax_tools.cover "$(RELSYSDIR)" chmod -R u+w "$(RELSYSDIR)" + @tar cf - *_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -) release_docs_spec: diff --git a/lib/syntax_tools/test/syntax_tools_SUITE.erl b/lib/syntax_tools/test/syntax_tools_SUITE.erl index 6fb3e5ccfb..3c6b33f459 100644 --- a/lib/syntax_tools/test/syntax_tools_SUITE.erl +++ b/lib/syntax_tools/test/syntax_tools_SUITE.erl @@ -24,12 +24,16 @@ init_per_group/2,end_per_group/2]). %% Test cases --export([app_test/1,appup_test/1,smoke_test/1,revert/1,revert_map/1]). +-export([app_test/1,appup_test/1,smoke_test/1,revert/1,revert_map/1, + t_abstract_type/1,t_erl_parse_type/1,t_epp_dodger/1, + t_comment_scan/1,t_igor/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [app_test,appup_test,smoke_test,revert,revert_map]. + [app_test,appup_test,smoke_test,revert,revert_map, + t_abstract_type,t_erl_parse_type,t_epp_dodger, + t_comment_scan,t_igor]. groups() -> []. @@ -54,15 +58,15 @@ appup_test(Config) when is_list(Config) -> %% Read and parse all source in the OTP release. smoke_test(Config) when is_list(Config) -> - ?line Dog = ?t:timetrap(?t:minutes(12)), - ?line Wc = filename:join([code:lib_dir(),"*","src","*.erl"]), - ?line Fs = filelib:wildcard(Wc), - ?line io:format("~p files\n", [length(Fs)]), - ?line case p_run(fun smoke_test_file/1, Fs) of - 0 -> ok; - N -> ?line ?t:fail({N,errors}) - end, - ?line ?t:timetrap_cancel(Dog). + Dog = ?t:timetrap(?t:minutes(12)), + Wc = filename:join([code:lib_dir(),"*","src","*.erl"]), + Fs = filelib:wildcard(Wc), + io:format("~p files\n", [length(Fs)]), + case p_run(fun smoke_test_file/1, Fs) of + 0 -> ok; + N -> ?t:fail({N,errors}) + end, + ?t:timetrap_cancel(Dog). smoke_test_file(File) -> case epp_dodger:parse_file(File) of @@ -94,9 +98,9 @@ revert(Config) when is_list(Config) -> io:format("~p files\n", [length(Fs)]), case p_run(fun (File) -> revert_file(File, Path) end, Fs) of 0 -> ok; - N -> ?line ?t:fail({N,errors}) + N -> ?t:fail({N,errors}) end, - ?line ?t:timetrap_cancel(Dog). + ?t:timetrap_cancel(Dog). revert_file(File, Path) -> case epp:parse_file(File, Path, []) of @@ -110,14 +114,298 @@ revert_file(File, Path) -> end. %% Testing bug fix for reverting map_field_assoc -revert_map(Config) -> +revert_map(Config) when is_list(Config) -> Dog = ?t:timetrap(?t:minutes(1)), - ?line [{map_field_assoc,16,{atom,17,name},{var,18,'Value'}}] = - erl_syntax:revert_forms([{tree,map_field_assoc, - {attr,16,[],none}, - {map_field_assoc, - {atom,17,name},{var,18,'Value'}}}]), - ?line ?t:timetrap_cancel(Dog). + [{map_field_assoc,16,{atom,17,name},{var,18,'Value'}}] = + erl_syntax:revert_forms([{tree,map_field_assoc, + {attr,16,[],none}, + {map_field_assoc,{atom,17,name},{var,18,'Value'}}}]), + ?t:timetrap_cancel(Dog). + + + +%% api tests + +t_abstract_type(Config) when is_list(Config) -> + F = fun validate_abstract_type/1, + ok = validate(F,[{hi,atom}, + {1,integer}, + {1.0,float}, + {$a,integer}, + {[],nil}, + {[<<1,2>>,a,b],list}, + {[2,3,<<1,2>>,a,b],list}, + {[$a,$b,$c],string}, + {"hello world",string}, + {<<1,2,3>>,binary}, + {#{a=>1,"b"=>2},map_expr}, + {#{#{i=>1}=>1,"b"=>#{v=>2}},map_expr}, + {{a,b,c},tuple}]), + ok. + +t_erl_parse_type(Config) when is_list(Config) -> + F = fun validate_erl_parse_type/1, + %% leaf types + ok = validate(F,[{"1",integer,true}, + {"123456789",integer,true}, + {"$h", char,true}, + {"3.1415", float,true}, + {"1.33e36", float,true}, + {"\"1.33e36: hello\"", string,true}, + {"Var1", variable,true}, + {"_", underscore,true}, + {"[]", nil,true}, + {"{}", tuple,true}, + {"#{}",map_expr,true}, + {"'some atom'", atom, true}]), + %% composite types + ok = validate(F,[{"case X of t -> t; f -> f end", case_expr,false}, + {"try X of t -> t catch C:R -> error end", try_expr,false}, + {"receive X -> X end", receive_expr,false}, + {"receive M -> X1 after T -> X2 end", receive_expr,false}, + {"catch (X)", catch_expr,false}, + {"fun(X) -> X end", fun_expr,false}, + {"fun Foo(X) -> X end", named_fun_expr,false}, + {"fun foo/2", implicit_fun,false}, + {"fun bar:foo/2", implicit_fun,false}, + {"if X -> t; true -> f end", if_expr,false}, + {"<<1,2,3,4>>", binary,false}, + {"<<1,2,3,4:5>>", binary,false}, + {"<<V1:63,V2:22/binary, V3/bits>>", binary,false}, + {"begin X end", block_expr,false}, + {"foo(X1,X2)", application,false}, + {"bar:foo(X1,X2)", application,false}, + {"[1,2,3,4]", list,false}, + {"[1|4]", list, false}, + {"[<<1>>,<<2>>,-2,<<>>,[more,list]]", list,false}, + {"[1|[2|[3|[4|[]]]]]", list,false}, + {"#{ a=>1, b=>2 }", map_expr,false}, + {"#{3=>3}#{ a=>1, b=>2 }", map_expr,false}, + {"#{ a:=1, b:=2 }", map_expr,false}, + {"M#{ a=>1, b=>2 }", map_expr,false}, + {"[V||V <- Vs]", list_comp,false}, + {"<< <<B>> || <<B>> <= Bs>>", binary_comp,false}, + {"#state{ a = A, b = B}", record_expr,false}, + {"#state{}", record_expr,false}, + {"#s{ a = #def{ a=A }, b = B}", record_expr,false}, + {"State#state{ a = A, b = B}", record_expr,false}, + {"State#state.a", record_access,false}, + {"#state.a", record_index_expr,false}, + {"-X", prefix_expr,false}, + {"X1 + X2", infix_expr,false}, + {"(X1 + X2) * X3", infix_expr,false}, + {"X1 = X2", match_expr,false}, + {"{a,b,c}", tuple,false}]), + ok. + +%% the macro ?MODULE seems faulty +t_epp_dodger(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + PrivDir = ?config(priv_dir, Config), + Filenames = ["syntax_tools_SUITE_test_module.erl", + "syntax_tools_test.erl"], + ok = test_epp_dodger(Filenames,DataDir,PrivDir), + ok. + +t_comment_scan(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + Filenames = ["syntax_tools_SUITE_test_module.erl", + "syntax_tools_test.erl"], + ok = test_comment_scan(Filenames,DataDir), + ok. + +t_igor(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + PrivDir = ?config(priv_dir, Config), + FileM1 = filename:join(DataDir,"m1.erl"), + FileM2 = filename:join(DataDir,"m2.erl"), + ["m.erl",_]=R = igor:merge(m,[FileM1,FileM2],[{outdir,PrivDir}]), + io:format("igor:merge/3 = ~p~n", [R]), + ok. + +test_comment_scan([],_) -> ok; +test_comment_scan([File|Files],DataDir) -> + Filename = filename:join(DataDir,File), + {ok, Fs0} = epp:parse_file(Filename, [], []), + Comments = erl_comment_scan:file(Filename), + Fun = fun(Node) -> + case erl_syntax:is_form(Node) of + true -> + C1 = erl_syntax:comment(2,[" This is a form."]), + Node1 = erl_syntax:add_precomments([C1],Node), + Node1; + false -> + Node + end + end, + Fs1 = erl_recomment:recomment_forms(Fs0, Comments), + Fs2 = erl_syntax_lib:map(Fun, Fs1), + io:format("File: ~s~n", [Filename]), + io:put_chars(erl_prettypr:format(Fs2, [{paper, 120}, + {ribbon, 110}])), + test_comment_scan(Files,DataDir). + + +test_epp_dodger([], _, _) -> ok; +test_epp_dodger([Filename|Files],DataDir,PrivDir) -> + io:format("Parsing ~p~n", [Filename]), + InFile = filename:join(DataDir, Filename), + Parsers = [{fun epp_dodger:parse_file/1,parse_file}, + {fun epp_dodger:quick_parse_file/1,quick_parse_file}, + {fun (File) -> + {ok,Dev} = file:open(File,[read]), + Res = epp_dodger:parse(Dev), + file:close(File), + Res + end, parse}, + {fun (File) -> + {ok,Dev} = file:open(File,[read]), + Res = epp_dodger:quick_parse(Dev), + file:close(File), + Res + end, quick_parse}], + FsForms = parse_with(Parsers, InFile), + ok = pretty_print_parse_forms(FsForms,PrivDir,Filename), + test_epp_dodger(Files,DataDir,PrivDir). + +parse_with([],_) -> []; +parse_with([{Fun,ParserType}|Funs],File) -> + {ok, Fs} = Fun(File), + [{Fs,ParserType}|parse_with(Funs,File)]. + +pretty_print_parse_forms([],_,_) -> ok; +pretty_print_parse_forms([{Fs0,Type}|FsForms],PrivDir,Filename) -> + Parser = atom_to_list(Type), + OutFile = filename:join(PrivDir, Parser ++"_" ++ Filename), + io:format("Pretty print ~p (~w) to ~p~n", [Filename,Type,OutFile]), + Comment = fun (Node,{CntCase,CntTry}=Cnt) -> + case erl_syntax:type(Node) of + case_expr -> + C1 = erl_syntax:comment(2,["Before a case expression"]), + Node1 = erl_syntax:add_precomments([C1],Node), + C2 = erl_syntax:comment(2,["After a case expression"]), + Node2 = erl_syntax:add_postcomments([C2],Node1), + {Node2,{CntCase+1,CntTry}}; + try_expr -> + C1 = erl_syntax:comment(2,["Before a try expression"]), + Node1 = erl_syntax:set_precomments(Node, + erl_syntax:get_precomments(Node) ++ [C1]), + C2 = erl_syntax:comment(2,["After a try expression"]), + Node2 = erl_syntax:set_postcomments(Node1, + erl_syntax:get_postcomments(Node1) ++ [C2]), + {Node2,{CntCase,CntTry+1}}; + _ -> + {Node,Cnt} + end + end, + Fs1 = erl_syntax:form_list(Fs0), + {Fs2,{CC,CT}} = erl_syntax_lib:mapfold(Comment,{0,0}, Fs1), + io:format("Commented on ~w cases and ~w tries~n", [CC,CT]), + PP = erl_prettypr:format(Fs2), + ok = file:write_file(OutFile,iolist_to_binary(PP)), + pretty_print_parse_forms(FsForms,PrivDir,Filename). + + +validate(_,[]) -> ok; +validate(F,[V|Vs]) -> + ok = F(V), + validate(F,Vs). + + +validate_abstract_type({Lit,Type}) -> + Tree = erl_syntax:abstract(Lit), + ok = validate_special_type(Type,Tree), + Type = erl_syntax:type(Tree), + true = erl_syntax:is_literal(Tree), + ErlT = erl_syntax:revert(Tree), + Type = erl_syntax:type(ErlT), + ok = validate_special_type(Type,ErlT), + Conc = erl_syntax:concrete(Tree), + Lit = Conc, + ok. + +validate_erl_parse_type({String,Type,Leaf}) -> + ErlT = string_to_expr(String), + ok = validate_special_type(Type,ErlT), + Type = erl_syntax:type(ErlT), + Leaf = erl_syntax:is_leaf(ErlT), + Tree = erl_syntax_lib:map(fun(Node) -> Node end, ErlT), + Type = erl_syntax:type(Tree), + _ = erl_syntax:meta(Tree), + ok = validate_special_type(Type,Tree), + RevT = erl_syntax:revert(Tree), + ok = validate_special_type(Type,RevT), + Type = erl_syntax:type(RevT), + ok. + +validate_special_type(string,Node) -> + Val = erl_syntax:string_value(Node), + true = erl_syntax:is_string(Node,Val), + _ = erl_syntax:string_literal(Node), + ok; +validate_special_type(variable,Node) -> + _ = erl_syntax:variable_literal(Node), + ok; +validate_special_type(fun_expr,Node) -> + A = erl_syntax:fun_expr_arity(Node), + true = is_integer(A), + ok; +validate_special_type(named_fun_expr,Node) -> + A = erl_syntax:named_fun_expr_arity(Node), + true = is_integer(A), + ok; +validate_special_type(tuple,Node) -> + Size = erl_syntax:tuple_size(Node), + true = is_integer(Size), + ok; +validate_special_type(float,Node) -> + Str = erl_syntax:float_literal(Node), + Val = list_to_float(Str), + Val = erl_syntax:float_value(Node), + false = erl_syntax:is_proper_list(Node), + false = erl_syntax:is_list_skeleton(Node), + ok; +validate_special_type(integer,Node) -> + Str = erl_syntax:integer_literal(Node), + Val = list_to_integer(Str), + true = erl_syntax:is_integer(Node,Val), + Val = erl_syntax:integer_value(Node), + false = erl_syntax:is_proper_list(Node), + ok; +validate_special_type(nil,Node) -> + true = erl_syntax:is_proper_list(Node), + ok; +validate_special_type(list,Node) -> + true = erl_syntax:is_list_skeleton(Node), + _ = erl_syntax:list_tail(Node), + ErrV = erl_syntax:list_head(Node), + false = erl_syntax:is_string(Node,ErrV), + Norm = erl_syntax:normalize_list(Node), + list = erl_syntax:type(Norm), + case erl_syntax:is_proper_list(Node) of + true -> + true = erl_syntax:is_list_skeleton(Node), + Compact = erl_syntax:compact_list(Node), + list = erl_syntax:type(Compact), + [_|_] = erl_syntax:list_elements(Node), + _ = erl_syntax:list_elements(Node), + N = erl_syntax:list_length(Node), + true = N > 0, + ok; + false -> + ok + end; +validate_special_type(_,_) -> + ok. + +%%% scan_and_parse + +string_to_expr(String) -> + io:format("Str: ~p~n", [String]), + {ok, Ts, _} = erl_scan:string(String++"."), + {ok,[Expr]} = erl_parse:parse_exprs(Ts), + Expr. + p_run(Test, List) -> N = erlang:system_info(schedulers), @@ -138,4 +426,3 @@ p_run_loop(Test, List, N, Refs0, Errors0) -> Refs = Refs0 -- [Ref], p_run_loop(Test, List, N, Refs, Errors) end. - diff --git a/lib/syntax_tools/test/syntax_tools_SUITE_data/m1.erl b/lib/syntax_tools/test/syntax_tools_SUITE_data/m1.erl new file mode 100644 index 0000000000..d0d1911199 --- /dev/null +++ b/lib/syntax_tools/test/syntax_tools_SUITE_data/m1.erl @@ -0,0 +1,22 @@ +%% +%% File: m1.erl +%% Author: Björn-Egil Dahlberg +%% Created: 2014-10-24 +%% + +-module(m1). + +-export([foo/0,bar/1,baz/2]). + +foo() -> + [m2:foo(), + m2:bar()]. + +bar(A) -> + [m2:foo(A), + m2:bar(A), + m2:record_update(3,m2:record())]. + +baz(A,B) -> + [m2:foo(A,B), + m2:bar(A,B)]. diff --git a/lib/syntax_tools/test/syntax_tools_SUITE_data/m2.erl b/lib/syntax_tools/test/syntax_tools_SUITE_data/m2.erl new file mode 100644 index 0000000000..781139317d --- /dev/null +++ b/lib/syntax_tools/test/syntax_tools_SUITE_data/m2.erl @@ -0,0 +1,26 @@ +%% +%% File: m2.erl +%% Author: Björn-Egil Dahlberg +%% Created: 2014-10-24 +%% + +-module(m2). + + +-export([foo/0,foo/1,foo/2, + bar/0,bar/1,bar/2, + record_update/2, record/0]). + +foo() -> ok. +foo(A) -> [item,A]. +foo(A,B) -> A + B. + +bar() -> true. +bar(A) -> {element,A}. +bar(A,B) -> A*B. + +-record(rec, {a,b}). + +record() -> #rec{a=3,b=0}. +record_update(V,#rec{a=V0}=R) -> + R#rec{a=V0+V,b=V0}. diff --git a/lib/syntax_tools/test/syntax_tools_SUITE_data/syntax_tools_SUITE_test_module.erl b/lib/syntax_tools/test/syntax_tools_SUITE_data/syntax_tools_SUITE_test_module.erl new file mode 100644 index 0000000000..07c419b4b7 --- /dev/null +++ b/lib/syntax_tools/test/syntax_tools_SUITE_data/syntax_tools_SUITE_test_module.erl @@ -0,0 +1,540 @@ +-module(syntax_tools_SUITE_test_module). + +-export([foo1/1,foo2/3,start_child/2]). + +-export([len/1,equal/2,concat/2,chr/2,rchr/2,str/2,rstr/2, + span/2,cspan/2,substr/2,substr/3,tokens/2,chars/2,chars/3]). +-export([copies/2,words/1,words/2,strip/1,strip/2,strip/3, + sub_word/2,sub_word/3,left/2,left/3,right/2,right/3, + sub_string/2,sub_string/3,centre/2,centre/3, join/2]). +-export([to_upper/1, to_lower/1]). + +-import(lists,[reverse/1,member/2]). + + +%% @type some_type() = map() +%% @type some_other_type() = {a, #{ list() => term()}} + +-type some_type() :: map(). +-type some_other_type() :: {'a', #{ list() => term()} }. + +-spec foo1(Map :: #{ 'a' => integer(), 'b' => term()}) -> term(). + +%% @doc Gets value from map. + +foo1(#{ a:= 1, b := V}) -> V. + +%% @spec foo2(some_type(), Type2 :: some_other_type(), map()) -> Value +%% @doc Gets value from map. + +-spec foo2( + Type1 :: some_type(), + Type2 :: some_other_type(), + Map :: #{ get => 'value', 'value' => binary()}) -> binary(). + +foo2(Type1, {a,#{ "a" := _}}, #{get := value, value := B}) when is_map(Type1) -> B. + +%% from supervisor 18.0 + +-type child() :: 'undefined' | pid(). +-type child_id() :: term(). +-type mfargs() :: {M :: module(), F :: atom(), A :: [term()] | undefined}. +-type modules() :: [module()] | 'dynamic'. +-type restart() :: 'permanent' | 'transient' | 'temporary'. +-type shutdown() :: 'brutal_kill' | timeout(). +-type worker() :: 'worker' | 'supervisor'. +-type sup_ref() :: (Name :: atom()) + | {Name :: atom(), Node :: node()} + | {'global', Name :: atom()} + | {'via', Module :: module(), Name :: any()} + | pid(). +-type child_spec() :: #{name => child_id(), % mandatory + start => mfargs(), % mandatory + restart => restart(), % optional + shutdown => shutdown(), % optional + type => worker(), % optional + modules => modules()} % optional + | {Id :: child_id(), + StartFunc :: mfargs(), + Restart :: restart(), + Shutdown :: shutdown(), + Type :: worker(), + Modules :: modules()}. + +-type startchild_err() :: 'already_present' + | {'already_started', Child :: child()} | term(). +-type startchild_ret() :: {'ok', Child :: child()} + | {'ok', Child :: child(), Info :: term()} + | {'error', startchild_err()}. + + +-spec start_child(SupRef, ChildSpec) -> startchild_ret() when + SupRef :: sup_ref(), + ChildSpec :: child_spec() | (List :: [term()]). +start_child(Supervisor, ChildSpec) -> + {Supervisor,ChildSpec}. + + +%% From string.erl +%% Robert's bit + +%% len(String) +%% Return the length of a string. + +-spec len(String) -> Length when + String :: string(), + Length :: non_neg_integer(). + +len(S) -> length(S). + +%% equal(String1, String2) +%% Test if 2 strings are equal. + +-spec equal(String1, String2) -> boolean() when + String1 :: string(), + String2 :: string(). + +equal(S, S) -> true; +equal(_, _) -> false. + +%% concat(String1, String2) +%% Concatenate 2 strings. + +-spec concat(String1, String2) -> String3 when + String1 :: string(), + String2 :: string(), + String3 :: string(). + +concat(S1, S2) -> S1 ++ S2. + +%% chr(String, Char) +%% rchr(String, Char) +%% Return the first/last index of the character in a string. + +-spec chr(String, Character) -> Index when + String :: string(), + Character :: char(), + Index :: non_neg_integer(). + +chr(S, C) when is_integer(C) -> chr(S, C, 1). + +chr([C|_Cs], C, I) -> I; +chr([_|Cs], C, I) -> chr(Cs, C, I+1); +chr([], _C, _I) -> 0. + +-spec rchr(String, Character) -> Index when + String :: string(), + Character :: char(), + Index :: non_neg_integer(). + +rchr(S, C) when is_integer(C) -> rchr(S, C, 1, 0). + +rchr([C|Cs], C, I, _L) -> %Found one, now find next! + rchr(Cs, C, I+1, I); +rchr([_|Cs], C, I, L) -> + rchr(Cs, C, I+1, L); +rchr([], _C, _I, L) -> L. + +%% str(String, SubString) +%% rstr(String, SubString) +%% index(String, SubString) +%% Return the first/last index of the sub-string in a string. +%% index/2 is kept for backwards compatibility. + +-spec str(String, SubString) -> Index when + String :: string(), + SubString :: string(), + Index :: non_neg_integer(). + +str(S, Sub) when is_list(Sub) -> str(S, Sub, 1). + +str([C|S], [C|Sub], I) -> + case prefix(Sub, S) of + true -> I; + false -> str(S, [C|Sub], I+1) + end; +str([_|S], Sub, I) -> str(S, Sub, I+1); +str([], _Sub, _I) -> 0. + +-spec rstr(String, SubString) -> Index when + String :: string(), + SubString :: string(), + Index :: non_neg_integer(). + +rstr(S, Sub) when is_list(Sub) -> rstr(S, Sub, 1, 0). + +rstr([C|S], [C|Sub], I, L) -> + case prefix(Sub, S) of + true -> rstr(S, [C|Sub], I+1, I); + false -> rstr(S, [C|Sub], I+1, L) + end; +rstr([_|S], Sub, I, L) -> rstr(S, Sub, I+1, L); +rstr([], _Sub, _I, L) -> L. + +prefix([C|Pre], [C|String]) -> prefix(Pre, String); +prefix([], String) when is_list(String) -> true; +prefix(Pre, String) when is_list(Pre), is_list(String) -> false. + +%% span(String, Chars) -> Length. +%% cspan(String, Chars) -> Length. + +-spec span(String, Chars) -> Length when + String :: string(), + Chars :: string(), + Length :: non_neg_integer(). + +span(S, Cs) when is_list(Cs) -> span(S, Cs, 0). + +span([C|S], Cs, I) -> + case member(C, Cs) of + true -> span(S, Cs, I+1); + false -> I + end; +span([], _Cs, I) -> I. + +-spec cspan(String, Chars) -> Length when + String :: string(), + Chars :: string(), + Length :: non_neg_integer(). + +cspan(S, Cs) when is_list(Cs) -> cspan(S, Cs, 0). + +cspan([C|S], Cs, I) -> + case member(C, Cs) of + true -> I; + false -> cspan(S, Cs, I+1) + end; +cspan([], _Cs, I) -> I. + +%% substr(String, Start) +%% substr(String, Start, Length) +%% Extract a sub-string from String. + +-spec substr(String, Start) -> SubString when + String :: string(), + SubString :: string(), + Start :: pos_integer(). + +substr(String, 1) when is_list(String) -> + String; +substr(String, S) when is_integer(S), S > 1 -> + substr2(String, S). + +-spec substr(String, Start, Length) -> SubString when + String :: string(), + SubString :: string(), + Start :: pos_integer(), + Length :: non_neg_integer(). + +substr(String, S, L) when is_integer(S), S >= 1, is_integer(L), L >= 0 -> + substr1(substr2(String, S), L). + +substr1([C|String], L) when L > 0 -> [C|substr1(String, L-1)]; +substr1(String, _L) when is_list(String) -> []. %Be nice! + +substr2(String, 1) when is_list(String) -> String; +substr2([_|String], S) -> substr2(String, S-1). + +%% tokens(String, Seperators). +%% Return a list of tokens seperated by characters in Seperators. + +-spec tokens(String, SeparatorList) -> Tokens when + String :: string(), + SeparatorList :: string(), + Tokens :: [Token :: nonempty_string()]. + +tokens(S, Seps) -> + tokens1(S, Seps, []). + +tokens1([C|S], Seps, Toks) -> + case member(C, Seps) of + true -> tokens1(S, Seps, Toks); + false -> tokens2(S, Seps, Toks, [C]) + end; +tokens1([], _Seps, Toks) -> + reverse(Toks). + +tokens2([C|S], Seps, Toks, Cs) -> + case member(C, Seps) of + true -> tokens1(S, Seps, [reverse(Cs)|Toks]); + false -> tokens2(S, Seps, Toks, [C|Cs]) + end; +tokens2([], _Seps, Toks, Cs) -> + reverse([reverse(Cs)|Toks]). + +-spec chars(Character, Number) -> String when + Character :: char(), + Number :: non_neg_integer(), + String :: string(). + +chars(C, N) -> chars(C, N, []). + +-spec chars(Character, Number, Tail) -> String when + Character :: char(), + Number :: non_neg_integer(), + Tail :: string(), + String :: string(). + +chars(C, N, Tail) when N > 0 -> + chars(C, N-1, [C|Tail]); +chars(C, 0, Tail) when is_integer(C) -> + Tail. + +%% Torbjörn's bit. + +%%% COPIES %%% + +-spec copies(String, Number) -> Copies when + String :: string(), + Copies :: string(), + Number :: non_neg_integer(). + +copies(CharList, Num) when is_list(CharList), is_integer(Num), Num >= 0 -> + copies(CharList, Num, []). + +copies(_CharList, 0, R) -> + R; +copies(CharList, Num, R) -> + copies(CharList, Num-1, CharList++R). + +%%% WORDS %%% + +-spec words(String) -> Count when + String :: string(), + Count :: pos_integer(). + +words(String) -> words(String, $\s). + +-spec words(String, Character) -> Count when + String :: string(), + Character :: char(), + Count :: pos_integer(). + +words(String, Char) when is_integer(Char) -> + w_count(strip(String, both, Char), Char, 0). + +w_count([], _, Num) -> Num+1; +w_count([H|T], H, Num) -> w_count(strip(T, left, H), H, Num+1); +w_count([_H|T], Char, Num) -> w_count(T, Char, Num). + +%%% SUB_WORDS %%% + +-spec sub_word(String, Number) -> Word when + String :: string(), + Word :: string(), + Number :: integer(). + +sub_word(String, Index) -> sub_word(String, Index, $\s). + +-spec sub_word(String, Number, Character) -> Word when + String :: string(), + Word :: string(), + Number :: integer(), + Character :: char(). + +sub_word(String, Index, Char) when is_integer(Index), is_integer(Char) -> + case words(String, Char) of + Num when Num < Index -> + []; + _Num -> + s_word(strip(String, left, Char), Index, Char, 1, []) + end. + +s_word([], _, _, _,Res) -> reverse(Res); +s_word([Char|_],Index,Char,Index,Res) -> reverse(Res); +s_word([H|T],Index,Char,Index,Res) -> s_word(T,Index,Char,Index,[H|Res]); +s_word([Char|T],Stop,Char,Index,Res) when Index < Stop -> + s_word(strip(T,left,Char),Stop,Char,Index+1,Res); +s_word([_|T],Stop,Char,Index,Res) when Index < Stop -> + s_word(T,Stop,Char,Index,Res). + +%%% STRIP %%% + +-spec strip(string()) -> string(). + +strip(String) -> strip(String, both). + +-spec strip(String, Direction) -> Stripped when + String :: string(), + Stripped :: string(), + Direction :: left | right | both. + +strip(String, left) -> strip_left(String, $\s); +strip(String, right) -> strip_right(String, $\s); +strip(String, both) -> + strip_right(strip_left(String, $\s), $\s). + +-spec strip(String, Direction, Character) -> Stripped when + String :: string(), + Stripped :: string(), + Direction :: left | right | both, + Character :: char(). + +strip(String, right, Char) -> strip_right(String, Char); +strip(String, left, Char) -> strip_left(String, Char); +strip(String, both, Char) -> + strip_right(strip_left(String, Char), Char). + +strip_left([Sc|S], Sc) -> + strip_left(S, Sc); +strip_left([_|_]=S, Sc) when is_integer(Sc) -> S; +strip_left([], Sc) when is_integer(Sc) -> []. + +strip_right([Sc|S], Sc) -> + case strip_right(S, Sc) of + [] -> []; + T -> [Sc|T] + end; +strip_right([C|S], Sc) -> + [C|strip_right(S, Sc)]; +strip_right([], Sc) when is_integer(Sc) -> + []. + +%%% LEFT %%% + +-spec left(String, Number) -> Left when + String :: string(), + Left :: string(), + Number :: non_neg_integer(). + +left(String, Len) when is_integer(Len) -> left(String, Len, $\s). + +-spec left(String, Number, Character) -> Left when + String :: string(), + Left :: string(), + Number :: non_neg_integer(), + Character :: char(). + +left(String, Len, Char) when is_integer(Char) -> + Slen = length(String), + if + Slen > Len -> substr(String, 1, Len); + Slen < Len -> l_pad(String, Len-Slen, Char); + Slen =:= Len -> String + end. + +l_pad(String, Num, Char) -> String ++ chars(Char, Num). + +%%% RIGHT %%% + +-spec right(String, Number) -> Right when + String :: string(), + Right :: string(), + Number :: non_neg_integer(). + +right(String, Len) when is_integer(Len) -> right(String, Len, $\s). + +-spec right(String, Number, Character) -> Right when + String :: string(), + Right :: string(), + Number :: non_neg_integer(), + Character :: char(). + +right(String, Len, Char) when is_integer(Char) -> + Slen = length(String), + if + Slen > Len -> substr(String, Slen-Len+1); + Slen < Len -> r_pad(String, Len-Slen, Char); + Slen =:= Len -> String + end. + +r_pad(String, Num, Char) -> chars(Char, Num, String). + +%%% CENTRE %%% + +-spec centre(String, Number) -> Centered when + String :: string(), + Centered :: string(), + Number :: non_neg_integer(). + +centre(String, Len) when is_integer(Len) -> centre(String, Len, $\s). + +-spec centre(String, Number, Character) -> Centered when + String :: string(), + Centered :: string(), + Number :: non_neg_integer(), + Character :: char(). + +centre(String, 0, Char) when is_list(String), is_integer(Char) -> + []; % Strange cases to centre string +centre(String, Len, Char) when is_integer(Char) -> + Slen = length(String), + if + Slen > Len -> substr(String, (Slen-Len) div 2 + 1, Len); + Slen < Len -> + N = (Len-Slen) div 2, + r_pad(l_pad(String, Len-(Slen+N), Char), N, Char); + Slen =:= Len -> String + end. + +%%% SUB_STRING %%% + +-spec sub_string(String, Start) -> SubString when + String :: string(), + SubString :: string(), + Start :: pos_integer(). + +sub_string(String, Start) -> substr(String, Start). + +-spec sub_string(String, Start, Stop) -> SubString when + String :: string(), + SubString :: string(), + Start :: pos_integer(), + Stop :: pos_integer(). + +sub_string(String, Start, Stop) -> substr(String, Start, Stop - Start + 1). + +%% ISO/IEC 8859-1 (latin1) letters are converted, others are ignored +%% + +to_lower_char(C) when is_integer(C), $A =< C, C =< $Z -> + C + 32; +to_lower_char(C) when is_integer(C), 16#C0 =< C, C =< 16#D6 -> + C + 32; +to_lower_char(C) when is_integer(C), 16#D8 =< C, C =< 16#DE -> + C + 32; +to_lower_char(C) -> + C. + +to_upper_char(C) when is_integer(C), $a =< C, C =< $z -> + C - 32; +to_upper_char(C) when is_integer(C), 16#E0 =< C, C =< 16#F6 -> + C - 32; +to_upper_char(C) when is_integer(C), 16#F8 =< C, C =< 16#FE -> + C - 32; +to_upper_char(C) -> + C. + +-spec to_lower(String) -> Result when + String :: io_lib:latin1_string(), + Result :: io_lib:latin1_string() + ; (Char) -> CharResult when + Char :: char(), + CharResult :: char(). + +to_lower(S) when is_list(S) -> + [to_lower_char(C) || C <- S]; +to_lower(C) when is_integer(C) -> + to_lower_char(C). + +-spec to_upper(String) -> Result when + String :: io_lib:latin1_string(), + Result :: io_lib:latin1_string() + ; (Char) -> CharResult when + Char :: char(), + CharResult :: char(). + +to_upper(S) when is_list(S) -> + [to_upper_char(C) || C <- S]; +to_upper(C) when is_integer(C) -> + to_upper_char(C). + +-spec join(StringList, Separator) -> String when + StringList :: [string()], + Separator :: string(), + String :: string(). + +join([], Sep) when is_list(Sep) -> + []; +join([H|T], Sep) -> + H ++ lists:append([Sep ++ X || X <- T]). diff --git a/lib/syntax_tools/test/syntax_tools_SUITE_data/syntax_tools_test.erl b/lib/syntax_tools/test/syntax_tools_SUITE_data/syntax_tools_test.erl new file mode 100644 index 0000000000..dd3f88d7a8 --- /dev/null +++ b/lib/syntax_tools/test/syntax_tools_SUITE_data/syntax_tools_test.erl @@ -0,0 +1,115 @@ +%% +%% File: syntax_tools_test.erl +%% Author: Björn-Egil Dahlberg +%% Created: 2014-10-23 +%% + +-module(syntax_tools_test). + +-export([foo1/0,foo2/2,foo3/0,foo4/3,foo5/1]). + +-include_lib("kernel/include/file.hrl"). +-record(state, { a, b, c, d}). +-attribute([foo/0]). + +-define(attrib, some_attrib). + +-?attrib([foo2/2]). + +-define(macro_simple1, ok). +-define(MACRO_SIMPLE2, (other)). +-define(macro_simple3, ?MODULE). +-define(macro_simple4, [?macro_simple3,?MODULE,?MACRO_SIMPLE2]). +-define(macro_simple5, (process_info)). +-define(macro_string, "hello world"). +-define(macro_argument1(X), (X + 3)). +-define(macro_argument2(X,Y), (X + 3 * Y)). +-define(macro_block(X), begin X end). +-define(macro_if(X1,X2), if X1 -> X2; true -> none end). + + +-ifdef(macro_def1). +-define(macro_cond1, yep). +-else. +-define(macro_cond1, nope). +-endif. +-ifndef(macro_def2). +-define(macro_cond2, nope). +-else. +-define(macro_cond2, yep). +-endif. +-undef(macro_def1). +-undef(macro_def2). + +%% basic test +foo1() -> + ok. + +%% macro test +foo2(A,B) -> + % string combining ? + [?macro_string, ?macro_string + ?macro_string, + "hello world " + "more hello", + [?macro_simple1, + ?MACRO_SIMPLE2, + ?macro_simple3, + ?macro_simple4, + ?macro_simple5, + ?macro_string, + ?macro_cond1, + ?macro_cond2, + ?macro_block(A), + ?macro_if(A,B), + ?macro_argument1(A), + ?macro_argument1(begin A end), + ?macro_block(<<"hello">>), + ?macro_block("hello"), + ?macro_block([$h,$e,$l,$l,$0]), + ?macro_argument1(id(<<"hello">>)), + ?macro_argument1(if A -> B; true -> 3.14 end), + ?macro_argument1(case A of ok -> B; C -> C end), + ?macro_argument1(receive M -> M after 100 -> 3 end), + ?macro_argument1(try foo5(A) catch C:?macro_simple5 -> {C,B} end), + ?macro_argument2(A,B)], + A,B,ok]. + +id(I) -> I. +%% basic terms + +foo3() -> + [atom, + 'some other atom', + {tuple,1,2,3}, + 1,2,3,3333, + 3,3333,2,1, + [$a,$b,$c], + "hello world", + <<"hello world">>, + <<1,2,3,4,5:6>>, + 3.1415, + 1.03e33]. + +%% application and records + +foo4(A,B,#state{c = C}=S) -> + Ls = foo3(), + S1 = #state{ a = 1, b = 2 }, + [foo2(A,Ls),B,C, + B(3,C), + erlang:process_info(self()), + erlang:?macro_simple5(self()), + A:?MACRO_SIMPLE2(), + A:?macro_simple1(), + A:process_info(self()), + A:B(3), + S#state{ a = 2, b = B, d = S1 }]. + +foo5(A) -> + try foo2(A,A) of + R -> R + catch + error:?macro_simple5 -> + nope + end. diff --git a/lib/wx/examples/demo/ex_graphicsContext.erl b/lib/wx/examples/demo/ex_graphicsContext.erl index 59bfe7ff64..9047f1d135 100644 --- a/lib/wx/examples/demo/ex_graphicsContext.erl +++ b/lib/wx/examples/demo/ex_graphicsContext.erl @@ -54,7 +54,7 @@ do_init(Config) -> %% Setup sizers MainSizer = wxBoxSizer:new(?wxVERTICAL), Sizer = wxStaticBoxSizer:new(?wxVERTICAL, Panel, - [{label, "wxGrapicsContext"}]), + [{label, "wxGraphicsContext"}]), Win = wxPanel:new(Panel, []), Pen = ?wxBLACK_PEN, |