diff options
Diffstat (limited to 'lib')
34 files changed, 469 insertions, 172 deletions
diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index b3c8c42af7..c6e61d543e 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -467,8 +467,10 @@ mpf(Ms) -> passes(Type, Opts) -> {Ext,Passes0} = passes_1(Opts), Passes1 = case Type of - file -> Passes0; - forms -> tl(Passes0) + file -> + Passes0; + forms -> + fix_first_pass(Passes0) end, Passes = select_passes(Passes1, Opts), @@ -505,6 +507,22 @@ pass(from_beam) -> {".beam",[?pass(read_beam_file)|binary_passes()]}; pass(_) -> none. +%% For compilation from forms, replace the first pass with a pass +%% that retrieves the module name. The module name is needed for +%% proper diagnostics and for compilation to native code. + +fix_first_pass([{parse_core,_}|Passes]) -> + [?pass(get_module_name_from_core)|Passes]; +fix_first_pass([{beam_consult_asm,_}|Passes]) -> + [?pass(get_module_name_from_asm)|Passes]; +fix_first_pass([{read_beam_file,_}|Passes]) -> + [?pass(get_module_name_from_beam)|Passes]; +fix_first_pass([_|Passes]) -> + %% When compiling from abstract code, the module name + %% will be set after running the v3_core pass. + Passes. + + %% select_passes([Command], Opts) -> [{Name,Function}] %% Interpret the lists of commands to return a pure list of passes. %% @@ -836,6 +854,12 @@ beam_consult_asm(_Code, St) -> {error,St#compile{errors=St#compile.errors ++ Es}} end. +get_module_name_from_asm({Mod,_,_,_,_}=Asm, St) -> + {ok,Asm,St#compile{module=Mod}}; +get_module_name_from_asm(Asm, St) -> + %% Invalid Beam assembly code. Let it crash in a later pass. + {ok,Asm,St}. + read_beam_file(_Code, St) -> case file:read_file(St#compile.ifile) of {ok,Beam} -> @@ -853,6 +877,16 @@ read_beam_file(_Code, St) -> {error,St#compile{errors=St#compile.errors ++ Es}} end. +get_module_name_from_beam(Beam, St) -> + case beam_lib:info(Beam) of + {error,beam_lib,Error} -> + Es = [{"((forms))",[{none,beam_lib,Error}]}], + {error,St#compile{errors=St#compile.errors ++ Es}}; + Info -> + {module,Mod} = keyfind(module, 1, Info), + {ok,Beam,St#compile{module=Mod}} + end. + no_native_compilation(BeamFile, #compile{options=Opts0}) -> case beam_lib:chunks(BeamFile, ["CInf"]) of {ok,{_,[{"CInf",Term0}]}} -> @@ -940,6 +974,16 @@ parse_core(_Code, St) -> {error,St#compile{errors=St#compile.errors ++ Es}} end. +get_module_name_from_core(Core, St) -> + try + Mod = cerl:concrete(cerl:module_name(Core)), + {ok,Core,St#compile{module=Mod}} + catch + _:_ -> + %% Invalid Core Erlang code. Let it crash in a later pass. + {ok,Core,St} + end. + compile_options([{attribute,_L,compile,C}|Fs]) when is_list(C) -> C ++ compile_options(Fs); compile_options([{attribute,_L,compile,C}|Fs]) -> diff --git a/lib/compiler/test/beam_validator_SUITE.erl b/lib/compiler/test/beam_validator_SUITE.erl index ca85eef688..75bfbf68cc 100644 --- a/lib/compiler/test/beam_validator_SUITE.erl +++ b/lib/compiler/test/beam_validator_SUITE.erl @@ -446,7 +446,7 @@ do_bin_opt(Mod, Asm) -> do_bin_opt(Transform, Mod, Asm0) -> Asm = Transform(Asm0), case compile:forms(Asm, [from_asm,no_postopt,return]) of - {ok,[],Code,_Warnings} when is_binary(Code) -> + {ok,Mod,Code,_Warnings} when is_binary(Code) -> ok; {error,Errors0,_} -> %% beam_validator must return errors, not simply crash, diff --git a/lib/compiler/test/compile_SUITE.erl b/lib/compiler/test/compile_SUITE.erl index f5e904a50a..4e2753ba5f 100644 --- a/lib/compiler/test/compile_SUITE.erl +++ b/lib/compiler/test/compile_SUITE.erl @@ -163,6 +163,24 @@ forms_2(Config) when is_list(Config) -> ok end, + {ok,simple,Core} = compile:forms(SimpleCode, [to_core0,binary]), + forms_compile_and_load(Core, [from_core]), + forms_compile_and_load(Core, [from_core,native]), + + {ok,simple,Asm} = compile:forms(SimpleCode, [to_asm,binary]), + forms_compile_and_load(Asm, [from_asm]), + forms_compile_and_load(Asm, [from_asm,native]), + + {ok,simple,Beam} = compile:forms(SimpleCode, []), + forms_compile_and_load(Beam, [from_beam]), + forms_compile_and_load(Beam, [from_beam,native]), + + %% Cover the error handling code. + error = compile:forms(bad_core, [from_core,report]), + error = compile:forms(bad_asm, [from_asm,report]), + error = compile:forms(<<"bad_beam">>, [from_beam,report]), + error = compile:forms(<<"bad_beam">>, [from_beam,native,report]), + ok. @@ -180,6 +198,14 @@ forms_load_code(Mod, Src, Bin) -> SourceOption. +forms_compile_and_load(Code, Opts) -> + Mod = simple, + {ok,Mod,Bin} = compile:forms(Code, Opts), + {module,Mod} = code:load_binary(Mod, "ignore", Bin), + _ = Mod:module_info(), + true = code:delete(simple), + false = code:purge(simple), + ok. module_mismatch(Config) when is_list(Config) -> DataDir = proplists:get_value(data_dir, Config), @@ -835,7 +861,7 @@ do_core_pp_1(M, A, Outdir) -> ok = file:delete(CoreFile), %% Compile as usual (including optimizations). - compile_forms(Core, [clint,from_core,binary]), + compile_forms(M, Core, [clint,from_core,binary]), %% Don't optimize to test that we are not dependent %% on the Core Erlang optmimization passes. @@ -844,13 +870,13 @@ do_core_pp_1(M, A, Outdir) -> %% records; if sys_core_fold was run it would fix %% that; if sys_core_fold was not run v3_kernel would %% crash.) - compile_forms(Core, [clint,from_core,no_copt,binary]), + compile_forms(M, Core, [clint,from_core,no_copt,binary]), ok. -compile_forms(Forms, Opts) -> +compile_forms(Mod, Forms, Opts) -> case compile:forms(Forms, [report_errors|Opts]) of - {ok,[],_} -> ok; + {ok,Mod,_} -> ok; Other -> throw({error,Other}) end. diff --git a/lib/erl_docgen/priv/xsl/db_html.xsl b/lib/erl_docgen/priv/xsl/db_html.xsl index 8a1a70448e..d863c056e9 100644 --- a/lib/erl_docgen/priv/xsl/db_html.xsl +++ b/lib/erl_docgen/priv/xsl/db_html.xsl @@ -95,7 +95,7 @@ <xsl:value-of select="$elem/@name"/> </xsl:when> <xsl:otherwise> - <xsl:value-of select="$elem"/> + <xsl:value-of select="substring-before($elem, '(')"/> </xsl:otherwise> </xsl:choose> </xsl:otherwise> diff --git a/lib/hipe/cerl/erl_types.erl b/lib/hipe/cerl/erl_types.erl index ea8cc1677d..4cfa80f153 100644 --- a/lib/hipe/cerl/erl_types.erl +++ b/lib/hipe/cerl/erl_types.erl @@ -5171,9 +5171,9 @@ cache_put(Key, Type, DeltaL, #cache{types = Types} = Cache) -> NewTypes = maps:put(Key, {Type, DeltaL}, Types), Cache#cache{types = NewTypes}. --spec t_var_names([erl_type()]) -> [atom()]. +-spec t_var_names([parse_form()]) -> [atom()]. -t_var_names([{var, _, Name}|L]) when L =/= '_' -> +t_var_names([{var, _, Name}|L]) when Name =/= '_' -> [Name|t_var_names(L)]; t_var_names([]) -> []. diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl index c99200777b..89c17a8679 100644 --- a/lib/inets/src/http_client/httpc_handler.erl +++ b/lib/inets/src/http_client/httpc_handler.erl @@ -1224,7 +1224,7 @@ close_socket(#session{socket = Socket, socket_type = SocketType}) -> http_transport:close(SocketType, Socket). activate_request_timeout( - #state{request = #request{timer = undefined} = Request} = State) -> + #state{request = #request{timer = OldRef} = Request} = State) -> Timeout = (Request#request.settings)#http_options.timeout, case Timeout of infinity -> @@ -1232,17 +1232,21 @@ activate_request_timeout( _ -> ReqId = Request#request.id, Msg = {timeout, ReqId}, + case OldRef of + undefined -> + ok; + _ -> + %% Timer is already running! This is the case for a redirect or retry + %% We need to restart the timer because the handler pid has changed + cancel_timer(OldRef, Msg) + end, Ref = erlang:send_after(Timeout, self(), Msg), Request2 = Request#request{timer = Ref}, ReqTimers = [{Request#request.id, Ref} | (State#state.timers)#timers.request_timers], Timers = #timers{request_timers = ReqTimers}, State#state{request = Request2, timers = Timers} - end; - -%% Timer is already running! This is the case for a redirect or retry -activate_request_timeout(State) -> - State. + end. activate_queue_timeout(infinity, State) -> State; diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index fc7f01245b..e6dcd2285f 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -108,6 +108,7 @@ only_simulated() -> tolerate_missing_CR, userinfo, bad_response, + timeout_redirect, internal_server_error, invalid_http, invalid_chunk_size, @@ -785,6 +786,14 @@ bad_response(Config) when is_list(Config) -> ct:print("Wrong Statusline: ~p~n", [Reason]). %%------------------------------------------------------------------------- +timeout_redirect() -> + [{doc, "Test that timeout works for redirects, check ERL-420."}]. +timeout_redirect(Config) when is_list(Config) -> + URL = url(group_name(Config), "/redirect_to_missing_crlf.html", Config), + {error, timeout} = httpc:request(get, {URL, []}, [{timeout, 400}], []). + +%%------------------------------------------------------------------------- + internal_server_error(doc) -> ["Test 50X codes"]; internal_server_error(Config) when is_list(Config) -> @@ -1915,6 +1924,16 @@ handle_uri(_,"/missing_crlf.html",_,_,_,_) -> "Content-Length:32\r\n" ++ "<HTML><BODY>foobar</BODY></HTML>"; +handle_uri(_,"/redirect_to_missing_crlf.html",Port,_,Socket,_) -> + NewUri = url_start(Socket) ++ + integer_to_list(Port) ++ "/missing_crlf.html", + Body = "<HTML><BODY><a href=" ++ NewUri ++ + ">New place</a></BODY></HTML>", + "HTTP/1.1 303 See Other \r\n" ++ + "Location:" ++ NewUri ++ "\r\n" ++ + "Content-Length:" ++ integer_to_list(length(Body)) + ++ "\r\n\r\n" ++ Body; + handle_uri(_,"/wrong_statusline.html",_,_,_,_) -> "ok 200 HTTP/1.1\r\n\r\n" ++ "Content-Length:32\r\n\r\n" ++ diff --git a/lib/kernel/doc/src/inet.xml b/lib/kernel/doc/src/inet.xml index b7c904ff45..b71e8a1e5d 100644 --- a/lib/kernel/doc/src/inet.xml +++ b/lib/kernel/doc/src/inet.xml @@ -75,8 +75,8 @@ Address ip_address() ------- ------------ ::1 {0,0,0,0,0,0,0,1} ::192.168.42.2 {0,0,0,0,0,0,(192 bsl 8) bor 168,(42 bsl 8) bor 2} -FFFF::192.168.42.2 - {16#FFFF,0,0,0,0,0,(192 bsl 8) bor 168,(42 bsl 8) bor 2} +::FFFF:192.168.42.2 + {0,0,0,0,0,16#FFFF,(192 bsl 8) bor 168,(42 bsl 8) bor 2} 3ffe:b80:1f8d:2:204:acff:fe17:bf38 {16#3ffe,16#b80,16#1f8d,16#2,16#204,16#acff,16#fe17,16#bf38} fe80::204:acff:fe17:bf38 @@ -87,8 +87,8 @@ fe80::204:acff:fe17:bf38 <pre> 1> <input>inet:parse_address("192.168.42.2").</input> {ok,{192,168,42,2}} -2> <input>inet:parse_address("FFFF::192.168.42.2").</input> -{ok,{65535,0,0,0,0,0,49320,10754}}</pre> +2> <input>inet:parse_address("::FFFF:192.168.42.2").</input> +{ok,{0,0,0,0,0,65535,49320,10754}}</pre> </description> <datatypes> diff --git a/lib/kernel/doc/src/kernel_app.xml b/lib/kernel/doc/src/kernel_app.xml index 9fccb4c7ac..75e1e18d86 100644 --- a/lib/kernel/doc/src/kernel_app.xml +++ b/lib/kernel/doc/src/kernel_app.xml @@ -58,6 +58,7 @@ </section> <section> + <marker id="erl_signal_server"/> <title>OS Signal Event Handler</title> <p>Asynchronous OS signals may be subscribed to via the Kernel applications event manager (see <seealso marker="doc/design_principles:des_princ">OTP Design Principles</seealso> and diff --git a/lib/kernel/doc/src/net_kernel.xml b/lib/kernel/doc/src/net_kernel.xml index 4e2b0c69db..7ddb849824 100644 --- a/lib/kernel/doc/src/net_kernel.xml +++ b/lib/kernel/doc/src/net_kernel.xml @@ -64,6 +64,19 @@ $ <input>erl -sname foobar</input></pre> by the magic cookie system, see section <seealso marker="doc/reference_manual:distributed">Distributed Erlang</seealso> in the Erlang Reference Manual.</p> + <warning> + <p> + Starting a distributed node without also specifying + <seealso marker="erts:erl#proto_dist"><c>-proto_dist inet_tls</c></seealso> + will expose the node to attacks that may give the attacker + complete access to the node and in extension the cluster. + When using un-secure distributed nodes, make sure that the + network is configured to keep potential attackers out. + See the <seealso marker="ssl:ssl_distribution"> + Using SSL for Erlang Distribution</seealso> User's Guide + for details on how to setup a secure distributed node. + </p> + </warning> </description> <funcs> diff --git a/lib/kernel/doc/src/os.xml b/lib/kernel/doc/src/os.xml index 64c5cbe571..0e9add4161 100644 --- a/lib/kernel/doc/src/os.xml +++ b/lib/kernel/doc/src/os.xml @@ -174,8 +174,9 @@ DirOut = os:cmd("dir"), % on Win32 platform</code> <tag><c>handle</c></tag> <item> - This signal will notify <c>erl_signal_server</c> when it is received by - the Erlang runtime system. + This signal will notify + <seealso marker="kernel_app#erl_signal_server"><c>erl_signal_server</c></seealso> + when it is received by the Erlang runtime system. </item> </taglist> </desc> diff --git a/lib/kernel/src/dist_util.erl b/lib/kernel/src/dist_util.erl index 1c326afca8..b3507e5d13 100644 --- a/lib/kernel/src/dist_util.erl +++ b/lib/kernel/src/dist_util.erl @@ -572,12 +572,25 @@ recv_name(#hs_data{socket = Socket, f_recv = Recv}) -> ?shutdown(no_node) end. -get_name([$n,VersionA, VersionB, Flag1, Flag2, Flag3, Flag4 | OtherNode]) -> - {?u32(Flag1, Flag2, Flag3, Flag4), list_to_atom(OtherNode), - ?u16(VersionA,VersionB)}; +get_name([$n,VersionA, VersionB, Flag1, Flag2, Flag3, Flag4 | OtherNode] = Data) -> + case is_valid_name(OtherNode) of + true -> + {?u32(Flag1, Flag2, Flag3, Flag4), list_to_atom(OtherNode), + ?u16(VersionA,VersionB)}; + false -> + ?shutdown(Data) + end; get_name(Data) -> ?shutdown(Data). +is_valid_name(OtherNodeName) -> + case string:lexemes(OtherNodeName,"@") of + [_OtherNodeName,_OtherNodeHost] -> + true; + _else -> + false + end. + publish_type(Flags) -> case Flags band ?DFLAG_PUBLISHED of 0 -> diff --git a/lib/kernel/test/gen_tcp_api_SUITE.erl b/lib/kernel/test/gen_tcp_api_SUITE.erl index 92a74465b7..3f11e25b93 100644 --- a/lib/kernel/test/gen_tcp_api_SUITE.erl +++ b/lib/kernel/test/gen_tcp_api_SUITE.erl @@ -302,9 +302,9 @@ t_implicit_inet6(Config) when is_list(Config) -> end. t_implicit_inet6(Host, Addr) -> - case gen_tcp:listen(0, [inet6]) of + Loopback = {0,0,0,0,0,0,0,1}, + case gen_tcp:listen(0, [inet6, {ip,Loopback}]) of {ok,S1} -> - Loopback = {0,0,0,0,0,0,0,1}, io:format("~s ~p~n", ["::1",Loopback]), implicit_inet6(S1, Loopback), ok = gen_tcp:close(S1), @@ -524,10 +524,10 @@ local_handshake(S, SAddr, C, CAddr) -> t_accept_inet6_tclass(Config) when is_list(Config) -> TClassOpt = {tclass,8#56 bsl 2}, % Expedited forwarding - case gen_tcp:listen(0, [inet6,TClassOpt]) of + Loopback = {0,0,0,0,0,0,0,1}, + case gen_tcp:listen(0, [inet6, {ip, Loopback}, TClassOpt]) of {ok,L} -> LPort = ok(inet:port(L)), - Loopback = {0,0,0,0,0,0,0,1}, Sa = ok(gen_tcp:connect(Loopback, LPort, [])), Sb = ok(gen_tcp:accept(L)), [TClassOpt] = ok(inet:getopts(Sb, [tclass])), diff --git a/lib/kernel/test/gen_udp_SUITE.erl b/lib/kernel/test/gen_udp_SUITE.erl index 1029d7ef0a..836e0c5a05 100644 --- a/lib/kernel/test/gen_udp_SUITE.erl +++ b/lib/kernel/test/gen_udp_SUITE.erl @@ -717,9 +717,9 @@ implicit_inet6(Config) when is_list(Config) -> implicit_inet6(Host, Addr) -> Active = {active,false}, - case gen_udp:open(0, [inet6,Active]) of + Loopback = {0,0,0,0,0,0,0,1}, + case gen_udp:open(0, [inet6,Active,{ip, Loopback}]) of {ok,S1} -> - Loopback = {0,0,0,0,0,0,0,1}, io:format("~s ~p~n", ["::1",Loopback]), implicit_inet6(S1, Active, Loopback), ok = gen_udp:close(S1), diff --git a/lib/kernel/test/inet_sockopt_SUITE.erl b/lib/kernel/test/inet_sockopt_SUITE.erl index 322b9f30fe..9413cbd976 100644 --- a/lib/kernel/test/inet_sockopt_SUITE.erl +++ b/lib/kernel/test/inet_sockopt_SUITE.erl @@ -620,7 +620,7 @@ ipv6_v6only_close(Module, Socket) -> %% Test using socket option ipv6_v6only for UDP. use_ipv6_v6only_udp(Config) when is_list(Config) -> - case gen_udp:open(0, [inet6,{ipv6_v6only,true}]) of + case gen_udp:open(0, [inet6,{ip,{0,0,0,0,0,0,0,1}}, {ipv6_v6only,true}]) of {ok,S6} -> case inet:getopts(S6, [ipv6_v6only]) of {ok,[{ipv6_v6only,true}]} -> diff --git a/lib/mnesia/src/mnesia.erl b/lib/mnesia/src/mnesia.erl index dece995d39..3b771e8c5b 100644 --- a/lib/mnesia/src/mnesia.erl +++ b/lib/mnesia/src/mnesia.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -166,7 +166,7 @@ -type select_continuation() :: term(). -type snmp_struct() :: [{atom(), snmp_type() | tuple_of(snmp_type())}]. -type snmp_type() :: 'fix_string' | 'string' | 'integer'. --type tuple_of(_) :: tuple(). +-type tuple_of(_T) :: tuple(). -define(DEFAULT_ACCESS, ?MODULE). diff --git a/lib/orber/test/multi_ORB_SUITE.erl b/lib/orber/test/multi_ORB_SUITE.erl index d739e47cc1..8becc11d6a 100644 --- a/lib/orber/test/multi_ORB_SUITE.erl +++ b/lib/orber/test/multi_ORB_SUITE.erl @@ -135,13 +135,12 @@ cases() -> setup_multi_connection_timeout_api, setup_multi_connection_timeout_attempts_api, setup_multi_connection_timeout_random_api, - ssl_1_multi_orber_api, ssl_1_multi_orber_generation_3_api, - ssl_2_multi_orber_api, ssl_2_multi_orber_generation_3_api, - ssl_reconfigure_api, ssl_reconfigure_generation_3_api]. +% ssl_1_multi_orber_api,ssl_2_multi_orber_api,ssl_reconfigure_api, + %%----------------------------------------------------------------- %% Init and cleanup functions. %%----------------------------------------------------------------- diff --git a/lib/orber/test/orber_test_lib.erl b/lib/orber/test/orber_test_lib.erl index 95ab26cd30..4a247ce492 100644 --- a/lib/orber/test/orber_test_lib.erl +++ b/lib/orber/test/orber_test_lib.erl @@ -356,11 +356,17 @@ get_options(ssl, Role, Level) -> get_options(ssl, Role, 2, Options) -> Dir = filename:join([code:lib_dir(ssl), "examples", "certs", "etc"]), - [{depth, 2}, - {verify, 2}, - {keyfile, filename:join([Dir, Role, "key.pem"])}, - {cacertfile, filename:join([Dir, Role, "cacerts.pem"])}, - {certfile, filename:join([Dir, Role, "cert.pem"])} |Options]; + Options1 = [{depth, 2}, + {verify, 2}, + {keyfile, filename:join([Dir, Role, "key.pem"])}, + {cacertfile, filename:join([Dir, Role, "cacerts.pem"])}, + {certfile, filename:join([Dir, Role, "cert.pem"])} |Options], + case Role of + client -> + [{server_name_indication, disable} |Options1]; + server -> + Options1 + end; get_options(iiop_ssl, _Role, 2, Options) -> Dir = filename:join([code:lib_dir(ssl), "examples", "certs", "etc"]), [{ssl_server_options, [{depth, 2}, @@ -369,10 +375,11 @@ get_options(iiop_ssl, _Role, 2, Options) -> {cacertfile, filename:join([Dir, "server", "cacerts.pem"])}, {keyfile, filename:join([Dir, "server", "key.pem"])}]}, {ssl_client_options, [{depth, 2}, - {verify, 2}, - {certfile, filename:join([Dir, "client", "cert.pem"])}, - {cacertfile, filename:join([Dir, "client", "cacerts.pem"])}, - {keyfile, filename:join([Dir, "client", "key.pem"])}]}, + {verify, 2}, + {server_name_indication, disable}, + {certfile, filename:join([Dir, "client", "cert.pem"])}, + {cacertfile, filename:join([Dir, "client", "cacerts.pem"])}, + {keyfile, filename:join([Dir, "client", "key.pem"])}]}, {secure, ssl} |Options]; get_options(iiop_ssl, _Role, 1, Options) -> Dir = filename:join([code:lib_dir(ssl), "examples", "certs", "etc"]), @@ -382,10 +389,11 @@ get_options(iiop_ssl, _Role, 1, Options) -> {cacertfile, filename:join([Dir, "server", "cacerts.pem"])}, {keyfile, filename:join([Dir, "server", "key.pem"])}]}, {ssl_client_options, [{depth, 1}, - {verify, 0}, - {certfile, filename:join([Dir, "client", "cert.pem"])}, - {cacertfile, filename:join([Dir, "client", "cacerts.pem"])}, - {keyfile, filename:join([Dir, "client", "key.pem"])}]}, + {verify, 0}, + {server_name_indication, disable}, + {certfile, filename:join([Dir, "client", "cert.pem"])}, + {cacertfile, filename:join([Dir, "client", "cacerts.pem"])}, + {keyfile, filename:join([Dir, "client", "key.pem"])}]}, {secure, ssl} |Options]. create_paths() -> diff --git a/lib/ssh/src/ssh_dbg.erl b/lib/ssh/src/ssh_dbg.erl index 7dfbfc3b4b..003b3856e6 100644 --- a/lib/ssh/src/ssh_dbg.erl +++ b/lib/ssh/src/ssh_dbg.erl @@ -22,9 +22,10 @@ -module(ssh_dbg). --export([messages/0, - messages/1, - messages/2, +-export([messages/0, messages/1, messages/2, + ct_messages/0, + auth/0, auth/1, auth/2, + ct_auth/0, stop/0 ]). @@ -43,75 +44,167 @@ messages() -> messages(fun(String,_D) -> io:format(String) end). +ct_messages() -> + messages(fun(String,_D) -> ct:log(String,[]) end). + messages(Write) when is_function(Write,2) -> messages(Write, fun(X) -> X end). messages(Write, MangleArg) when is_function(Write,2), is_function(MangleArg,1) -> - catch dbg:start(), - setup_tracer(Write, MangleArg), - dbg:p(new,[c,timestamp]), - dbg_ssh_messages(). + cond_start(msg, Write, MangleArg), + dbg_ssh_messages(), + dbg_ssh_auth(). + + +auth() -> + auth(fun(String,_D) -> io:format(String) end). + +ct_auth() -> + auth(fun(String,_D) -> ct:log(String,[]) end). + +auth(Write) when is_function(Write,2) -> + auth(Write, fun(X) -> X end). + +auth(Write, MangleArg) when is_function(Write,2), + is_function(MangleArg,1) -> + cond_start(auth, Write, MangleArg), + dbg_ssh_auth(). + dbg_ssh_messages() -> dbg:tp(ssh_message,encode,1, x), dbg:tp(ssh_message,decode,1, x), dbg:tpl(ssh_transport,select_algorithm,4, x), dbg:tp(ssh_transport,hello_version_msg,1, x), - dbg:tp(ssh_transport,handle_hello_version,1, x). + dbg:tp(ssh_transport,handle_hello_version,1, x), + dbg:tpl(ssh_connection_handler,ext_info,2, x). + +dbg_ssh_auth() -> + dbg:tp(ssh_transport,hello_version_msg,1, x), + dbg:tp(ssh_transport,handle_hello_version,1, x), + dbg:tp(ssh_message,encode,1, x), + dbg:tpl(ssh_transport,select_algorithm,4, x), + dbg:tpl(ssh_connection_handler,ext_info,2, x), + lists:foreach(fun(F) -> dbg:tp(ssh_auth, F, x) end, + [publickey_msg, password_msg, keyboard_interactive_msg]). %%%---------------------------------------------------------------- stop() -> dbg:stop(). %%%================================================================ -msg_formater({trace_ts,Pid,call,{ssh_message,encode,[Msg]},TS}, D) -> +cond_start(Type, Write, MangleArg) -> + try + dbg:start(), + setup_tracer(Type, Write, MangleArg), + dbg:p(new,[c,timestamp]) + catch + _:_ -> ok + end. + + +msg_formater(msg, {trace_ts,Pid,call,{ssh_message,encode,[Msg]},TS}, D) -> fmt("~n~s SEND ~p ~s~n", [ts(TS),Pid,wr_record(shrink_bin(Msg))], D); -msg_formater({trace_ts,_Pid,return_from,{ssh_message,encode,1},_Res,_TS}, D) -> +msg_formater(msg, {trace_ts,_Pid,return_from,{ssh_message,encode,1},_Res,_TS}, D) -> D; -msg_formater({trace_ts,_Pid,call,{ssh_message,decode,_},_TS}, D) -> +msg_formater(msg, {trace_ts,_Pid,call,{ssh_message,decode,_},_TS}, D) -> D; -msg_formater({trace_ts,Pid,return_from,{ssh_message,decode,1},Msg,TS}, D) -> +msg_formater(msg, {trace_ts,Pid,return_from,{ssh_message,decode,1},Msg,TS}, D) -> fmt("~n~s ~p RECV ~s~n", [ts(TS),Pid,wr_record(shrink_bin(Msg))], D); + +msg_formater(auth, {trace_ts,Pid,return_from,{ssh_message,decode,1},#ssh_msg_userauth_failure{authentications=As},TS}, D) -> + fmt("~n~s ~p Client login FAILURE. Try ~s~n", [ts(TS),Pid,As], D); + +msg_formater(auth, {trace_ts,Pid,return_from,{ssh_message,decode,1},#ssh_msg_userauth_success{},TS}, D) -> + fmt("~n~s ~p Client login SUCCESS~n", [ts(TS),Pid], D); + -msg_formater({trace_ts,_Pid,call,{ssh_transport,select_algorithm,_},_TS}, D) -> +msg_formater(_, {trace_ts,_Pid,call,{ssh_transport,select_algorithm,_},_TS}, D) -> D; -msg_formater({trace_ts,Pid,return_from,{ssh_transport,select_algorithm,_},{ok,Alg},TS}, D) -> +msg_formater(_, {trace_ts,Pid,return_from,{ssh_transport,select_algorithm,_},{ok,Alg},TS}, D) -> fmt("~n~s ~p ALGORITHMS~n~s~n", [ts(TS),Pid, wr_record(Alg)], D); -msg_formater({trace_ts,_Pid,call,{ssh_transport,hello_version_msg,_},_TS}, D) -> +msg_formater(_, {trace_ts,_Pid,call,{ssh_transport,hello_version_msg,_},_TS}, D) -> D; -msg_formater({trace_ts,Pid,return_from,{ssh_transport,hello_version_msg,1},Hello,TS}, D) -> +msg_formater(_, {trace_ts,Pid,return_from,{ssh_transport,hello_version_msg,1},Hello,TS}, D) -> fmt("~n~s ~p TCP SEND HELLO~n ~p~n", [ts(TS),Pid,lists:flatten(Hello)], D); -msg_formater({trace_ts,Pid,call,{ssh_transport,handle_hello_version,[Hello]},TS}, D) -> +msg_formater(_, {trace_ts,Pid,call,{ssh_transport,handle_hello_version,[Hello]},TS}, D) -> fmt("~n~s ~p RECV HELLO~n ~p~n", [ts(TS),Pid,lists:flatten(Hello)], D); -msg_formater({trace_ts,_Pid,return_from,{ssh_transport,handle_hello_version,1},_,_TS}, D) -> +msg_formater(_, {trace_ts,_Pid,return_from,{ssh_transport,handle_hello_version,1},_,_TS}, D) -> D; -msg_formater({trace_ts,Pid,send,{tcp,Sock,Bytes},Pid,TS}, D) -> +msg_formater(_, {trace_ts,Pid,call,{ssh_connection_handler,ext_info,[{"server-sig-algs",_SigAlgs},State]},TS}, D) -> + try lists:keyfind(ssh, 1, tuple_to_list(State)) of + false -> + D; + #ssh{userauth_pubkeys = PKs} -> + fmt("~n~s ~p Client got suggestion to use user public key sig-algs~n ~p~n", [ts(TS),Pid,PKs], D) + catch + _:_ -> + D + end; + +msg_formater(_, {trace_ts,Pid,return_from,{ssh_connection_handler,ext_info,2},State,TS}, D) -> + try lists:keyfind(ssh, 1, tuple_to_list(State)) of + false -> + D; + #ssh{userauth_pubkeys = PKs} -> + fmt("~n~s ~p Client will try user public key sig-algs~n ~p~n", [ts(TS),Pid,PKs], D) + catch + _:_ -> + D + end; + +msg_formater(_, {trace_ts,Pid,call,{ssh_auth,publickey_msg,[[SigAlg,#ssh{user=User}]]},TS}, D) -> + fmt("~n~s ~p Client will try to login user ~p with public key algorithm ~p~n", [ts(TS),Pid,User,SigAlg], D); +msg_formater(_, {trace_ts,Pid,return_from,{ssh_auth,publickey_msg,1},{not_ok,#ssh{user=User}},TS}, D) -> + fmt("~s ~p User ~p can't login with that kind of public key~n", [ts(TS),Pid,User], D); + +msg_formater(_, {trace_ts,Pid,call,{ssh_auth,password_msg,[[#ssh{user=User}]]},TS}, D) -> + fmt("~n~s ~p Client will try to login user ~p with password~n", [ts(TS),Pid,User], D); +msg_formater(_, {trace_ts,Pid,return_from,{ssh_auth,password_msg,1},{not_ok,#ssh{user=User}},TS}, D) -> + fmt("~s ~p User ~p can't login with password~n", [ts(TS),Pid,User], D); + +msg_formater(_, {trace_ts,Pid,call,{ssh_auth,keyboard_interactive_msg,[[#ssh{user=User}]]},TS}, D) -> + fmt("~n~s ~p Client will try to login user ~p with password~n", [ts(TS),Pid,User], D); +msg_formater(_, {trace_ts,Pid,return_from,{ssh_auth,keyboard_interactive_msg,1},{not_ok,#ssh{user=User}},TS}, D) -> + fmt("~s ~p User ~p can't login with keyboard_interactive password~n", [ts(TS),Pid,User], D); + +msg_formater(msg, {trace_ts,Pid,send,{tcp,Sock,Bytes},Pid,TS}, D) -> fmt("~n~s ~p TCP SEND on ~p~n ~p~n", [ts(TS),Pid,Sock, shrink_bin(Bytes)], D); -msg_formater({trace_ts,Pid,send,{tcp,Sock,Bytes},Dest,TS}, D) -> +msg_formater(msg, {trace_ts,Pid,send,{tcp,Sock,Bytes},Dest,TS}, D) -> fmt("~n~s ~p TCP SEND from ~p TO ~p~n ~p~n", [ts(TS),Pid,Sock,Dest, shrink_bin(Bytes)], D); -msg_formater({trace_ts,Pid,send,ErlangMsg,Dest,TS}, D) -> +msg_formater(msg, {trace_ts,Pid,send,ErlangMsg,Dest,TS}, D) -> fmt("~n~s ~p ERL MSG SEND TO ~p~n ~p~n", [ts(TS),Pid,Dest, shrink_bin(ErlangMsg)], D); -msg_formater({trace_ts,Pid,'receive',{tcp,Sock,Bytes},TS}, D) -> +msg_formater(msg, {trace_ts,Pid,'receive',{tcp,Sock,Bytes},TS}, D) -> fmt("~n~s ~p TCP RECEIVE on ~p~n ~p~n", [ts(TS),Pid,Sock,shrink_bin(Bytes)], D); -msg_formater({trace_ts,Pid,'receive',ErlangMsg,TS}, D) -> +msg_formater(msg, {trace_ts,Pid,'receive',ErlangMsg,TS}, D) -> fmt("~n~s ~p ERL MSG RECEIVE~n ~p~n", [ts(TS),Pid,shrink_bin(ErlangMsg)], D); -msg_formater(M, D) -> - fmt("~nDBG ~n~p~n", [shrink_bin(M)], D). +%% msg_formater(_, {trace_ts,_Pid,return_from,MFA,_Ret,_TS}=M, D) -> +%% case lists:member(MFA, [{ssh_auth,keyboard_interactive_msg,1}, +%% {ssh_auth,password_msg,1}, +%% {ssh_auth,publickey_msg,1}]) of +%% true -> +%% D; +%% false -> +%% fmt("~nDBG ~n~p~n", [shrink_bin(M)], D) +%% end; + +%% msg_formater(_, M, D) -> +%% fmt("~nDBG ~n~p~n", [shrink_bin(M)], D). -%% msg_formater(_, D) -> -%% D. +msg_formater(_, _, D) -> + D. fmt(Fmt, Args, D=#data{writer=Write,acc=Acc}) -> @@ -123,9 +216,9 @@ ts({_,_,Usec}=Now) -> ts(_) -> "-". %%%---------------------------------------------------------------- -setup_tracer(Write, MangleArg) -> +setup_tracer(Type, Write, MangleArg) -> Handler = fun(Arg, D) -> - msg_formater(MangleArg(Arg), D) + msg_formater(Type, MangleArg(Arg), D) end, InitialData = #data{writer = Write}, {ok,_} = dbg:tracer(process, {Handler, InitialData}), diff --git a/lib/ssh/src/ssh_message.erl b/lib/ssh/src/ssh_message.erl index 4f2eeca026..b1fc05ae33 100644 --- a/lib/ssh/src/ssh_message.erl +++ b/lib/ssh/src/ssh_message.erl @@ -611,13 +611,3 @@ encode_signature({{#'ECPoint'{}, {namedCurve,OID}},_}, Signature) -> CurveName = public_key:oid2ssh_curvename(OID), <<?Ebinary(<<"ecdsa-sha2-",CurveName/binary>>), ?Ebinary(Signature)>>. -%% encode_signature(#'RSAPublicKey'{}, Signature) -> -%% SignName = <<"ssh-rsa">>, -%% <<?Ebinary(SignName), ?Ebinary(Signature)>>; -%% encode_signature({_, #'Dss-Parms'{}}, Signature) -> -%% <<?Ebinary(<<"ssh-dss">>), ?Ebinary(Signature)>>; -%% encode_signature({#'ECPoint'{}, {namedCurve,OID}}, Signature) -> -%% CurveName = public_key:oid2ssh_curvename(OID), -%% <<?Ebinary(<<"ecdsa-sha2-",CurveName/binary>>), ?Ebinary(Signature)>>. - - diff --git a/lib/ssh/test/property_test/ssh_eqc_encode_decode.erl b/lib/ssh/test/property_test/ssh_eqc_encode_decode.erl index 0995182623..165274241c 100644 --- a/lib/ssh/test/property_test/ssh_eqc_encode_decode.erl +++ b/lib/ssh/test/property_test/ssh_eqc_encode_decode.erl @@ -280,21 +280,21 @@ msg_code(Num) -> Name -include_lib("ssh/src/ssh_transport.hrl"). %%% Encoding and decodeing is asymetric so out=binary in=string. Sometimes. :( +-define(fix_asym_Xdh_reply(S), + fix_asym(#S{public_host_key = Key, h_sig = {Alg,Sig}} = M) -> + M#S{public_host_key = {Key, list_to_atom(Alg)}, h_sig = Sig} +). + + fix_asym(#ssh_msg_global_request{name=N} = M) -> M#ssh_msg_global_request{name = binary_to_list(N)}; fix_asym(#ssh_msg_debug{message=D,language=L} = M) -> M#ssh_msg_debug{message = binary_to_list(D), language = binary_to_list(L)}; fix_asym(#ssh_msg_kexinit{cookie=C} = M) -> M#ssh_msg_kexinit{cookie = <<C:128>>}; - -fix_asym(#ssh_msg_kexdh_reply{public_host_key = Key} = M) -> M#ssh_msg_kexdh_reply{public_host_key = key_sigalg(Key)}; -fix_asym(#ssh_msg_kex_dh_gex_reply{public_host_key = Key} = M) -> M#ssh_msg_kex_dh_gex_reply{public_host_key = key_sigalg(Key)}; -fix_asym(#ssh_msg_kex_ecdh_reply{public_host_key = Key} = M) -> M#ssh_msg_kex_ecdh_reply{public_host_key = key_sigalg(Key)}; - +?fix_asym_Xdh_reply(ssh_msg_kexdh_reply); +?fix_asym_Xdh_reply(ssh_msg_kex_dh_gex_reply); +?fix_asym_Xdh_reply(ssh_msg_kex_ecdh_reply); fix_asym(M) -> M. -%%% Keys now contains an sig-algorithm name -key_sigalg(#'RSAPublicKey'{} = Key) -> {Key,'ssh-rsa'}; -key_sigalg({_, #'Dss-Parms'{}} = Key) -> {Key,'ssh-dss'}; -key_sigalg({#'ECPoint'{}, {namedCurve,OID}} = Key) -> {Key,"ecdsa-sha2-256"}. %%% Message codes 30 and 31 are overloaded depending on kex family so arrange the decoder %%% input as the test object does diff --git a/lib/ssh/test/ssh.spec b/lib/ssh/test/ssh.spec index 68268cb20d..b4e3d36072 100644 --- a/lib/ssh/test/ssh.spec +++ b/lib/ssh/test/ssh.spec @@ -1,6 +1,7 @@ {suites,"../ssh_test",all}. -{skip_suites, "../ssh_test", [ssh_bench_SUITE +{skip_suites, "../ssh_test", [ssh_bench_SUITE, + ssh_upgrade_SUITE ], "Benchmarks run separately"}. diff --git a/lib/ssh/test/ssh_algorithms_SUITE.erl b/lib/ssh/test/ssh_algorithms_SUITE.erl index 736461624d..0f69910e40 100644 --- a/lib/ssh/test/ssh_algorithms_SUITE.erl +++ b/lib/ssh/test/ssh_algorithms_SUITE.erl @@ -171,6 +171,7 @@ init_per_testcase(_, {public_key,Alg}, Config) -> Opts = pubkey_opts(Config), case {ssh_file:user_key(Alg,Opts), ssh_file:host_key(Alg,Opts)} of {{ok,_}, {ok,_}} -> + ssh_dbg:ct_auth(), start_pubkey_daemon([proplists:get_value(pref_algs,Config)], [{extra_daemon,true}|Config]); {{ok,_}, _} -> @@ -192,6 +193,7 @@ init_per_testcase(_, _, Config) -> end_per_testcase(_TC, Config) -> + catch ssh_dbg:stop(), case proplists:get_value(extra_daemon, Config, false) of true -> case proplists:get_value(srvr_pid,Config) of diff --git a/lib/stdlib/doc/src/gen_statem.xml b/lib/stdlib/doc/src/gen_statem.xml index 1aac88c308..ad7f2f2e95 100644 --- a/lib/stdlib/doc/src/gen_statem.xml +++ b/lib/stdlib/doc/src/gen_statem.xml @@ -346,7 +346,7 @@ ok <p> To compare styles, here follows the same example using <seealso marker="#type-callback_mode"><em>callback mode</em></seealso> - <c>state_functions</c>, or rather the code to replace + <c>handle_event_function</c>, or rather the code to replace after function <c>init/1</c> of the <c>pushbutton.erl</c> example file above: </p> diff --git a/lib/stdlib/doc/src/string.xml b/lib/stdlib/doc/src/string.xml index 343904a49a..9d5edd9ecf 100644 --- a/lib/stdlib/doc/src/string.xml +++ b/lib/stdlib/doc/src/string.xml @@ -311,7 +311,9 @@ true</pre> <desc> <p> Returns the first codepoint in <c><anno>String</anno></c> - and the rest of <c><anno>String</anno></c> in the tail. + and the rest of <c><anno>String</anno></c> in the tail. Returns + an empty list if <c><anno>String</anno></c> is empty or an + <c>{error, String}</c> tuple if the next byte is invalid. </p> <p><em>Example:</em></p> <pre> @@ -326,7 +328,9 @@ true</pre> <desc> <p> Returns the first grapheme cluster in <c><anno>String</anno></c> - and the rest of <c><anno>String</anno></c> in the tail. + and the rest of <c><anno>String</anno></c> in the tail. Returns + an empty list if <c><anno>String</anno></c> is empty or an + <c>{error, String}</c> tuple if the next byte is invalid. </p> <p><em>Example:</em></p> <pre> diff --git a/lib/stdlib/src/erl_parse.yrl b/lib/stdlib/src/erl_parse.yrl index 2dcddeb8c2..733932e711 100644 --- a/lib/stdlib/src/erl_parse.yrl +++ b/lib/stdlib/src/erl_parse.yrl @@ -1052,6 +1052,9 @@ build_typed_attribute({atom,Aa,record}, build_typed_attribute({atom,Aa,Attr}, {type_def, {call,_,{atom,_,TypeName},Args}, Type}) when Attr =:= 'type' ; Attr =:= 'opaque' -> + lists:foreach(fun({var, A, '_'}) -> ret_err(A, "bad type variable"); + (_) -> ok + end, Args), case lists:all(fun({var, _, _}) -> true; (_) -> false end, Args) of diff --git a/lib/stdlib/src/string.erl b/lib/stdlib/src/string.erl index 17135dd64a..6f7009b5d9 100644 --- a/lib/stdlib/src/string.erl +++ b/lib/stdlib/src/string.erl @@ -486,12 +486,14 @@ find(String, SearchPattern, trailing) -> %% Fetch first codepoint and return rest in tail -spec next_grapheme(String::unicode:chardata()) -> - maybe_improper_list(grapheme_cluster(),unicode:chardata()). + maybe_improper_list(grapheme_cluster(),unicode:chardata()) | + {error,unicode:chardata()}. next_grapheme(CD) -> unicode_util:gc(CD). %% Fetch first grapheme cluster and return rest in tail -spec next_codepoint(String::unicode:chardata()) -> - maybe_improper_list(char(),unicode:chardata()). + maybe_improper_list(char(),unicode:chardata()) | + {error,unicode:chardata()}. next_codepoint(CD) -> unicode_util:cp(CD). %% Internals @@ -508,7 +510,7 @@ equal_1(A0,B0) -> case {unicode_util:cp(A0), unicode_util:cp(B0)} of {[CP|A],[CP|B]} -> equal_1(A,B); {[], []} -> true; - _ -> false + {L1,L2} when is_list(L1), is_list(L2) -> false end. equal_nocase(A, A) -> true; @@ -517,7 +519,7 @@ equal_nocase(A0, B0) -> unicode_util:cp(unicode_util:casefold(B0))} of {[CP|A],[CP|B]} -> equal_nocase(A,B); {[], []} -> true; - _ -> false + {L1,L2} when is_list(L1), is_list(L2) -> false end. equal_norm(A, A, _Norm) -> true; @@ -526,7 +528,7 @@ equal_norm(A0, B0, Norm) -> unicode_util:cp(unicode_util:Norm(B0))} of {[CP|A],[CP|B]} -> equal_norm(A,B, Norm); {[], []} -> true; - _ -> false + {L1,L2} when is_list(L1), is_list(L2) -> false end. equal_norm_nocase(A, A, _Norm) -> true; @@ -535,7 +537,7 @@ equal_norm_nocase(A0, B0, Norm) -> unicode_util:cp(unicode_util:casefold(unicode_util:Norm(B0)))} of {[CP|A],[CP|B]} -> equal_norm_nocase(A,B, Norm); {[], []} -> true; - _ -> false + {L1,L2} when is_list(L1), is_list(L2) -> false end. reverse_1(CD, Acc) -> diff --git a/lib/stdlib/src/unicode.erl b/lib/stdlib/src/unicode.erl index aa1da400ce..fbe8a94074 100644 --- a/lib/stdlib/src/unicode.erl +++ b/lib/stdlib/src/unicode.erl @@ -250,89 +250,110 @@ encoding_to_bom(latin1) -> -define(GC_N, 200). %% arbitrary number %% Canonical decompose string to list of chars --spec characters_to_nfd_list(chardata()) -> [char()]. +-spec characters_to_nfd_list(chardata()) -> [char()] | {error, [char()], chardata()}. characters_to_nfd_list(CD) -> + characters_to_nfd_list(CD, []). +characters_to_nfd_list(CD, Acc) -> case unicode_util:nfd(CD) of - [GC|Str] when is_list(GC) -> GC++characters_to_nfd_list(Str); - [CP|Str] -> [CP|characters_to_nfd_list(Str)]; - [] -> [] + [GC|Str] when is_list(GC) -> characters_to_nfd_list(Str, lists:reverse(GC, Acc)); + [CP|Str] -> characters_to_nfd_list(Str, [CP | Acc]); + [] -> lists:reverse(Acc); + {error,Error} -> {error, lists:reverse(Acc), Error} end. --spec characters_to_nfd_binary(chardata()) -> unicode_binary(). +-spec characters_to_nfd_binary(chardata()) -> unicode_binary() | {error, unicode_binary(), chardata()}. characters_to_nfd_binary(CD) -> - list_to_binary(characters_to_nfd_binary(CD, ?GC_N, [])). + characters_to_nfd_binary(CD, ?GC_N, [], []). -characters_to_nfd_binary(CD, N, Row) when N > 0 -> +characters_to_nfd_binary(CD, N, Row, Acc) when N > 0 -> case unicode_util:nfd(CD) of - [GC|Str] -> characters_to_nfd_binary(Str, N-1, [GC|Row]); - [] -> [characters_to_binary(lists:reverse(Row))] + [GC|Str] -> characters_to_nfd_binary(Str, N-1, [GC|Row], Acc); + [] -> acc_to_binary(prepend_row_to_acc(Row, Acc)); + {error, Error} -> {error, acc_to_binary(prepend_row_to_acc(Row, Acc)), Error} end; -characters_to_nfd_binary(CD, _, Row) -> - [characters_to_binary(lists:reverse(Row))|characters_to_nfd_binary(CD,?GC_N,[])]. +characters_to_nfd_binary(CD, _, Row, Acc) -> + characters_to_nfd_binary(CD, ?GC_N, [], prepend_row_to_acc(Row, Acc)). %% Compability Canonical decompose string to list of chars. --spec characters_to_nfkd_list(chardata()) -> [char()]. +-spec characters_to_nfkd_list(chardata()) -> [char()] | {error, [char()], chardata()}. characters_to_nfkd_list(CD) -> + characters_to_nfkd_list(CD, []). +characters_to_nfkd_list(CD, Acc) -> case unicode_util:nfkd(CD) of - [GC|Str] when is_list(GC) -> GC++characters_to_nfkd_list(Str); - [CP|Str] -> [CP|characters_to_nfkd_list(Str)]; - [] -> [] + [GC|Str] when is_list(GC) -> characters_to_nfkd_list(Str, lists:reverse(GC, Acc)); + [CP|Str] -> characters_to_nfkd_list(Str, [CP | Acc]); + [] -> lists:reverse(Acc); + {error,Error} -> {error, lists:reverse(Acc), Error} end. --spec characters_to_nfkd_binary(chardata()) -> unicode_binary(). +-spec characters_to_nfkd_binary(chardata()) -> unicode_binary() | {error, unicode_binary(), chardata()}. characters_to_nfkd_binary(CD) -> - list_to_binary(characters_to_nfkd_binary(CD, ?GC_N, [])). + characters_to_nfkd_binary(CD, ?GC_N, [], []). -characters_to_nfkd_binary(CD, N, Row) when N > 0 -> +characters_to_nfkd_binary(CD, N, Row, Acc) when N > 0 -> case unicode_util:nfkd(CD) of - [GC|Str] -> characters_to_nfkd_binary(Str, N-1, [GC|Row]); - [] -> [characters_to_binary(lists:reverse(Row))] + [GC|Str] -> characters_to_nfkd_binary(Str, N-1, [GC|Row], Acc); + [] -> acc_to_binary(prepend_row_to_acc(Row, Acc)); + {error, Error} -> {error, acc_to_binary(prepend_row_to_acc(Row, Acc)), Error} end; -characters_to_nfkd_binary(CD, _, Row) -> - [characters_to_binary(lists:reverse(Row))|characters_to_nfkd_binary(CD,?GC_N,[])]. +characters_to_nfkd_binary(CD, _, Row, Acc) -> + characters_to_nfkd_binary(CD, ?GC_N, [], prepend_row_to_acc(Row, Acc)). %% Canonical compose string to list of chars --spec characters_to_nfc_list(chardata()) -> [char()]. +-spec characters_to_nfc_list(chardata()) -> [char()] | {error, [char()], chardata()}. characters_to_nfc_list(CD) -> + characters_to_nfc_list(CD, []). +characters_to_nfc_list(CD, Acc) -> case unicode_util:nfc(CD) of - [CPs|Str] when is_list(CPs) -> CPs ++ characters_to_nfc_list(Str); - [CP|Str] -> [CP|characters_to_nfc_list(Str)]; - [] -> [] + [GC|Str] when is_list(GC) -> characters_to_nfc_list(Str, lists:reverse(GC, Acc)); + [CP|Str] -> characters_to_nfc_list(Str, [CP | Acc]); + [] -> lists:reverse(Acc); + {error,Error} -> {error, lists:reverse(Acc), Error} end. --spec characters_to_nfc_binary(chardata()) -> unicode_binary(). +-spec characters_to_nfc_binary(chardata()) -> unicode_binary() | {error, unicode_binary(), chardata()}. characters_to_nfc_binary(CD) -> - list_to_binary(characters_to_nfc_binary(CD, ?GC_N, [])). + characters_to_nfc_binary(CD, ?GC_N, [], []). -characters_to_nfc_binary(CD, N, Row) when N > 0 -> +characters_to_nfc_binary(CD, N, Row, Acc) when N > 0 -> case unicode_util:nfc(CD) of - [GC|Str] -> characters_to_nfc_binary(Str, N-1, [GC|Row]); - [] -> [characters_to_binary(lists:reverse(Row))] + [GC|Str] -> characters_to_nfc_binary(Str, N-1, [GC|Row], Acc); + [] -> acc_to_binary(prepend_row_to_acc(Row, Acc)); + {error, Error} -> {error, acc_to_binary(prepend_row_to_acc(Row, Acc)), Error} end; -characters_to_nfc_binary(CD, _, Row) -> - [characters_to_binary(lists:reverse(Row))|characters_to_nfc_binary(CD,?GC_N,[])]. +characters_to_nfc_binary(CD, _, Row, Acc) -> + characters_to_nfc_binary(CD, ?GC_N, [], prepend_row_to_acc(Row, Acc)). %% Compability Canonical compose string to list of chars --spec characters_to_nfkc_list(chardata()) -> [char()]. +-spec characters_to_nfkc_list(chardata()) -> [char()] | {error, [char()], chardata()}. characters_to_nfkc_list(CD) -> + characters_to_nfkc_list(CD, []). +characters_to_nfkc_list(CD, Acc) -> case unicode_util:nfkc(CD) of - [CPs|Str] when is_list(CPs) -> CPs ++ characters_to_nfkc_list(Str); - [CP|Str] -> [CP|characters_to_nfkc_list(Str)]; - [] -> [] + [GC|Str] when is_list(GC) -> characters_to_nfkc_list(Str, lists:reverse(GC, Acc)); + [CP|Str] -> characters_to_nfkc_list(Str, [CP | Acc]); + [] -> lists:reverse(Acc); + {error,Error} -> {error, lists:reverse(Acc), Error} end. --spec characters_to_nfkc_binary(chardata()) -> unicode_binary(). +-spec characters_to_nfkc_binary(chardata()) -> unicode_binary() | {error, unicode_binary(), chardata()}. characters_to_nfkc_binary(CD) -> - list_to_binary(characters_to_nfkc_binary(CD, ?GC_N, [])). + characters_to_nfkc_binary(CD, ?GC_N, [], []). -characters_to_nfkc_binary(CD, N, Row) when N > 0 -> +characters_to_nfkc_binary(CD, N, Row, Acc) when N > 0 -> case unicode_util:nfkc(CD) of - [GC|Str] -> characters_to_nfkc_binary(Str, N-1, [GC|Row]); - [] -> [characters_to_binary(lists:reverse(Row))] + [GC|Str] -> characters_to_nfkc_binary(Str, N-1, [GC|Row], Acc); + [] -> acc_to_binary(prepend_row_to_acc(Row, Acc)); + {error, Error} -> {error, acc_to_binary(prepend_row_to_acc(Row, Acc)), Error} end; -characters_to_nfkc_binary(CD, _, Row) -> - [characters_to_binary(lists:reverse(Row))|characters_to_nfkc_binary(CD,?GC_N,[])]. +characters_to_nfkc_binary(CD, _, Row, Acc) -> + characters_to_nfkc_binary(CD, ?GC_N, [], prepend_row_to_acc(Row, Acc)). + +acc_to_binary(Acc) -> + list_to_binary(lists:reverse(Acc)). +prepend_row_to_acc(Row, Acc) -> + [characters_to_binary(lists:reverse(Row))|Acc]. %% internals diff --git a/lib/stdlib/test/erl_lint_SUITE.erl b/lib/stdlib/test/erl_lint_SUITE.erl index 02524679fa..cc3d605840 100644 --- a/lib/stdlib/test/erl_lint_SUITE.erl +++ b/lib/stdlib/test/erl_lint_SUITE.erl @@ -3978,7 +3978,11 @@ otp_14323(Config) -> {13,erl_lint,{undefined_function,{a,1}}}, {14,erl_lint,{bad_dialyzer_attribute, {nowarn_function,{a,-1}}}}], - []}}], + []}}, + {otp_14323_2, + <<"-type t(_) :: atom().">>, + [], + {errors,[{1,erl_parse,"bad type variable"}],[]}}], [] = run(Config, Ts), ok. diff --git a/lib/stdlib/test/string_SUITE.erl b/lib/stdlib/test/string_SUITE.erl index 4320b735ac..90f980c0e5 100644 --- a/lib/stdlib/test/string_SUITE.erl +++ b/lib/stdlib/test/string_SUITE.erl @@ -582,6 +582,8 @@ cd_gc(_) -> [$e,778] = string:next_codepoint([$e,778]), [$e|<<204,138>>] = string:next_codepoint(<<$e,778/utf8>>), [778|_] = string:next_codepoint(tl(string:next_codepoint(<<$e,778/utf8>>))), + [0|<<128,1>>] = string:next_codepoint(<<0,128,1>>), + {error,<<128,1>>} = string:next_codepoint(<<128,1>>), [] = string:next_grapheme(""), [] = string:next_grapheme(<<>>), @@ -589,6 +591,8 @@ cd_gc(_) -> "abcd" = string:next_grapheme("abcd"), [[$e,778]] = string:next_grapheme([$e,778]), [[$e,778]] = string:next_grapheme(<<$e,778/utf8>>), + [0|<<128,1>>] = string:next_grapheme(<<0,128,1>>), + {error,<<128,1>>} = string:next_grapheme(<<128,1>>), ok. diff --git a/lib/stdlib/test/unicode_SUITE.erl b/lib/stdlib/test/unicode_SUITE.erl index 3d97ab93f1..e01ba3fbb0 100644 --- a/lib/stdlib/test/unicode_SUITE.erl +++ b/lib/stdlib/test/unicode_SUITE.erl @@ -998,6 +998,30 @@ normalize(_) -> true = unicode:characters_to_nfkc_list("ホンダ") =:= unicode:characters_to_nfkc_list("ﾎﾝﾀﾞ"), true = unicode:characters_to_nfkd_list("32") =:= unicode:characters_to_nfkd_list("３２"), + + {error, [0], <<128>>} = unicode:characters_to_nfc_list(<<0, 128>>), + {error, [0], <<128>>} = unicode:characters_to_nfkc_list(<<0, 128>>), + {error, [0], <<128>>} = unicode:characters_to_nfd_list(<<0, 128>>), + {error, [0], <<128>>} = unicode:characters_to_nfkd_list(<<0, 128>>), + + {error, <<0>>, <<128>>} = unicode:characters_to_nfc_binary(<<0, 128>>), + {error, <<0>>, <<128>>} = unicode:characters_to_nfkc_binary(<<0, 128>>), + {error, <<0>>, <<128>>} = unicode:characters_to_nfd_binary(<<0, 128>>), + {error, <<0>>, <<128>>} = unicode:characters_to_nfkd_binary(<<0, 128>>), + + LargeBin = binary:copy(<<"abcde">>, 50), + LargeList = binary_to_list(LargeBin), + + {error, LargeList, <<128>>} = unicode:characters_to_nfc_list(<<LargeBin/binary, 128>>), + {error, LargeList, <<128>>} = unicode:characters_to_nfkc_list(<<LargeBin/binary, 128>>), + {error, LargeList, <<128>>} = unicode:characters_to_nfd_list(<<LargeBin/binary, 128>>), + {error, LargeList, <<128>>} = unicode:characters_to_nfkd_list(<<LargeBin/binary, 128>>), + + {error, LargeBin, <<128>>} = unicode:characters_to_nfc_binary(<<LargeBin/binary, 128>>), + {error, LargeBin, <<128>>} = unicode:characters_to_nfkc_binary(<<LargeBin/binary, 128>>), + {error, LargeBin, <<128>>} = unicode:characters_to_nfd_binary(<<LargeBin/binary, 128>>), + {error, LargeBin, <<128>>} = unicode:characters_to_nfkd_binary(<<LargeBin/binary, 128>>), + ok. diff --git a/lib/stdlib/test/unicode_util_SUITE.erl b/lib/stdlib/test/unicode_util_SUITE.erl index e9b3d7f98d..03c24c7027 100644 --- a/lib/stdlib/test/unicode_util_SUITE.erl +++ b/lib/stdlib/test/unicode_util_SUITE.erl @@ -97,6 +97,8 @@ cp(_) -> "hejsan" = fetch(<<"hejsan">>, Get), "hejsan" = fetch(["hej",<<"san">>], Get), "hejsan" = fetch(["hej"|<<"san">>], Get), + {error, <<128>>} = Get(<<128>>), + {error, [<<128>>, 0]} = Get([<<128>>, 0]), ok. gc(Config) -> @@ -106,6 +108,8 @@ gc(Config) -> "hejsan" = fetch(<<"hejsan">>, Get), "hejsan" = fetch(["hej",<<"san">>], Get), "hejsan" = fetch(["hej"|<<"san">>], Get), + {error, <<128>>} = Get(<<128>>), + {error, [<<128>>, 0]} = Get([<<128>>, 0]), 0 = fold(fun verify_gc/3, 0, DataDir ++ "/GraphemeBreakTest.txt"), ok. diff --git a/lib/stdlib/uc_spec/gen_unicode_mod.escript b/lib/stdlib/uc_spec/gen_unicode_mod.escript index c8b815e435..fefd7d3b70 100755 --- a/lib/stdlib/uc_spec/gen_unicode_mod.escript +++ b/lib/stdlib/uc_spec/gen_unicode_mod.escript @@ -170,7 +170,7 @@ gen_header(Fd) -> io:put_chars(Fd, "-export([spec_version/0, lookup/1, get_case/1]).\n"), io:put_chars(Fd, "-inline([class/1]).\n"), io:put_chars(Fd, "-compile(nowarn_unused_vars).\n"), - io:put_chars(Fd, "-dialyzer({no_improper_lists, cp/1}).\n"), + io:put_chars(Fd, "-dialyzer({no_improper_lists, [cp/1, gc_prepend/2, gc_e_cont/2]}).\n"), io:put_chars(Fd, "-type gc() :: char()|[char()].\n\n\n"), ok. @@ -237,39 +237,43 @@ gen_static(Fd) -> gen_norm(Fd) -> io:put_chars(Fd, - "-spec nfd(unicode:chardata()) -> maybe_improper_list(gc(),unicode:chardata()).\n" + "-spec nfd(unicode:chardata()) -> maybe_improper_list(gc(),unicode:chardata()) | {error, unicode:chardata()}.\n" "nfd(Str0) ->\n" " case gc(Str0) of\n" " [GC|R] when GC < 127 -> [GC|R];\n" " [GC|Str] -> [decompose(GC)|Str];\n" - " [] -> []\n end.\n\n" + " [] -> [];\n" + " {error,_}=Error -> Error\n end.\n\n" ), io:put_chars(Fd, - "-spec nfkd(unicode:chardata()) -> maybe_improper_list(gc(),unicode:chardata()).\n" + "-spec nfkd(unicode:chardata()) -> maybe_improper_list(gc(),unicode:chardata()) | {error, unicode:chardata()}.\n" "nfkd(Str0) ->\n" " case gc(Str0) of\n" " [GC|R] when GC < 127 -> [GC|R];\n" " [GC|Str] -> [decompose_compat(GC)|Str];\n" - " [] -> []\n end.\n\n" + " [] -> [];\n" + " {error,_}=Error -> Error\n end.\n\n" ), io:put_chars(Fd, - "-spec nfc(unicode:chardata()) -> maybe_improper_list(gc(),unicode:chardata()).\n" + "-spec nfc(unicode:chardata()) -> maybe_improper_list(gc(),unicode:chardata()) | {error, unicode:chardata()}.\n" "nfc(Str0) ->\n" " case gc(Str0) of\n" " [GC|R] when GC < 255 -> [GC|R];\n" " [GC|Str] -> [compose(decompose(GC))|Str];\n" - " [] -> []\n end.\n\n" + " [] -> [];\n" + " {error,_}=Error -> Error\n end.\n\n" ), io:put_chars(Fd, - "-spec nfkc(unicode:chardata()) -> maybe_improper_list(gc(),unicode:chardata()).\n" + "-spec nfkc(unicode:chardata()) -> maybe_improper_list(gc(),unicode:chardata()) | {error, unicode:chardata()}.\n" "nfkc(Str0) ->\n" " case gc(Str0) of\n" " [GC|R] when GC < 127 -> [GC|R];\n" " [GC|Str] -> [compose_compat_0(decompose_compat(GC))|Str];\n" - " [] -> []\n end.\n\n" + " [] -> [];\n" + " {error,_}=Error -> Error\n end.\n\n" ), io:put_chars(Fd, @@ -448,18 +452,20 @@ gen_ws(Fd, Props) -> gen_cp(Fd) -> io:put_chars(Fd, "-spec cp(String::unicode:chardata()) ->" - " maybe_improper_list().\n"), + " maybe_improper_list() | {error, unicode:chardata()}.\n"), io:put_chars(Fd, "cp([C|_]=L) when is_integer(C) -> L;\n"), io:put_chars(Fd, "cp([List]) -> cp(List);\n"), io:put_chars(Fd, "cp([List|R]) ->\n"), io:put_chars(Fd, " case cp(List) of\n"), io:put_chars(Fd, " [] -> cp(R);\n"), io:put_chars(Fd, " [CP] -> [CP|R];\n"), - io:put_chars(Fd, " [C|R0] -> [C|[R0|R]]\n"), + io:put_chars(Fd, " [C|R0] -> [C|[R0|R]];\n"), + io:put_chars(Fd, " {error,Error} -> {error,[Error|R]}\n"), io:put_chars(Fd, " end;\n"), io:put_chars(Fd, "cp([]) -> [];\n"), io:put_chars(Fd, "cp(<<C/utf8, R/binary>>) -> [C|R];\n"), - io:put_chars(Fd, "cp(<<>>) -> [].\n\n"), + io:put_chars(Fd, "cp(<<>>) -> [];\n"), + io:put_chars(Fd, "cp(<<R/binary>>) -> {error,R}.\n\n"), ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -468,7 +474,7 @@ gen_gc(Fd, GBP) -> %% see http://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundary_Rules io:put_chars(Fd, "-spec gc(String::unicode:chardata()) ->" - " maybe_improper_list().\n"), + " maybe_improper_list() | {error, unicode:chardata()}.\n"), io:put_chars(Fd, "gc(Str) ->\n" " gc_1(cp(Str)).\n\n" @@ -521,7 +527,8 @@ gen_gc(Fd, GBP) -> [GenEBG(CP) || CP <- merge_ranges(maps:get(e_base_gaz,GBP))], io:put_chars(Fd, "gc_1([CP|R]) -> gc_extend(R, CP);\n"), - io:put_chars(Fd, "gc_1([]) -> [].\n\n"), + io:put_chars(Fd, "gc_1([]) -> [];\n"), + io:put_chars(Fd, "gc_1({error,_}=Error) -> Error.\n\n"), io:put_chars(Fd, "%% Handle Prepend\n"), io:put_chars(Fd, @@ -536,7 +543,8 @@ gen_gc(Fd, GBP) -> " [GC|R1] -> [[CP0|GC]|R1]\n" " end\n" " end;\n" - " [] -> [CP0]\n" + " [] -> [CP0];\n" + " {error,R} -> [CP0|R]\n" " end.\n\n"), IsCtrl = fun(Range) -> io:format(Fd, "is_control~s true;\n", [gen_single_clause(Range)]) end, @@ -574,7 +582,10 @@ gen_gc(Fd, GBP) -> " [_]=Acc -> Acc;\n" " [_|_]=Acc -> [lists:reverse(Acc)];\n" " Acc -> [Acc]\n" - " end.\n\n"), + " end;\n" + "gc_extend({error,R}, T, Acc0) ->\n" + " gc_extend([], T, Acc0) ++ [R].\n\n" + ), [ZWJ] = maps:get(zwj, GBP), GenExtend = fun(R) when R =:= ZWJ -> io:format(Fd, "is_extend~s zwj;\n", [gen_single_clause(ZWJ)]); (Range) -> io:format(Fd, "is_extend~s true;\n", [gen_single_clause(Range)]) @@ -604,6 +615,11 @@ gen_gc(Fd, GBP) -> " case Acc of\n" " [A] -> [A];\n" " _ -> [lists:reverse(Acc)]\n" + " end;\n" + " {error,R} ->\n" + " case Acc of\n" + " [A] -> [A|R];\n" + " _ -> [lists:reverse(Acc)|R]\n" " end\n" " end.\n\n"), @@ -660,6 +676,7 @@ gen_gc(Fd, GBP) -> [GenHangulT_1(CP) || CP <- merge_ranges(maps:get(t,GBP))], io:put_chars(Fd, " R1 -> gc_extend(R1, R0, Acc)\n end.\n\n"), + io:put_chars(Fd, "gc_h_lv_lvt({error,_}=Error, Acc) -> gc_extend(Error, [], Acc);\n"), io:put_chars(Fd, "%% Handle Hangul LV\n"), GenHangulLV = fun(Range) -> io:format(Fd, "gc_h_lv_lvt~s gc_h_V(R1,[CP|Acc]);\n", [gen_clause2(Range)]) end, diff --git a/lib/wx/c_src/wxe_impl.cpp b/lib/wx/c_src/wxe_impl.cpp index 7817e7fa8a..1510866f09 100644 --- a/lib/wx/c_src/wxe_impl.cpp +++ b/lib/wx/c_src/wxe_impl.cpp @@ -267,7 +267,7 @@ int WxeApp::dispatch_cmds() return more; } -#define BREAK_BATCH 200 +#define BREAK_BATCH 10000 int WxeApp::dispatch(wxeFifo * batch) { @@ -284,7 +284,7 @@ int WxeApp::dispatch(wxeFifo * batch) if(blevel>0) { blevel--; if(blevel==0) - wait += BREAK_BATCH*100; + wait += BREAK_BATCH/4; } break; case WXE_BATCH_BEGIN: @@ -317,7 +317,7 @@ int WxeApp::dispatch(wxeFifo * batch) erl_drv_mutex_lock(wxe_batch_locker_m); batch->Cleanup(); } - if(blevel <= 0 || wait > BREAK_BATCH) { + if(blevel <= 0 || wait >= BREAK_BATCH) { erl_drv_mutex_unlock(wxe_batch_locker_m); if(blevel > 0) { return 1; // We are still in a batch but we can let wx check for events |