diff options
Diffstat (limited to 'lib')
35 files changed, 945 insertions, 280 deletions
diff --git a/lib/asn1/src/asn1ct.erl b/lib/asn1/src/asn1ct.erl index f36d71a601..81a2735a0d 100644 --- a/lib/asn1/src/asn1ct.erl +++ b/lib/asn1/src/asn1ct.erl @@ -1335,25 +1335,39 @@ test_value(Module, Type, Value) -> in_process(fun() -> case catch Module:encode(Type, Value) of {ok, Bytes} -> - NewBytes = prepare_bytes(Bytes), - case Module:decode(Type, NewBytes) of - {ok, Value} -> - {ok, {Module, Type, Value}}; - {ok, Res} -> - {error, {asn1, - {encode_decode_mismatch, - {{Module, Type, Value}, Res}}}}; - Error -> - {error, {asn1, - {{decode, - {Module, Type, Value}, Error}}}} - end; + test_value_decode(Module, Type, Value, Bytes); + Bytes when is_binary(Bytes) -> + test_value_decode(Module, Type, Value, Bytes); Error -> {error, {asn1, {encode, {{Module, Type, Value}, Error}}}} end end). + +test_value_decode(Module, Type, Value, Bytes) -> + NewBytes = prepare_bytes(Bytes), + case Module:decode(Type, NewBytes) of + {ok,Value} -> {ok, {Module,Type,Value}}; + {ok,Value,<<>>} -> {ok, {Module,Type,Value}}; + Value -> {ok, {Module,Type,Value}}; + {Value,<<>>} -> {ok, {Module,Type,Value}}; + + %% Errors: + {ok, Res} -> + {error, {asn1, + {encode_decode_mismatch, + {{Module, Type, Value}, Res}}}}; + {ok, Res, Rest} -> + {error, {asn1, + {encode_decode_mismatch, + {{Module, Type, Value}, {Res,Rest}}}}}; + Error -> + {error, {asn1, + {{decode, + {Module, Type, Value}, Error}}}} + end. + value(Module, Type) -> value(Module, Type, []). value(Module, Type, Includes) -> diff --git a/lib/asn1/src/asn1ct_gen.erl b/lib/asn1/src/asn1ct_gen.erl index 84c7f39d55..efbc6d6380 100644 --- a/lib/asn1/src/asn1ct_gen.erl +++ b/lib/asn1/src/asn1ct_gen.erl @@ -707,6 +707,7 @@ gen_exports([_|_]=L0, Prefix, Arity) -> pgen_dispatcher(Erules, []) -> gen_info_functions(Erules); pgen_dispatcher(Gen, Types) -> + %% MODULE HEAD emit(["-export([encode/2,decode/2]).",nl,nl]), gen_info_functions(Gen), @@ -714,6 +715,7 @@ pgen_dispatcher(Gen, Types) -> NoFinalPadding = lists:member(no_final_padding, Options), NoOkWrapper = proplists:get_bool(no_ok_wrapper, Options), + %% ENCODER Call = case Gen of #gen{erule=per,aligned=true} -> asn1ct_func:need({per,complete,1}), @@ -740,6 +742,7 @@ pgen_dispatcher(Gen, Types) -> end, emit([nl,nl]), + %% DECODER ReturnRest = proplists:get_bool(undec_rest, Gen#gen.options), Data = case Gen#gen.erule =:= ber andalso ReturnRest of true -> "Data0"; @@ -747,6 +750,12 @@ pgen_dispatcher(Gen, Types) -> end, emit(["decode(Type, ",Data,") ->",nl]), + + case NoOkWrapper of + false -> emit(["try",nl]); + true -> ok + end, + DecWrap = case {Gen,ReturnRest} of {#gen{erule=ber},false} -> @@ -754,32 +763,38 @@ pgen_dispatcher(Gen, Types) -> "element(1, ber_decode_nif(Data))"; {#gen{erule=ber},true} -> asn1ct_func:need({ber,ber_decode_nif,1}), - emit(["{Data,Rest} = ber_decode_nif(Data0),",nl]), + emit([" {Data,Rest} = ber_decode_nif(Data0),",nl]), "Data"; {_,_} -> "Data" end, - emit([case NoOkWrapper of - false -> "try"; - true -> "case" - end, " decode_disp(Type, ",DecWrap,") of",nl]), - case Gen of - #gen{erule=ber} -> - emit([" Result ->",nl]); - #gen{erule=per} -> - emit([" {Result,Rest} ->",nl]) - end, - case ReturnRest of - false -> result_line(NoOkWrapper, ["Result"]); - true -> result_line(NoOkWrapper, ["Result","Rest"]) + + DecodeDisp = ["decode_disp(Type, ",DecWrap,")"], + case {Gen,ReturnRest} of + {#gen{erule=ber},true} -> + emit([" Result = ",DecodeDisp,",",nl]), + result_line(NoOkWrapper, ["Result","Rest"]); + {#gen{erule=ber},false} -> + emit([" Result = ",DecodeDisp,",",nl]), + result_line(NoOkWrapper, ["Result"]); + + + {#gen{erule=per},true} -> + emit([" {Result,Rest} = ",DecodeDisp,",",nl]), + result_line(NoOkWrapper, ["Result","Rest"]); + {#gen{erule=per},false} -> + emit([" {Result,_Rest} = ",DecodeDisp,",",nl]), + result_line(NoOkWrapper, ["Result"]) end, + case NoOkWrapper of false -> emit([nl,try_catch(),nl,nl]); true -> - emit([nl,"end.",nl,nl]) + emit([".",nl,nl]) end, + %% REST of MODULE gen_decode_partial_incomplete(Gen), gen_partial_inc_dispatcher(Gen), @@ -787,7 +802,7 @@ pgen_dispatcher(Gen, Types) -> gen_dispatcher(Types, "decode_disp", "dec_"). result_line(NoOkWrapper, Items) -> - S = [" "|case NoOkWrapper of + S = [" "|case NoOkWrapper of false -> result_line_1(["ok"|Items]); true -> result_line_1(Items) end], diff --git a/lib/compiler/src/beam_listing.erl b/lib/compiler/src/beam_listing.erl index d1d00017e9..73c6501fe5 100644 --- a/lib/compiler/src/beam_listing.erl +++ b/lib/compiler/src/beam_listing.erl @@ -54,19 +54,6 @@ module(Stream, {Mod,Exp,Attr,Code,NumLabels}) -> [Name, Arity, Entry]), io:put_chars(Stream, format_asm(Asm)) end, Code); -module(Stream, Code) when is_binary(Code) -> - #beam_file{ module = Module, compile_info = CInfo } = beam_disasm:file(Code), - Loaded = code:is_loaded(Module), - Sticky = code:is_sticky(Module), - [code:unstick_mod(Module) || Sticky], - - {module, Module} = code:load_binary(Module, proplists:get_value(source, CInfo), Code), - ok = erts_debug:df(Stream, Module), - - %% Restore loaded module - _ = [{module, Module} = code:load_file(Module) || Loaded =/= false], - [code:stick_mod(Module) || Sticky], - ok; module(Stream, [_|_]=Fs) -> %% Form-based abstract format. foreach(fun (F) -> io:format(Stream, "~p.\n", [F]) end, Fs). diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index f7beed430c..327fecf0e6 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -787,7 +787,7 @@ asm_passes() -> | binary_passes()]. binary_passes() -> - [{iff,'to_dis',{listing,"dis"}}, + [{iff,'to_dis',?pass(to_dis)}, {native_compile,fun test_native/1,fun native_compile/2}, {unless,binary,?pass(save_binary,not_werror)} ]. @@ -1766,6 +1766,21 @@ listing(LFun, Ext, Code, St) -> {error,St#compile{errors=St#compile.errors ++ Es}} end. +to_dis(Code, #compile{module=Module,ofile=Outfile}=St) -> + Loaded = code:is_loaded(Module), + Sticky = code:is_sticky(Module), + _ = [code:unstick_mod(Module) || Sticky], + + {module,Module} = code:load_binary(Module, "", Code), + DestDir = filename:dirname(Outfile), + DisFile = filename:join(DestDir, atom_to_list(Module) ++ ".dis"), + ok = erts_debug:dis_to_file(Module, DisFile), + + %% Restore loaded module + _ = [{module, Module} = code:load_file(Module) || Loaded =/= false], + [code:stick_mod(Module) || Sticky], + {ok,Code,St}. + output_encoding(F, #compile{encoding = none}) -> ok = io:setopts(F, [{encoding, epp:default_encoding()}]); output_encoding(F, #compile{encoding = Encoding}) -> diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c index b29c5082ba..f05bfa10b3 100644 --- a/lib/crypto/c_src/crypto.c +++ b/lib/crypto/c_src/crypto.c @@ -4011,7 +4011,7 @@ static int get_pkey_private_key(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_ return PKEY_BADARG; password = get_key_password(env, key); *pkey = ENGINE_load_private_key(e, id, NULL, password); - if (!pkey) + if (!*pkey) return PKEY_BADARG; enif_free(id); #else @@ -4657,7 +4657,6 @@ static ERL_NIF_TERM pkey_crypt_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM enif_alloc_binary(outlen, &out_bin); - ERL_VALGRIND_ASSERT_MEM_DEFINED(out_bin.data, out_bin.size); if (is_private) { if (is_encrypt) { /* private_encrypt */ @@ -4795,7 +4794,6 @@ static ERL_NIF_TERM privkey_to_pubkey_nif(ErlNifEnv* env, int argc, const ERL_NI EVP_PKEY *pkey; ERL_NIF_TERM alg = argv[0]; ERL_NIF_TERM result[8]; - if (get_pkey_private_key(env, alg, argv[1], &pkey) != PKEY_OK) { return enif_make_badarg(env); } diff --git a/lib/crypto/c_src/otp_test_engine.c b/lib/crypto/c_src/otp_test_engine.c index a66bee2ddf..5c6122c06a 100644 --- a/lib/crypto/c_src/otp_test_engine.c +++ b/lib/crypto/c_src/otp_test_engine.c @@ -218,9 +218,9 @@ EVP_PKEY* test_key_load(ENGINE *er, const char *id, UI_METHOD *ui_method, void * fclose(f); if (!pkey) { - fprintf(stderr, "%s:%d Key read from file failed. ", __FILE__,__LINE__); + fprintf(stderr, "%s:%d Key read from file %s failed.\r\n", __FILE__,__LINE__,id); if (callback_data) - fprintf(stderr, "Pwd = \"%s\". ", (char *)callback_data); + fprintf(stderr, "Pwd = \"%s\".\r\n", (char *)callback_data); fprintf(stderr, "Contents of file \"%s\":\r\n",id); f = fopen(id, "r"); { /* Print the contents of the key file */ @@ -228,12 +228,14 @@ EVP_PKEY* test_key_load(ENGINE *er, const char *id, UI_METHOD *ui_method, void * while (!feof(f)) { switch (c=fgetc(f)) { case '\n': - case '\r': putc('\r',stdout); putc('\n',stdout); break; - default: putc(c, stdout); + case '\r': putc('\r',stderr); putc('\n',stderr); break; + default: putc(c, stderr); } } } + fprintf(stderr, "File contents printed.\r\n"); fclose(f); + return NULL; } return pkey; diff --git a/lib/crypto/doc/src/Makefile b/lib/crypto/doc/src/Makefile index a902779383..aa987d2b39 100644 --- a/lib/crypto/doc/src/Makefile +++ b/lib/crypto/doc/src/Makefile @@ -39,7 +39,7 @@ XML_REF3_FILES = crypto.xml XML_REF6_FILES = crypto_app.xml XML_PART_FILES = usersguide.xml -XML_CHAPTER_FILES = notes.xml licenses.xml fips.xml engine_load.xml +XML_CHAPTER_FILES = notes.xml licenses.xml fips.xml engine_load.xml engine_keys.xml BOOK_FILES = book.xml diff --git a/lib/crypto/doc/src/crypto.xml b/lib/crypto/doc/src/crypto.xml index 3ab8f9c832..c681ead797 100644 --- a/lib/crypto/doc/src/crypto.xml +++ b/lib/crypto/doc/src/crypto.xml @@ -136,11 +136,12 @@ See also <seealso marker="#supports-0">crypto:supports/0</seealso> </p> + <marker id="engine_key_ref_type"/> <code>engine_key_ref() = #{engine := engine_ref(), key_id := key_id(), password => password()}</code> - <code>engine_key_ref() = term()</code> + <code>engine_ref() = term()</code> <p>The result of a call to <seealso marker="#engine_load-3">engine_load/3</seealso>. </p> @@ -628,6 +629,10 @@ <p>Fetches the corresponding public key from a private key stored in an Engine. The key must be of the type indicated by the Type parameter. </p> + <p> + May throw exception notsup in case there is + no engine support in the underlying OpenSSL implementation. + </p> </desc> </func> diff --git a/lib/crypto/doc/src/engine_keys.xml b/lib/crypto/doc/src/engine_keys.xml new file mode 100644 index 0000000000..38714fed8a --- /dev/null +++ b/lib/crypto/doc/src/engine_keys.xml @@ -0,0 +1,129 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>2017</year><year>2017</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + </legalnotice> + <title>Engine Stored Keys</title> + <prepared>Hans Nilsson</prepared> + <date>2017-11-10</date> + <file>engine_keys.xml</file> + </header> + <p> + <marker id="engine_key"></marker> + This chapter describes the support in the crypto application for using public and private keys stored in encryption engines. + </p> + + <section> + <title>Background</title> + <p> + <url href="https://www.openssl.org/">OpenSSL</url> exposes an Engine API, which makes + it possible to plug in alternative implementations for some of the cryptographic + operations implemented by OpenSSL. + See the chapter <seealso marker="crypto:engine_load#engine_load">Engine Load</seealso> + for details and how to load an Engine. + </p> + <p> + An engine could among other tasks provide a storage for + private or public keys. Such a storage could be made safer than the normal file system. Thoose techniques are not + described in this User's Guide. Here we concentrate on how to use private or public keys stored in + such an engine. + </p> + <p> + The storage engine must call <c>ENGINE_set_load_privkey_function</c> and <c>ENGINE_set_load_pubkey_function</c>. + See the OpenSSL cryptolib's <url href="https://www.openssl.org/docs/manpages.html">manpages</url>. + </p> + <p> + OTP/Crypto requires that the user provides two or three items of information about the key. The application used + by the user is usually on a higher level, for example in + <seealso marker="ssl:ssl#key_option_def">SSL</seealso>. If using + the crypto application directly, it is required that: + </p> + <list> + <item>an Engine is loaded, see the chapter on <seealso marker="crypto:engine_load#engine_load">Engine Load</seealso> + or the <seealso marker="crypto:crypto#engine_load-3">Reference Manual</seealso> + </item> + <item>a reference to a key in the Engine is available. This should be an Erlang string or binary and depends + on the Engine loaded + </item> + <item>an Erlang map is constructed with the Engine reference, the key reference and possibly a key passphrase if + needed by the Engine. See the <seealso marker="crypto:crypto#engine_key_ref_type">Reference Manual</seealso> for + details of the map. + </item> + </list> + </section> + + <section> + <title>Use Cases</title> + <section> + <title>Sign with an engine stored private key</title> + <p> + This example shows how to construct a key reference that is used in a sign operation. + The actual key is stored in the engine that is loaded at prompt 1. + </p> + <code> +1> {ok, EngineRef} = crypto:engine_load(....). +... +{ok,#Ref<0.2399045421.3028942852.173962>} +2> PrivKey = #{engine => EngineRef, + key_id => "id of the private key in Engine"}. +... +3> Signature = crypto:sign(rsa, sha, <<"The message">>, PrivKey). +<<65,6,125,254,54,233,84,77,83,63,168,28,169,214,121,76, + 207,177,124,183,156,185,160,243,36,79,125,230,231,...>> + </code> + </section> + + <section> + <title>Verify with an engine stored public key</title> + <p> + Here the signature and message in the last example is verifyed using the public key. + The public key is stored in an engine, only to exemplify that it is possible. The public + key could of course be handled openly as usual. + </p> + <code> +4> PublicKey = #{engine => EngineRef, + key_id => "id of the public key in Engine"}. +... +5> crypto:verify(rsa, sha, <<"The message">>, Signature, PublicKey). +true +6> + </code> + </section> + + <section> + <title>Using a password protected private key</title> + <p> + The same example as the first sign example, except that a password protects the key down in the Engine. + </p> + <code> +6> PrivKeyPwd = #{engine => EngineRef, + key_id => "id of the pwd protected private key in Engine", + password => "password"}. +... +7> crypto:sign(rsa, sha, <<"The message">>, PrivKeyPwd). +<<140,80,168,101,234,211,146,183,231,190,160,82,85,163, + 175,106,77,241,141,120,72,149,181,181,194,154,175,76, + 223,...>> +8> + </code> + + </section> + + </section> +</chapter> diff --git a/lib/crypto/doc/src/usersguide.xml b/lib/crypto/doc/src/usersguide.xml index f637a1db79..e2ba1fe160 100644 --- a/lib/crypto/doc/src/usersguide.xml +++ b/lib/crypto/doc/src/usersguide.xml @@ -49,4 +49,5 @@ <xi:include href="licenses.xml"/> <xi:include href="fips.xml"/> <xi:include href="engine_load.xml"/> + <xi:include href="engine_keys.xml"/> </part> diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index 9ba1623348..91e154280f 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -1100,7 +1100,7 @@ ec_curve(X) -> privkey_to_pubkey(Alg, EngineMap) when Alg == rsa; Alg == dss; Alg == ecdsa -> - case privkey_to_pubkey_nif(Alg, format_pkey(Alg,EngineMap)) of + case notsup_to_error(privkey_to_pubkey_nif(Alg, format_pkey(Alg,EngineMap))) of [_|_]=L -> map_ensure_bin_as_int(L); X -> X end. diff --git a/lib/crypto/test/engine_SUITE.erl b/lib/crypto/test/engine_SUITE.erl index 72bd59f8ab..5967331d8e 100644 --- a/lib/crypto/test/engine_SUITE.erl +++ b/lib/crypto/test/engine_SUITE.erl @@ -53,10 +53,15 @@ groups() -> sign_verify_dsa, sign_verify_ecdsa, sign_verify_rsa_pwd, + sign_verify_rsa_pwd_bad_pwd, priv_encrypt_pub_decrypt_rsa, priv_encrypt_pub_decrypt_rsa_pwd, pub_encrypt_priv_decrypt_rsa, pub_encrypt_priv_decrypt_rsa_pwd, + get_pub_from_priv_key_rsa, + get_pub_from_priv_key_rsa_pwd, + get_pub_from_priv_key_rsa_pwd_no_pwd, + get_pub_from_priv_key_rsa_pwd_bad_pwd, get_pub_from_priv_key_dsa, get_pub_from_priv_key_ecdsa ]}]. @@ -382,6 +387,18 @@ sign_verify_rsa_pwd(Config) -> key_id => key_id(Config, "rsa_public_key_pwd.pem")}, sign_verify(rsa, sha, Priv, Pub). +sign_verify_rsa_pwd_bad_pwd(Config) -> + Priv = #{engine => engine_ref(Config), + key_id => key_id(Config, "rsa_private_key_pwd.pem"), + password => "Bad password"}, + Pub = #{engine => engine_ref(Config), + key_id => key_id(Config, "rsa_public_key_pwd.pem")}, + try sign_verify(rsa, sha, Priv, Pub) of + _ -> {fail, "PWD prot pubkey sign succeded with no pwd!"} + catch + error:badarg -> ok + end. + priv_encrypt_pub_decrypt_rsa(Config) -> Priv = #{engine => engine_ref(Config), key_id => key_id(Config, "rsa_private_key.pem")}, @@ -406,35 +423,74 @@ pub_encrypt_priv_decrypt_rsa(Config) -> pub_encrypt_priv_decrypt_rsa_pwd(Config) -> Priv = #{engine => engine_ref(Config), - key_id => key_id(Config, "rsa_private_key.pem"), + key_id => key_id(Config, "rsa_private_key_pwd.pem"), password => "password"}, Pub = #{engine => engine_ref(Config), - key_id => key_id(Config, "rsa_public_key.pem")}, + key_id => key_id(Config, "rsa_public_key_pwd.pem")}, pub_enc_priv_dec(rsa, Pub, Priv, rsa_pkcs1_padding). get_pub_from_priv_key_rsa(Config) -> Priv = #{engine => engine_ref(Config), key_id => key_id(Config, "rsa_private_key.pem")}, - Pub = crypto:privkey_to_pubkey(rsa, Priv), - ct:log("rsa Pub = ~p",[Pub]), - sign_verify(rsa, sha, Priv, Pub). + try crypto:privkey_to_pubkey(rsa, Priv) of + Pub -> + ct:log("rsa Pub = ~p",[Pub]), + sign_verify(rsa, sha, Priv, Pub) + catch + error:notsup -> {skip, "RSA not implemented"} + end. + +get_pub_from_priv_key_rsa_pwd(Config) -> + Priv = #{engine => engine_ref(Config), + key_id => key_id(Config, "rsa_private_key_pwd.pem"), + password => "password"}, + try crypto:privkey_to_pubkey(rsa, Priv) of + Pub -> + ct:log("rsa Pub = ~p",[Pub]), + sign_verify(rsa, sha, Priv, Pub) + catch + error:notsup -> {skip, "RSA not supported"} + end. + +get_pub_from_priv_key_rsa_pwd_no_pwd(Config) -> + Priv = #{engine => engine_ref(Config), + key_id => key_id(Config, "rsa_private_key_pwd.pem")}, + try crypto:privkey_to_pubkey(rsa, Priv) of + _ -> {fail, "PWD prot pubkey fetch succeded although no pwd!"} + catch + error:badarg -> ok + end. + +get_pub_from_priv_key_rsa_pwd_bad_pwd(Config) -> + Priv = #{engine => engine_ref(Config), + key_id => key_id(Config, "rsa_private_key_pwd.pem"), + password => "Bad password"}, + try crypto:privkey_to_pubkey(rsa, Priv) of + _ -> {fail, "PWD prot pubkey fetch succeded with bad pwd!"} + catch + error:badarg -> ok + end. get_pub_from_priv_key_dsa(Config) -> Priv = #{engine => engine_ref(Config), key_id => key_id(Config, "dsa_private_key.pem")}, - Pub = crypto:privkey_to_pubkey(dss, Priv), - ct:log("dsa Pub = ~p",[Pub]), - sign_verify(dss, sha, Priv, Pub). + try crypto:privkey_to_pubkey(dss, Priv) of + Pub -> + ct:log("dsa Pub = ~p",[Pub]), + sign_verify(dss, sha, Priv, Pub) + catch + error:notsup -> {skip, "DSA not supported"} + end. get_pub_from_priv_key_ecdsa(Config) -> Priv = #{engine => engine_ref(Config), key_id => key_id(Config, "ecdsa_private_key.pem")}, - Pub = crypto:privkey_to_pubkey(ecdsa, Priv), - case Pub of - notsup -> {skip, "ECDSA not implemented"}; - _ -> + try crypto:privkey_to_pubkey(ecdsa, Priv) of + Pub -> ct:log("ecdsa Pub = ~p",[Pub]), sign_verify(ecdsa, sha, Priv, Pub) + catch + error:notsup -> {skip, "ECDSA not supported"} end. %%%================================================================ diff --git a/lib/erl_interface/test/ei_accept_SUITE.erl b/lib/erl_interface/test/ei_accept_SUITE.erl index 16eb0901ef..6676e73e21 100644 --- a/lib/erl_interface/test/ei_accept_SUITE.erl +++ b/lib/erl_interface/test/ei_accept_SUITE.erl @@ -25,7 +25,8 @@ -include("ei_accept_SUITE_data/ei_accept_test_cases.hrl"). -export([all/0, suite/0, - ei_accept/1, ei_threaded_accept/1]). + ei_accept/1, ei_threaded_accept/1, + monitor_ei_process/1]). -import(runner, [get_term/1,send_term/2]). @@ -34,7 +35,8 @@ suite() -> {timetrap, {seconds, 30}}]. all() -> - [ei_accept, ei_threaded_accept]. + [ei_accept, ei_threaded_accept, + monitor_ei_process]. ei_accept(Config) when is_list(Config) -> P = runner:start(?interpret), @@ -58,6 +60,7 @@ ei_accept(Config) when is_list(Config) -> Port = 6543, {ok, ListenFd} = ei_publish(P, Port), {any, EINode} ! TermToSend, + {ok, Fd, _Node} = ei_accept(P, ListenFd), Got1 = ei_receive(P, Fd), @@ -85,6 +88,41 @@ ei_threaded_accept(Config) when is_list(Config) -> [receive I -> ok end || I <- lists:seq(0, N-1) ], ok. + +%% Test erlang:monitor toward erl_interface "processes" +monitor_ei_process(Config) when is_list(Config) -> + P = runner:start(?interpret), + 0 = ei_connect_init(P, 42, erlang:get_cookie(), 0), + + Myname = hd(tl(string:tokens(atom_to_list(node()), "@"))), + io:format("Myname ~p ~n", [Myname]), + EINode = list_to_atom("c42@"++Myname), + io:format("EINode ~p ~n", [EINode]), + + Self = self(), + Port = 6543, + {ok, ListenFd} = ei_publish(P, Port), + MRef1 = erlang:monitor(process, {any, EINode}), + {any, EINode} ! hello, + + {ok, Fd, _Node} = ei_accept(P, ListenFd), + hello = ei_receive(P, Fd), + + %% Again, now on an established connection. + MRef2 = erlang:monitor(process, {any, EINode}), + {any, EINode} ! hello, + hello = ei_receive(P, Fd), + + ok = receive M -> M after 0 -> ok end, + + runner:finish(P), + + [{'DOWN', MRef1, process, {any, EINode}, noconnection}, + {'DOWN', MRef2, process, {any, EINode}, noconnection} + ] = lists:sort(flush(2, 1000)), + + ok. + waitfornode(String,0) -> io:format("~s never published itself.~n",[String]), false; @@ -156,3 +194,12 @@ ei_receive(P, Fd) -> send_command(P, Name, Args) -> runner:send_term(P, {Name,list_to_tuple(Args)}). + +flush(0, Timeout) -> + flush(1, Timeout div 10); +flush(Expected, Timeout) -> + receive M -> + [M | flush(Expected-1, Timeout)] + after Timeout -> + [] + end. diff --git a/lib/erl_interface/test/ei_accept_SUITE_data/ei_accept_test.c b/lib/erl_interface/test/ei_accept_SUITE_data/ei_accept_test.c index 2355192cc2..2b62d7c3a4 100644 --- a/lib/erl_interface/test/ei_accept_SUITE_data/ei_accept_test.c +++ b/lib/erl_interface/test/ei_accept_SUITE_data/ei_accept_test.c @@ -224,7 +224,7 @@ static void cmd_ei_receive(char* buf, int len) if (got == ERL_TICK) continue; if (got == ERL_ERROR) - fail("ei_xreceive_msg"); + fail1("ei_xreceive_msg, got==%d", got); break; } index = 1; diff --git a/lib/erl_interface/test/runner.erl b/lib/erl_interface/test/runner.erl index 1084eec2a3..cbdeb3c8a9 100644 --- a/lib/erl_interface/test/runner.erl +++ b/lib/erl_interface/test/runner.erl @@ -67,7 +67,12 @@ start({Prog, Tc}) when is_list(Prog), is_integer(Tc) -> finish(Port) when is_port(Port) -> send_eot(Port), - recv_eot(Port). + ok = recv_eot(Port), + 0 = receive + {Port,{exit_status,Status}} -> + Status + end, + ok. %% Sends an Erlang term to a C program. diff --git a/lib/kernel/src/erts_debug.erl b/lib/kernel/src/erts_debug.erl index ea8d64b2c7..3456c8511e 100644 --- a/lib/kernel/src/erts_debug.erl +++ b/lib/kernel/src/erts_debug.erl @@ -21,7 +21,7 @@ %% Low-level debugging support. EXPERIMENTAL! --export([size/1,df/1,df/2,df/3,df/4,ic/1]). +-export([size/1,df/1,df/2,df/3,dis_to_file/2,ic/1]). %% This module contains the following *experimental* BIFs: %% disassemble/1 @@ -347,45 +347,47 @@ is_term_seen(_, []) -> false. -spec df(module()) -> df_ret(). df(Mod) when is_atom(Mod) -> - df(lists:concat([Mod, ".dis"]), Mod). - --spec df(module(), atom()) -> df_ret(); - (file:io_device() | file:filename(), module()) -> df_ret(). - -df(Mod, Func) when is_atom(Mod), is_atom(Func) -> - df(lists:concat([Mod, "_", Func, ".dis"]), Mod, Func); -df(Name, Mod) when is_atom(Mod) -> try Mod:module_info(functions) of Fs0 when is_list(Fs0) -> + Name = lists:concat([Mod, ".dis"]), Fs = [{Mod,Func,Arity} || {Func,Arity} <- Fs0], dff(Name, Fs) catch _:_ -> {undef,Mod} end. +-spec df(module(), atom()) -> df_ret(). --spec df(module(), atom(), arity()) -> df_ret(); - (file:io_device() | file:filename(), module(), atom()) -> df_ret(). - -df(Mod, Func, Arity) when is_atom(Mod), is_atom(Func), is_integer(Arity) -> - df(lists:concat([Mod, "_", Func, "_", Arity, ".dis"]), Mod, Func, Arity); -df(Name, Mod, Func) when is_atom(Mod), is_atom(Func) -> +df(Mod, Func) when is_atom(Mod), is_atom(Func) -> try Mod:module_info(functions) of Fs0 when is_list(Fs0) -> + Name = lists:concat([Mod, "_", Func, ".dis"]), Fs = [{Mod,Func1,Arity} || {Func1,Arity} <- Fs0, Func1 =:= Func], dff(Name, Fs) catch _:_ -> {undef,Mod} end. --spec df(file:io_device() | file:filename(), module(), atom(), arity()) -> df_ret(). -df(Name, Mod, Func, Arity) when is_atom(Mod), is_atom(Func), is_integer(Arity) -> +-spec df(module(), atom(), arity()) -> df_ret(). + +df(Mod, Func, Arity) when is_atom(Mod), is_atom(Func) -> try Mod:module_info(functions) of Fs0 when is_list(Fs0) -> + Name = lists:concat([Mod, "_", Func, "_", Arity, ".dis"]), Fs = [{Mod,Func1,Arity1} || {Func1,Arity1} <- Fs0, Func1 =:= Func, Arity1 =:= Arity], dff(Name, Fs) catch _:_ -> {undef,Mod} end. +-spec dis_to_file(module(), file:filename()) -> df_ret(). + +dis_to_file(Mod, Name) when is_atom(Mod) -> + try Mod:module_info(functions) of + Fs0 when is_list(Fs0) -> + Fs = [{Mod,Func,Arity} || {Func,Arity} <- Fs0], + dff(Name, Fs) + catch _:_ -> {undef,Mod} + end. + dff(Name, Fs) -> case file:open(Name, [write,raw,delayed_write]) of {ok,F} -> diff --git a/lib/kernel/test/erl_distribution_wb_SUITE.erl b/lib/kernel/test/erl_distribution_wb_SUITE.erl index 1145d30e5e..8256444bdc 100644 --- a/lib/kernel/test/erl_distribution_wb_SUITE.erl +++ b/lib/kernel/test/erl_distribution_wb_SUITE.erl @@ -677,6 +677,8 @@ build_rex_message(Cookie,OurName) -> %% Receive a distribution message recv_message(Socket) -> case gen_tcp:recv(Socket, 0) of + {ok,[]} -> + recv_message(Socket); %% a tick, ignore {ok,Data} -> B0 = list_to_binary(Data), <<?PASS_THROUGH, B1/binary>> = B0, diff --git a/lib/observer/src/crashdump_viewer.erl b/lib/observer/src/crashdump_viewer.erl index 50e9102ff0..f197e72b90 100644 --- a/lib/observer/src/crashdump_viewer.erl +++ b/lib/observer/src/crashdump_viewer.erl @@ -105,8 +105,10 @@ % line_head/1 function can return -define(not_available,"N/A"). -define(binary_size_progress_limit,10000). --define(max_dump_version,[0,4]). +-define(max_dump_version,[0,5]). +%% The value of the next define must be divisible by 4. +-define(base64_chunk_size, (4*256)). %% All possible tags - use macros in order to avoid misspelling in the code -define(abort,abort). @@ -145,6 +147,7 @@ -record(state,{file,dump_vsn,wordsize=4,num_atoms="unknown"}). +-record(dec_opts, {bin_addr_adj=0,base64=true}). %%%----------------------------------------------------------------- %%% Debugging @@ -367,10 +370,12 @@ handle_call(general_info,_From,State=#state{file=File}) -> ets:insert(cdv_reg_proc_table, {cdv_dump_node_name,GenInfo#general_info.node_name}), {reply,{ok,GenInfo,TW},State#state{wordsize=WS, num_atoms=NumAtoms}}; -handle_call({expand_binary,{Offset,Size,Pos}},_From,State=#state{file=File}) -> +handle_call({expand_binary,{Offset,Size,Pos}},_From, + #state{file=File,dump_vsn=DumpVsn}=State) -> Fd = open(File), pos_bof(Fd,Pos), - {Bin,_Line} = get_binary(Offset,Size,bytes(Fd)), + DecodeOpts = get_decode_opts(DumpVsn), + {Bin,_Line} = get_binary(Offset,Size,bytes(Fd),DecodeOpts), close(Fd), {reply,{ok,Bin},State}; handle_call(procs_summary,_From,State=#state{file=File,wordsize=WS}) -> @@ -443,9 +448,11 @@ handle_call(loaded_mods,_From,State=#state{file=File}) -> TW = truncated_warning([?mod]), {_CC,_OC,Mods} = loaded_mods(File), {reply,{ok,Mods,TW},State}; -handle_call({loaded_mod_details,Mod},_From,State=#state{file=File}) -> +handle_call({loaded_mod_details,Mod},_From, + #state{dump_vsn=DumpVsn,file=File}=State) -> TW = truncated_warning([{?mod,Mod}]), - ModInfo = get_loaded_mod_details(File,Mod), + DecodeOpts = get_decode_opts(DumpVsn), + ModInfo = get_loaded_mod_details(File,Mod,DecodeOpts), {reply,{ok,ModInfo,TW},State}; handle_call(funs,_From,State=#state{file=File}) -> TW = truncated_warning([?fu]), @@ -828,8 +835,8 @@ do_read_file(File) -> reset_tables(), insert_index(Tag,Id,N1+1), put_last_tag(Tag,""), - AddrAdj = get_bin_addr_adj(DumpVsn), - indexify(Fd,AddrAdj,Rest,N1), + DecodeOpts = get_decode_opts(DumpVsn), + indexify(Fd,DecodeOpts,Rest,N1), end_progress(), check_if_truncated(), close(Fd), @@ -877,7 +884,7 @@ check_dump_version(Vsn) -> {ok,DumpVsn} end. -indexify(Fd,AddrAdj,Bin,N) -> +indexify(Fd,DecodeOpts,Bin,N) -> case binary:match(Bin,<<"\n=">>) of {Start,Len} -> Pos = Start+Len, @@ -890,7 +897,7 @@ indexify(Fd,AddrAdj,Bin,N) -> %% order to minimize lookup time. Key is the %% translated address. {HexAddr,_} = get_hex(Id), - Addr = HexAddr bor AddrAdj, + Addr = HexAddr bor DecodeOpts#dec_opts.bin_addr_adj, insert_binary_index(Addr,NewPos); _ -> insert_index(Tag,Id,NewPos) @@ -914,7 +921,7 @@ indexify(Fd,AddrAdj,Bin,N) -> end; _ -> ok end, - indexify(Fd,AddrAdj,Rest,N1); + indexify(Fd,DecodeOpts,Rest,N1); nomatch -> case progress_read(Fd) of {ok,Chunk0} when is_binary(Chunk0) -> @@ -925,7 +932,7 @@ indexify(Fd,AddrAdj,Bin,N) -> _ -> {Chunk0,N+byte_size(Bin)} end, - indexify(Fd,AddrAdj,Chunk,N1); + indexify(Fd,DecodeOpts,Chunk,N1); eof -> eof end @@ -1441,21 +1448,21 @@ maybe_other_node2(Channel) -> expand_memory(Fd,Pid,DumpVsn) -> - BinAddrAdj = get_bin_addr_adj(DumpVsn), + DecodeOpts = get_decode_opts(DumpVsn), put(fd,Fd), Dict0 = case get(?literals) of undefined -> - Literals = read_literals(Fd), + Literals = read_literals(Fd,DecodeOpts), put(?literals,Literals), put(fd,Fd), Literals; Literals -> Literals end, - Dict = read_heap(Fd,Pid,BinAddrAdj,Dict0), - Expanded = {read_stack_dump(Fd,Pid,BinAddrAdj,Dict), - read_messages(Fd,Pid,BinAddrAdj,Dict), - read_dictionary(Fd,Pid,BinAddrAdj,Dict)}, + Dict = read_heap(Fd,Pid,DecodeOpts,Dict0), + Expanded = {read_stack_dump(Fd,Pid,DecodeOpts,Dict), + read_messages(Fd,Pid,DecodeOpts,Dict), + read_dictionary(Fd,Pid,DecodeOpts,Dict)}, erase(fd), IncompleteWarning = case erase(incomplete_heap) of @@ -1467,52 +1474,59 @@ expand_memory(Fd,Pid,DumpVsn) -> end, {Expanded,IncompleteWarning}. -read_literals(Fd) -> +read_literals(Fd,DecodeOpts) -> case lookup_index(?literals,[]) of [{_,Start}] -> [{_,Chars}] = ets:lookup(cdv_heap_file_chars,literals), init_progress("Reading literals",Chars), pos_bof(Fd,Start), - read_heap(0,gb_trees:empty()); + read_heap(DecodeOpts,gb_trees:empty()); [] -> gb_trees:empty() end. -%%%----------------------------------------------------------------- -%%% This is a workaround for a bug in dump versions prior to 0.3: -%%% Addresses were truncated to 32 bits. This could cause binaries to -%%% get the same address as heap terms in the dump. To work around it -%%% we always store binaries on very high addresses in the gb_tree. -get_bin_addr_adj(DumpVsn) when DumpVsn < [0,3] -> - 16#f bsl 64; -get_bin_addr_adj(_) -> - 0. +get_decode_opts(DumpVsn) -> + BinAddrAdj = if + DumpVsn < [0,3] -> + %% This is a workaround for a bug in dump + %% versions prior to 0.3: Addresses were + %% truncated to 32 bits. This could cause + %% binaries to get the same address as heap + %% terms in the dump. To work around it we + %% always store binaries on very high + %% addresses in the gb_tree. + 16#f bsl 64; + true -> + 0 + end, + Base64 = DumpVsn >= [0,5], + #dec_opts{bin_addr_adj=BinAddrAdj,base64=Base64}. %%% %%% Read top level section. %%% -read_stack_dump(Fd,Pid,BinAddrAdj,Dict) -> +read_stack_dump(Fd,Pid,DecodeOpts,Dict) -> case lookup_index(?proc_stack,Pid) of [{_,Start}] -> pos_bof(Fd,Start), - read_stack_dump1(Fd,BinAddrAdj,Dict,[]); + read_stack_dump1(Fd,DecodeOpts,Dict,[]); [] -> [] end. -read_stack_dump1(Fd,BinAddrAdj,Dict,Acc) -> +read_stack_dump1(Fd,DecodeOpts,Dict,Acc) -> %% This function is never called if the dump is truncated in {?proc_heap,Pid} case bytes(Fd) of "=" ++ _next_tag -> lists:reverse(Acc); Line -> - Stack = parse_top(Line,BinAddrAdj,Dict), - read_stack_dump1(Fd,BinAddrAdj,Dict,[Stack|Acc]) + Stack = parse_top(Line,DecodeOpts,Dict), + read_stack_dump1(Fd,DecodeOpts,Dict,[Stack|Acc]) end. -parse_top(Line0, BinAddrAdj, D) -> +parse_top(Line0, DecodeOpts, D) -> {Label,Line1} = get_label(Line0), - {Term,Line,D} = parse_term(Line1, BinAddrAdj, D), + {Term,Line,D} = parse_term(Line1, DecodeOpts, D), [] = skip_blanks(Line), {Label,Term}. @@ -1520,27 +1534,27 @@ parse_top(Line0, BinAddrAdj, D) -> %%% Read message queue. %%% -read_messages(Fd,Pid,BinAddrAdj,Dict) -> +read_messages(Fd,Pid,DecodeOpts,Dict) -> case lookup_index(?proc_messages,Pid) of [{_,Start}] -> pos_bof(Fd,Start), - read_messages1(Fd,BinAddrAdj,Dict,[]); + read_messages1(Fd,DecodeOpts,Dict,[]); [] -> [] end. -read_messages1(Fd,BinAddrAdj,Dict,Acc) -> +read_messages1(Fd,DecodeOpts,Dict,Acc) -> %% This function is never called if the dump is truncated in {?proc_heap,Pid} case bytes(Fd) of "=" ++ _next_tag -> lists:reverse(Acc); Line -> - Msg = parse_message(Line,BinAddrAdj,Dict), - read_messages1(Fd,BinAddrAdj,Dict,[Msg|Acc]) + Msg = parse_message(Line,DecodeOpts,Dict), + read_messages1(Fd,DecodeOpts,Dict,[Msg|Acc]) end. -parse_message(Line0, BinAddrAdj, D) -> - {Msg,":"++Line1,_} = parse_term(Line0, BinAddrAdj, D), - {Token,Line,_} = parse_term(Line1, BinAddrAdj, D), +parse_message(Line0, DecodeOpts, D) -> + {Msg,":"++Line1,_} = parse_term(Line0, DecodeOpts, D), + {Token,Line,_} = parse_term(Line1, DecodeOpts, D), [] = skip_blanks(Line), {Msg,Token}. @@ -1548,26 +1562,26 @@ parse_message(Line0, BinAddrAdj, D) -> %%% Read process dictionary %%% -read_dictionary(Fd,Pid,BinAddrAdj,Dict) -> +read_dictionary(Fd,Pid,DecodeOpts,Dict) -> case lookup_index(?proc_dictionary,Pid) of [{_,Start}] -> pos_bof(Fd,Start), - read_dictionary1(Fd,BinAddrAdj,Dict,[]); + read_dictionary1(Fd,DecodeOpts,Dict,[]); [] -> [] end. -read_dictionary1(Fd,BinAddrAdj,Dict,Acc) -> +read_dictionary1(Fd,DecodeOpts,Dict,Acc) -> %% This function is never called if the dump is truncated in {?proc_heap,Pid} case bytes(Fd) of "=" ++ _next_tag -> lists:reverse(Acc); Line -> - Msg = parse_dictionary(Line,BinAddrAdj,Dict), - read_dictionary1(Fd,BinAddrAdj,Dict,[Msg|Acc]) + Msg = parse_dictionary(Line,DecodeOpts,Dict), + read_dictionary1(Fd,DecodeOpts,Dict,[Msg|Acc]) end. -parse_dictionary(Line0, BinAddrAdj, D) -> - {Entry,Line,_} = parse_term(Line0, BinAddrAdj, D), +parse_dictionary(Line0, DecodeOpts, D) -> + {Entry,Line,_} = parse_term(Line0, DecodeOpts, D), [] = skip_blanks(Line), Entry. @@ -1575,18 +1589,18 @@ parse_dictionary(Line0, BinAddrAdj, D) -> %%% Read heap data. %%% -read_heap(Fd,Pid,BinAddrAdj,Dict0) -> +read_heap(Fd,Pid,DecodeOpts,Dict0) -> case lookup_index(?proc_heap,Pid) of [{_,Pos}] -> [{_,Chars}] = ets:lookup(cdv_heap_file_chars,Pid), init_progress("Reading process heap",Chars), pos_bof(Fd,Pos), - read_heap(BinAddrAdj,Dict0); + read_heap(DecodeOpts,Dict0); [] -> Dict0 end. -read_heap(BinAddrAdj,Dict0) -> +read_heap(DecodeOpts,Dict0) -> %% This function is never called if the dump is truncated in {?proc_heap,Pid} case get(fd) of end_of_heap -> @@ -1600,14 +1614,14 @@ read_heap(BinAddrAdj,Dict0) -> Dict0; Line -> update_progress(length(Line)+1), - Dict = parse(Line,BinAddrAdj,Dict0), - read_heap(BinAddrAdj,Dict) + Dict = parse(Line,DecodeOpts,Dict0), + read_heap(DecodeOpts,Dict) end end. -parse(Line0, BinAddrAdj, Dict0) -> +parse(Line0, DecodeOpts, Dict0) -> {Addr,":"++Line1} = get_hex(Line0), - {_Term,Line,Dict} = parse_heap_term(Line1, Addr, BinAddrAdj, Dict0), + {_Term,Line,Dict} = parse_heap_term(Line1, Addr, DecodeOpts, Dict0), [] = skip_blanks(Line), Dict. @@ -1929,12 +1943,15 @@ get_nodeinfo(Fd,Nod) -> %%----------------------------------------------------------------- %% Page with details about one loaded modules -get_loaded_mod_details(File,Mod) -> +get_loaded_mod_details(File,Mod,DecodeOpts) -> [{_,Start}] = lookup_index(?mod,Mod), Fd = open(File), pos_bof(Fd,Start), InitLM = #loaded_mod{mod=Mod,old_size="No old code exists"}, - ModInfo = get_loaded_mod_info(Fd,InitLM,fun all_modinfo/3), + Fun = fun(F, LM, LineHead) -> + all_modinfo(F, LM, LineHead, DecodeOpts) + end, + ModInfo = get_loaded_mod_info(Fd,InitLM,Fun), close(Fd), ModInfo. @@ -1992,59 +2009,44 @@ get_loaded_mod_info(Fd,LM,Fun) -> main_modinfo(_Fd,LM,_LineHead) -> LM. -all_modinfo(Fd,LM,LineHead) -> +all_modinfo(Fd,LM,LineHead,DecodeOpts) -> case LineHead of "Current attributes" -> - Str = hex_to_str(bytes(Fd,"")), + Str = get_attribute(Fd, DecodeOpts), LM#loaded_mod{current_attrib=Str}; "Current compilation info" -> - Str = hex_to_str(bytes(Fd,"")), + Str = get_attribute(Fd, DecodeOpts), LM#loaded_mod{current_comp_info=Str}; "Old attributes" -> - Str = hex_to_str(bytes(Fd,"")), + Str = get_attribute(Fd, DecodeOpts), LM#loaded_mod{old_attrib=Str}; "Old compilation info" -> - Str = hex_to_str(bytes(Fd,"")), + Str = get_attribute(Fd, DecodeOpts), LM#loaded_mod{old_comp_info=Str}; Other -> unexpected(Fd,Other,"loaded modules info"), LM end. - -hex_to_str(Hex) -> - Term = hex_to_term(Hex,[]), - io_lib:format("~tp~n",[Term]). - -hex_to_term([X,Y|Hex],Acc) -> - MS = hex_to_dec([X]), - LS = hex_to_dec([Y]), - Z = 16*MS+LS, - hex_to_term(Hex,[Z|Acc]); -hex_to_term([],Acc) -> - Bin = list_to_binary(lists:reverse(Acc)), - case catch binary_to_term(Bin) of - {'EXIT',_Reason} -> - {"WARNING: The term is probably truncated!", - "I can not do binary_to_term.", - Bin}; - Term -> - Term - end; -hex_to_term(Rest,Acc) -> - {"WARNING: The term is probably truncated!", - "I can not convert hex to term.", - Rest,list_to_binary(lists:reverse(Acc))}. - - -hex_to_dec("F") -> 15; -hex_to_dec("E") -> 14; -hex_to_dec("D") -> 13; -hex_to_dec("C") -> 12; -hex_to_dec("B") -> 11; -hex_to_dec("A") -> 10; -hex_to_dec(N) -> list_to_integer(N). - +get_attribute(Fd, DecodeOpts) -> + Bytes = bytes(Fd, ""), + try get_binary(Bytes, DecodeOpts) of + {Bin,_} -> + try binary_to_term(Bin) of + Term -> + io_lib:format("~tp~n",[Term]) + catch + _:_ -> + {"WARNING: The term is probably truncated!", + "I cannot do binary_to_term/1.", + Bin} + end + catch + _:_ -> + {"WARNING: The term is probably truncated!", + "I cannot convert to binary.", + Bytes} + end. %%----------------------------------------------------------------- %% Page with list of all funs @@ -2601,112 +2603,110 @@ get_limited_stack(Fd, N, Ds) -> %%%----------------------------------------------------------------- %%% Parse memory in crashdump version 0.1 and newer %%% -parse_heap_term([$l|Line0], Addr, BinAddrAdj, D0) -> %Cons cell. - {H,"|"++Line1,D1} = parse_term(Line0, BinAddrAdj, D0), - {T,Line,D2} = parse_term(Line1, BinAddrAdj, D1), +parse_heap_term([$l|Line0], Addr, DecodeOpts, D0) -> %Cons cell. + {H,"|"++Line1,D1} = parse_term(Line0, DecodeOpts, D0), + {T,Line,D2} = parse_term(Line1, DecodeOpts, D1), Term = [H|T], D = gb_trees:insert(Addr, Term, D2), {Term,Line,D}; -parse_heap_term([$t|Line0], Addr, BinAddrAdj, D) -> %Tuple +parse_heap_term([$t|Line0], Addr, DecodeOpts, D) -> %Tuple {N,":"++Line} = get_hex(Line0), - parse_tuple(N, Line, Addr, BinAddrAdj, D, []); -parse_heap_term([$F|Line0], Addr, _BinAddrAdj, D0) -> %Float + parse_tuple(N, Line, Addr, DecodeOpts, D, []); +parse_heap_term([$F|Line0], Addr, _DecodeOpts, D0) -> %Float {N,":"++Line1} = get_hex(Line0), {Chars,Line} = get_chars(N, Line1), Term = list_to_float(Chars), D = gb_trees:insert(Addr, Term, D0), {Term,Line,D}; -parse_heap_term("B16#"++Line0, Addr, _BinAddrAdj, D0) -> %Positive big number. +parse_heap_term("B16#"++Line0, Addr, _DecodeOpts, D0) -> %Positive big number. {Term,Line} = get_hex(Line0), D = gb_trees:insert(Addr, Term, D0), {Term,Line,D}; -parse_heap_term("B-16#"++Line0, Addr, _BinAddrAdj, D0) -> %Negative big number +parse_heap_term("B-16#"++Line0, Addr, _DecodeOpts, D0) -> %Negative big number {Term0,Line} = get_hex(Line0), Term = -Term0, D = gb_trees:insert(Addr, Term, D0), {Term,Line,D}; -parse_heap_term("B"++Line0, Addr, _BinAddrAdj, D0) -> %Decimal big num +parse_heap_term("B"++Line0, Addr, _DecodeOpts, D0) -> %Decimal big num case string:to_integer(Line0) of {Int,Line} when is_integer(Int) -> D = gb_trees:insert(Addr, Int, D0), {Int,Line,D} end; -parse_heap_term([$P|Line0], Addr, _BinAddrAdj, D0) -> % External Pid. +parse_heap_term([$P|Line0], Addr, _DecodeOpts, D0) -> % External Pid. {Pid0,Line} = get_id(Line0), Pid = ['#CDVPid'|Pid0], D = gb_trees:insert(Addr, Pid, D0), {Pid,Line,D}; -parse_heap_term([$p|Line0], Addr, _BinAddrAdj, D0) -> % External Port. +parse_heap_term([$p|Line0], Addr, _DecodeOpts, D0) -> % External Port. {Port0,Line} = get_id(Line0), Port = ['#CDVPort'|Port0], D = gb_trees:insert(Addr, Port, D0), {Port,Line,D}; -parse_heap_term("E"++Line0, Addr, _BinAddrAdj, D0) -> %Term encoded in external format. - {Bin,Line} = get_binary(Line0), +parse_heap_term("E"++Line0, Addr, DecodeOpts, D0) -> %Term encoded in external format. + {Bin,Line} = get_binary(Line0, DecodeOpts), Term = binary_to_term(Bin), D = gb_trees:insert(Addr, Term, D0), {Term,Line,D}; -parse_heap_term("Yh"++Line0, Addr, _BinAddrAdj, D0) -> %Heap binary. - {Term,Line} = get_binary(Line0), +parse_heap_term("Yh"++Line0, Addr, DecodeOpts, D0) -> %Heap binary. + {Term,Line} = get_binary(Line0, DecodeOpts), D = gb_trees:insert(Addr, Term, D0), {Term,Line,D}; -parse_heap_term("Yc"++Line0, Addr, BinAddrAdj, D0) -> %Reference-counted binary. +parse_heap_term("Yc"++Line0, Addr, DecodeOpts, D0) -> %Reference-counted binary. {Binp0,":"++Line1} = get_hex(Line0), {Offset,":"++Line2} = get_hex(Line1), {Sz,Line} = get_hex(Line2), - Binp = Binp0 bor BinAddrAdj, - Term = case lookup_binary_index(Binp) of - [{_,Start}] -> cdvbin(Offset,Sz,{'#CDVBin',Start}); - [] -> '#CDVNonexistingBinary' - end, - D = gb_trees:insert(Addr, Term, D0), - {Term,Line,D}; -parse_heap_term("Ys"++Line0, Addr, BinAddrAdj, D0) -> %Sub binary. + Binp = Binp0 bor DecodeOpts#dec_opts.bin_addr_adj, + case lookup_binary_index(Binp) of + [{_,Start}] -> + SymbolicBin = {'#CDVBin',Start}, + Term = cdvbin(Offset, Sz, SymbolicBin), + D1 = gb_trees:insert(Addr, Term, D0), + D = gb_trees:insert(Binp, SymbolicBin, D1), + {Term,Line,D}; + [] -> + Term = '#CDVNonexistingBinary', + D1 = gb_trees:insert(Addr, Term, D0), + D = gb_trees:insert(Binp, Term, D1), + {Term,Line,D} + end; +parse_heap_term("Ys"++Line0, Addr, DecodeOpts, D0) -> %Sub binary. {Binp0,":"++Line1} = get_hex(Line0), {Offset,":"++Line2} = get_hex(Line1), - {Sz,Line} = get_hex(Line2), - Binp = Binp0 bor BinAddrAdj, - Term = case lookup_binary_index(Binp) of - [{_,Start}] -> cdvbin(Offset,Sz,{'#CDVBin',Start}); - [] -> - %% Might it be on the heap? - case gb_trees:lookup(Binp, D0) of - {value,Bin} -> cdvbin(Offset,Sz,Bin); - none -> '#CDVNonexistingBinary' - end - end, - D = gb_trees:insert(Addr, Term, D0), + {Sz,Line3} = get_hex(Line2), + {Term,Line,D1} = deref_bin(Binp0, Offset, Sz, Line3, DecodeOpts, D0), + D = gb_trees:insert(Addr, Term, D1), {Term,Line,D}; -parse_heap_term("Mf"++Line0, Addr, BinAddrAdj, D0) -> %Flatmap. +parse_heap_term("Mf"++Line0, Addr, DecodeOpts, D0) -> %Flatmap. {Size,":"++Line1} = get_hex(Line0), - {Keys,":"++Line2,D1} = parse_term(Line1, BinAddrAdj, D0), - {Values,Line,D2} = parse_tuple(Size, Line2, Addr,BinAddrAdj, D1, []), + {Keys,":"++Line2,D1} = parse_term(Line1, DecodeOpts, D0), + {Values,Line,D2} = parse_tuple(Size, Line2, Addr,DecodeOpts, D1, []), Pairs = zip_tuples(tuple_size(Keys), Keys, Values, []), Map = maps:from_list(Pairs), D = gb_trees:update(Addr, Map, D2), {Map,Line,D}; -parse_heap_term("Mh"++Line0, Addr, BinAddrAdj, D0) -> %Head node in a hashmap. +parse_heap_term("Mh"++Line0, Addr, DecodeOpts, D0) -> %Head node in a hashmap. {MapSize,":"++Line1} = get_hex(Line0), {N,":"++Line2} = get_hex(Line1), - {Nodes,Line,D1} = parse_tuple(N, Line2, Addr, BinAddrAdj, D0, []), + {Nodes,Line,D1} = parse_tuple(N, Line2, Addr, DecodeOpts, D0, []), Map = maps:from_list(flatten_hashmap_nodes(Nodes)), MapSize = maps:size(Map), %Assertion. D = gb_trees:update(Addr, Map, D1), {Map,Line,D}; -parse_heap_term("Mn"++Line0, Addr, BinAddrAdj, D) -> %Interior node in a hashmap. +parse_heap_term("Mn"++Line0, Addr, DecodeOpts, D) -> %Interior node in a hashmap. {N,":"++Line} = get_hex(Line0), - parse_tuple(N, Line, Addr, BinAddrAdj, D, []). + parse_tuple(N, Line, Addr, DecodeOpts, D, []). parse_tuple(0, Line, Addr, _, D0, Acc) -> Tuple = list_to_tuple(lists:reverse(Acc)), D = gb_trees:insert(Addr, Tuple, D0), {Tuple,Line,D}; -parse_tuple(N, Line0, Addr, BinAddrAdj, D0, Acc) -> - case parse_term(Line0, BinAddrAdj, D0) of +parse_tuple(N, Line0, Addr, DecodeOpts, D0, Acc) -> + case parse_term(Line0, DecodeOpts, D0) of {Term,[$,|Line],D} when N > 1 -> - parse_tuple(N-1, Line, Addr, BinAddrAdj, D, [Term|Acc]); + parse_tuple(N-1, Line, Addr, DecodeOpts, D, [Term|Acc]); {Term,Line,D}-> - parse_tuple(N-1, Line, Addr, BinAddrAdj, D, [Term|Acc]) + parse_tuple(N-1, Line, Addr, DecodeOpts, D, [Term|Acc]) end. zip_tuples(0, _T1, _T2, Acc) -> @@ -2728,9 +2728,9 @@ flatten_hashmap_nodes_1(N, Tuple0, Acc0) -> flatten_hashmap_nodes_1(tuple_size(Tuple), Tuple, Acc) end. -parse_term([$H|Line0], BinAddrAdj, D) -> %Pointer to heap term. +parse_term([$H|Line0], DecodeOpts, D) -> %Pointer to heap term. {Ptr,Line} = get_hex(Line0), - deref_ptr(Ptr, Line, BinAddrAdj, D); + deref_ptr(Ptr, Line, DecodeOpts, D); parse_term([$N|Line], _, D) -> %[] (nil). {[],Line,D}; parse_term([$I|Line0], _, D) -> %Small. @@ -2747,11 +2747,11 @@ parse_term([$p|Line0], _, D) -> %Port. parse_term([$S|Str0], _, D) -> %Information string. Str = lists:reverse(skip_blanks(lists:reverse(Str0))), {Str,[],D}; -parse_term([$D|Line0], _, D) -> %DistExternal +parse_term([$D|Line0], DecodeOpts, D) -> %DistExternal try {AttabSize,":"++Line1} = get_hex(Line0), {Attab, "E"++Line2} = parse_atom_translation_table(AttabSize, Line1, []), - {Bin,Line3} = get_binary(Line2), + {Bin,Line3} = get_binary(Line2, DecodeOpts), {try erts_debug:dist_ext_to_term(Attab, Bin) catch @@ -2784,11 +2784,39 @@ parse_atom_translation_table(0, Line0, As) -> parse_atom_translation_table(N, Line0, As) -> {A, Line1, _} = parse_atom(Line0, []), parse_atom_translation_table(N-1, Line1, [A|As]). - - -deref_ptr(Ptr, Line, BinAddrAdj, D0) -> - case gb_trees:lookup(Ptr, D0) of + +deref_ptr(Ptr, Line, DecodeOpts, D) -> + Lookup = fun(D0) -> + gb_trees:lookup(Ptr, D0) + end, + do_deref_ptr(Lookup, Line, DecodeOpts, D). + +deref_bin(Binp0, Offset, Sz, Line, DecodeOpts, D) -> + Binp = Binp0 bor DecodeOpts#dec_opts.bin_addr_adj, + Lookup = fun(D0) -> + lookup_binary(Binp, Offset, Sz, D0) + end, + do_deref_ptr(Lookup, Line, DecodeOpts, D). + +lookup_binary(Binp, Offset, Sz, D) -> + case lookup_binary_index(Binp) of + [{_,Start}] -> + Term = cdvbin(Offset, Sz, {'#CDVBin',Start}), + {value,Term}; + [] -> + case gb_trees:lookup(Binp, D) of + {value,<<_:Offset/bytes,Sub:Sz/bytes,_/bytes>>} -> + {value,Sub}; + {value,SymbolicBin} -> + {value,cdvbin(Offset, Sz, SymbolicBin)}; + none -> + none + end + end. + +do_deref_ptr(Lookup, Line, DecodeOpts, D0) -> + case Lookup(D0) of {value,Term} -> {Term,Line,D0}; none -> @@ -2800,11 +2828,11 @@ deref_ptr(Ptr, Line, BinAddrAdj, D0) -> case bytes(Fd) of "="++_ -> put(fd, end_of_heap), - deref_ptr(Ptr, Line, BinAddrAdj, D0); + do_deref_ptr(Lookup, Line, DecodeOpts, D0); L -> update_progress(length(L)+1), - D = parse(L, BinAddrAdj, D0), - deref_ptr(Ptr, Line, BinAddrAdj, D) + D = parse(L, DecodeOpts, D0), + do_deref_ptr(Lookup, Line, DecodeOpts, D) end end end. @@ -2867,36 +2895,80 @@ get_label([$:|Line], Acc) -> get_label([H|T], Acc) -> get_label(T, [H|Acc]). -get_binary(Line0) -> +get_binary(Line0,DecodeOpts) -> case get_hex(Line0) of {N,":"++Line} -> - do_get_binary(N, Line, [], false); + get_binary_1(N, Line, DecodeOpts); _ -> {'#CDVTruncatedBinary',[]} end. -get_binary(Offset,Size,Line0) -> +get_binary_1(N,Line,#dec_opts{base64=false}) -> + get_binary_hex(N, Line, [], false); +get_binary_1(N,Line0,#dec_opts{base64=true}) -> + NumBytes = ((N+2) div 3) * 4, + {Base64,Line} = lists:split(NumBytes, Line0), + Bin = get_binary_base64(list_to_binary(Base64), <<>>, false), + {Bin,Line}. + +get_binary(Offset,Size,Line0,DecodeOpts) -> case get_hex(Line0) of {_N,":"++Line} -> - Progress = Size>?binary_size_progress_limit, - Progress andalso init_progress("Reading binary",Size), - do_get_binary(Size, lists:sublist(Line,(Offset*2)+1,Size*2), [], - Progress); - _ -> - {'#CDVTruncatedBinary',[]} - end. - -do_get_binary(0, Line, Acc, Progress) -> + get_binary_1(Offset,Size,Line,DecodeOpts); + _ -> + {'#CDVTruncatedBinary',[]} + end. + +get_binary_1(Offset,Size,Line,#dec_opts{base64=false}) -> + Progress = Size > ?binary_size_progress_limit, + Progress andalso init_progress("Reading binary",Size), + get_binary_hex(Size, lists:sublist(Line,(Offset*2)+1,Size*2), [], + Progress); +get_binary_1(StartOffset,Size,Line,#dec_opts{base64=true}) -> + Progress = Size > ?binary_size_progress_limit, + Progress andalso init_progress("Reading binary",Size), + EndOffset = StartOffset + Size, + StartByte = (StartOffset div 3) * 4, + EndByte = ((EndOffset + 2) div 3) * 4, + NumBytes = EndByte - StartByte, + case list_to_binary(Line) of + <<_:StartByte/bytes,Base64:NumBytes/bytes,_/bytes>> -> + Bin0 = get_binary_base64(Base64, <<>>, Progress), + Skip = StartOffset - (StartOffset div 3) * 3, + <<_:Skip/bytes,Bin:Size/bytes,_/bytes>> = Bin0, + {Bin,[]}; + _ -> + {'#CDVTruncatedBinary',[]} + end. + +get_binary_hex(0, Line, Acc, Progress) -> Progress andalso end_progress(), {list_to_binary(lists:reverse(Acc)),Line}; -do_get_binary(N, [A,B|Line], Acc, Progress) -> +get_binary_hex(N, [A,B|Line], Acc, Progress) -> Byte = (get_hex_digit(A) bsl 4) bor get_hex_digit(B), Progress andalso update_progress(), - do_get_binary(N-1, Line, [Byte|Acc], Progress); -do_get_binary(_N, [], _Acc, Progress) -> + get_binary_hex(N-1, Line, [Byte|Acc], Progress); +get_binary_hex(_N, [], _Acc, Progress) -> Progress andalso end_progress(), {'#CDVTruncatedBinary',[]}. +get_binary_base64(<<Chunk0:?base64_chunk_size/bytes,T/bytes>>, + Acc0, Progress) -> + Chunk = base64:decode(Chunk0), + Acc = <<Acc0/binary,Chunk/binary>>, + Progress andalso update_progress(?base64_chunk_size * 3 div 4), + get_binary_base64(T, Acc, Progress); +get_binary_base64(Chunk0, Acc, Progress) -> + case Progress of + true -> + update_progress(?base64_chunk_size * 3 div 4), + end_progress(); + false -> + ok + end, + Chunk = base64:decode(Chunk0), + <<Acc/binary,Chunk/binary>>. + cdvbin(Offset,Size,{'#CDVBin',Pos}) -> ['#CDVBin',Offset,Size,Pos]; cdvbin(Offset,Size,['#CDVBin',_,_,Pos]) -> diff --git a/lib/observer/test/crashdump_helper.erl b/lib/observer/test/crashdump_helper.erl index 41041682c2..bb1755f530 100644 --- a/lib/observer/test/crashdump_helper.erl +++ b/lib/observer/test/crashdump_helper.erl @@ -21,7 +21,7 @@ -module(crashdump_helper). -export([n1_proc/2,remote_proc/2, dump_maps/0,create_maps/0, - create_binaries/0]). + create_binaries/0,create_sub_binaries/1]). -compile(r18). -include_lib("common_test/include/ct.hrl"). @@ -64,6 +64,7 @@ n1_proc(Creator,_N2,Pid2,Port2,_L) -> put(bin,Bin), put(bins,create_binaries()), put(sub_bin,SubBin), + put(sub_bins,create_sub_binaries(get(bins))), put(bignum,83974938738373873), put(neg_bignum,-38748762783736367), put(ext_pid,Pid2), @@ -104,6 +105,17 @@ create_binaries() -> <<Data:Size/unit:8>> end || Size <- Sizes]. +create_sub_binaries(Bins) -> + [create_sub_binary(Bin, Start, LenSub) || + Bin <- Bins, + Start <- [0,1,2,3,4,5,10,22], + LenSub <- [0,1,2,3,4,6,9]]. + +create_sub_binary(Bin, Start, LenSub) -> + Len = byte_size(Bin) - LenSub - Start, + <<_:Start/bytes,Sub:Len/bytes,_/bytes>> = Bin, + Sub. + %%% %%% Test dumping of maps. Dumping of maps only from OTP 20.2. %%% diff --git a/lib/observer/test/crashdump_viewer_SUITE.erl b/lib/observer/test/crashdump_viewer_SUITE.erl index 29b9e406ae..9fbd1a62a4 100644 --- a/lib/observer/test/crashdump_viewer_SUITE.erl +++ b/lib/observer/test/crashdump_viewer_SUITE.erl @@ -403,6 +403,10 @@ special(File,Procs) -> verify_binaries(Binaries, proplists:get_value(bins,Dict)), io:format(" binaries ok",[]), + SubBinaries = crashdump_helper:create_sub_binaries(Binaries), + verify_binaries(SubBinaries, proplists:get_value(sub_bins,Dict)), + io:format(" sub binaries ok",[]), + #proc{last_calls=LastCalls} = ProcDetails, true = length(LastCalls) =< 4, @@ -461,7 +465,9 @@ special(File,Procs) -> %% i.e. no binary exist in the dump [#proc{pid=Pid0}|_Rest] = lists:keysort(#proc.name,Procs), Pid = pid_to_list(Pid0), - {ok,ProcDetails=#proc{},[]} = crashdump_viewer:proc_details(Pid), + %%WarnIncompleteHeap = ["WARNING: This process has an incomplete heap. Some information might be missing."], + {ok,ProcDetails=#proc{},[]} = + crashdump_viewer:proc_details(Pid), io:format(" process details ok",[]), #proc{dict=Dict} = ProcDetails, @@ -668,7 +674,7 @@ truncate_dump(File) -> end, %% Split after "our binary" created by crashdump_helper %% (it may not be the first binary). - RE = <<"\n=binary:(?=[0-9A-Z]+",NewLine/binary,"FF:010203)">>, + RE = <<"\n=binary:(?=[0-9A-Z]+",NewLine/binary,"FF:AQID)">>, [StartBin,AfterTag] = re:split(Bin,RE,[{parts,2}]), [AddrAndSize,BinaryAndRest] = binary:split(AfterTag,Colon), [Binary,_Rest] = binary:split(BinaryAndRest,NewLine), diff --git a/lib/public_key/doc/src/notes.xml b/lib/public_key/doc/src/notes.xml index 7a7c828760..a4c0194328 100644 --- a/lib/public_key/doc/src/notes.xml +++ b/lib/public_key/doc/src/notes.xml @@ -35,6 +35,30 @@ <file>notes.xml</file> </header> +<section><title>Public_Key 1.5.1</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Hostname verification: Add handling of the general name + <c>iPAddress</c> in certificate's subject alternative + name extension (<c>subjAltName</c>).</p> + <p> + Own Id: OTP-14653</p> + </item> + <item> + <p> + Correct key handling in pkix_test_data/1 and use a + generic example mail address instead of an existing one.</p> + <p> + Own Id: OTP-14766</p> + </item> + </list> + </section> + +</section> + <section><title>Public_Key 1.5</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/public_key/src/pubkey_ssh.erl b/lib/public_key/src/pubkey_ssh.erl index a7d018e440..02c061efc9 100644 --- a/lib/public_key/src/pubkey_ssh.erl +++ b/lib/public_key/src/pubkey_ssh.erl @@ -38,6 +38,8 @@ -define(Empint(X), (mpint(X))/binary ). -define(Estring(X), (string(X))/binary ). +-define(b64enc(X), base64:encode(iolist_to_binary(X)) ). +-define(b64mime_dec(X), base64:mime_decode(iolist_to_binary(X)) ). %% Max encoded line length is 72, but conformance examples use 68 %% Comment from rfc 4716: "The following are some examples of public @@ -163,7 +165,7 @@ rfc4716_decode_line(Line, Lines, Acc) -> rfc4716_decode_lines(Lines, [{string_decode(Tag), unicode_decode(Value)} | Acc]); _ -> {Body, Rest} = join_entry([Line | Lines], []), - {lists:reverse(Acc), rfc4716_pubkey_decode(base64:mime_decode(Body)), Rest} + {lists:reverse(Acc), rfc4716_pubkey_decode(?b64mime_dec(Body)), Rest} end. join_entry([<<"---- END SSH2 PUBLIC KEY ----", _/binary>>| Lines], Entry) -> @@ -257,11 +259,11 @@ decode_comment(Comment) -> openssh_pubkey_decode(Type, Base64Enc) -> try - <<?DEC_BIN(Type,_TL), Bin/binary>> = base64:mime_decode(Base64Enc), + <<?DEC_BIN(Type,_TL), Bin/binary>> = ?b64mime_dec(Base64Enc), ssh2_pubkey_decode(Type, Bin) catch _:_ -> - {Type, base64:mime_decode(Base64Enc)} + {Type, ?b64mime_dec(Base64Enc)} end. @@ -292,12 +294,12 @@ do_encode(Type, Key, Attributes) -> rfc4716_encode(Key, [],[]) -> iolist_to_binary([begin_marker(),"\n", - split_lines(base64:encode(ssh2_pubkey_encode(Key))), + split_lines(?b64enc(ssh2_pubkey_encode(Key))), "\n", end_marker(), "\n"]); rfc4716_encode(Key, [], [_|_] = Acc) -> iolist_to_binary([begin_marker(), "\n", lists:reverse(Acc), - split_lines(base64:encode(ssh2_pubkey_encode(Key))), + split_lines(?b64enc(ssh2_pubkey_encode(Key))), "\n", end_marker(), "\n"]); rfc4716_encode(Key, [ Header | Headers], Acc) -> LinesStr = rfc4716_encode_header(Header), @@ -326,7 +328,7 @@ rfc4716_encode_value(Value) -> openssh_encode(openssh_public_key, Key, Attributes) -> Comment = proplists:get_value(comment, Attributes, ""), - Enc = base64:encode(ssh2_pubkey_encode(Key)), + Enc = ?b64enc(ssh2_pubkey_encode(Key)), iolist_to_binary([key_type(Key), " ", Enc, " ", Comment, "\n"]); openssh_encode(auth_keys, Key, Attributes) -> @@ -351,10 +353,10 @@ openssh_encode(known_hosts, Key, Attributes) -> end. openssh_ssh2_auth_keys_encode(undefined, Key, Comment) -> - iolist_to_binary([key_type(Key)," ", base64:encode(ssh2_pubkey_encode(Key)), line_end(Comment)]); + iolist_to_binary([key_type(Key)," ", ?b64enc(ssh2_pubkey_encode(Key)), line_end(Comment)]); openssh_ssh2_auth_keys_encode(Options, Key, Comment) -> iolist_to_binary([comma_list_encode(Options, []), " ", - key_type(Key)," ", base64:encode(ssh2_pubkey_encode(Key)), line_end(Comment)]). + key_type(Key)," ", ?b64enc(ssh2_pubkey_encode(Key)), line_end(Comment)]). openssh_ssh1_auth_keys_encode(undefined, Bits, #'RSAPublicKey'{modulus = N, publicExponent = E}, @@ -369,7 +371,7 @@ openssh_ssh1_auth_keys_encode(Options, Bits, openssh_ssh2_know_hosts_encode(Hostnames, Key, Comment) -> iolist_to_binary([comma_list_encode(Hostnames, []), " ", - key_type(Key)," ", base64:encode(ssh2_pubkey_encode(Key)), line_end(Comment)]). + key_type(Key)," ", ?b64enc(ssh2_pubkey_encode(Key)), line_end(Comment)]). openssh_ssh1_known_hosts_encode(Hostnames, Bits, #'RSAPublicKey'{modulus = N, publicExponent = E}, diff --git a/lib/public_key/test/public_key_SUITE.erl b/lib/public_key/test/public_key_SUITE.erl index 38e8f30a25..9e5e288a1a 100644 --- a/lib/public_key/test/public_key_SUITE.erl +++ b/lib/public_key/test/public_key_SUITE.erl @@ -1034,8 +1034,6 @@ pkix_verify_hostname_subjAltName_IP(Config) -> true = public_key:pkix_verify_hostname(Cert, [{ip, {10,67,16,75}}]), false = public_key:pkix_verify_hostname(Cert, [{ip, {1,2,3,4}}]), false = public_key:pkix_verify_hostname(Cert, [{ip, {10,11,12,13}}]). - - %%-------------------------------------------------------------------- pkix_iso_rsa_oid() -> [{doc, "Test workaround for supporting certs that use ISO oids" diff --git a/lib/public_key/test/public_key_SUITE_data/pkix_verify_hostname_subjAltName_IP.pem b/lib/public_key/test/public_key_SUITE_data/pkix_verify_hostname_subjAltName_IP.pem index 97d12cdadf..dc20285f30 100644 --- a/lib/public_key/test/public_key_SUITE_data/pkix_verify_hostname_subjAltName_IP.pem +++ b/lib/public_key/test/public_key_SUITE_data/pkix_verify_hostname_subjAltName_IP.pem @@ -1,4 +1,5 @@ -----BEGIN CERTIFICATE----- +<<<<<<< HEAD MIICBzCCAXCgAwIBAgIJAJgbo5FL73LuMA0GCSqGSIb3DQEBCwUAMCMxCzAJBgNV BAYTAlNFMRQwEgYDVQQDEwtleGFtcGxlLmNvbTAeFw0xNzEwMTExMDM0NDJaFw0x NzExMTAxMDM0NDJaMCMxCzAJBgNVBAYTAlNFMRQwEgYDVQQDEwtleGFtcGxlLmNv @@ -10,4 +11,17 @@ S4cQq80A7wAAAAAAAAAAAAAAAYYTaHR0cHM6Ly8xMC4xMS4xMi4xMzANBgkqhkiG 9w0BAQsFAAOBgQDMn8aqs/5FkkWhspvN2n+D2l87M+33a5My54ZVZhayZ/KRmhCN Gix/BiVYJ3UlmWmGcnQXb3MLt/LQHaD3S2whDaLN3xJ8BbnX7A4ZTybitdyeFhDw K3iDVUM3bSsBJ4EcBPWIMnow3ALP5HlGRMlH/87Qt+uVPXuwNh9pmyIhRQ== +======= +MIIB/zCCAWigAwIBAgIJAMoSejmTjwAGMA0GCSqGSIb3DQEBCwUAMB8xCzAJBgNV +BAYTAlNFMRAwDgYDVQQDEwc1LjYuNy44MB4XDTE3MDkyODE0MDAxNVoXDTE3MTAy +ODE0MDAxNVowHzELMAkGA1UEBhMCU0UxEDAOBgNVBAMTBzUuNi43LjgwgZ8wDQYJ +KoZIhvcNAQEBBQADgY0AMIGJAoGBAMUPU89KwVbTCDkyxQSz3wprMbZTLe35K6jm +Q7oY1rJyVXjsFHwZrFqqNMScEyX40rJhczQ2Z9etEX6qYLbdb/DZeFcKo14fR583 +QMFZC+qqpLWHdvjaQN0KwD99VFeZIGpRgywG8SR+BXZjDHUkGsMrikAEJtf0Tgih +IPyiFtiJAgMBAAGjQzBBMD8GA1UdEQQ4MDaCBzEuMi4zLjSHBAUGBwiHEKvNAO8A +AAAAAAAAAAAAAAGGE2h0dHBzOi8vMTAuMTEuMTIuMTMwDQYJKoZIhvcNAQELBQAD +gYEAtWVeQaRFZ0kH/pzSWMSsOCUrjbwlWRwDNbagNKoM6nCRv0QQ59fG6XrVZwR3 +c0s5arlMh3U2+bjKE+Iq9+b/lN1lGzf8iaAqBNa7KptwTSUEY3TiNG5X0zlSXKTI +3z7AaUEtghL9ImCPj5V3tVksqWd7U0zLmeeLZnM+wGAL9Hc= +>>>>>>> maint-20 -----END CERTIFICATE----- diff --git a/lib/public_key/test/public_key_SUITE_data/verify_hostname_ip.conf b/lib/public_key/test/public_key_SUITE_data/verify_hostname_ip.conf index 798592e4f6..f27dac07ec 100644 --- a/lib/public_key/test/public_key_SUITE_data/verify_hostname_ip.conf +++ b/lib/public_key/test/public_key_SUITE_data/verify_hostname_ip.conf @@ -5,13 +5,21 @@ distinguished_name = DN [DN] C=SE CN=example.com +<<<<<<< HEAD +======= +CN=5.6.7.8 +>>>>>>> maint-20 [SAN] subjectAltName = @alt_names [alt_names] DNS = 1.2.3.4 +<<<<<<< HEAD IP.1 = 10.67.16.75 +======= +IP.1 = 5.6.7.8 +>>>>>>> maint-20 IP.2 = abcd:ef::1 URI = https://10.11.12.13 diff --git a/lib/public_key/vsn.mk b/lib/public_key/vsn.mk index bb96c2237d..c01d8820f2 100644 --- a/lib/public_key/vsn.mk +++ b/lib/public_key/vsn.mk @@ -1 +1 @@ -PUBLIC_KEY_VSN = 1.5 +PUBLIC_KEY_VSN = 1.5.1 diff --git a/lib/ssh/test/ssh_engine_SUITE.erl b/lib/ssh/test/ssh_engine_SUITE.erl index 035446932b..daf93891e9 100644 --- a/lib/ssh/test/ssh_engine_SUITE.erl +++ b/lib/ssh/test/ssh_engine_SUITE.erl @@ -57,7 +57,6 @@ init_per_suite(Config) -> ?CHECK_CRYPTO( case load_engine() of {ok,E} -> - ssh_dbg:messages(fun ct:pal/2), [{engine,E}|Config]; {error, notsup} -> {skip, "Engine not supported on this OpenSSL version"}; diff --git a/lib/ssl/doc/src/notes.xml b/lib/ssl/doc/src/notes.xml index 4c6a204e63..37c916e585 100644 --- a/lib/ssl/doc/src/notes.xml +++ b/lib/ssl/doc/src/notes.xml @@ -27,6 +27,81 @@ </header> <p>This document describes the changes made to the SSL application.</p> +<section><title>SSL 8.2.2</title> + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + TLS sessions must be registered with SNI if provided, so + that sessions where client hostname verification would + fail can not connect reusing a session created when the + server name verification succeeded.</p> + <p> + Own Id: OTP-14632</p> + </item> + <item> + <p> An erlang TLS server configured with cipher suites + using rsa key exchange, may be vulnerable to an Adaptive + Chosen Ciphertext attack (AKA Bleichenbacher attack) + against RSA, which when exploited, may result in + plaintext recovery of encrypted messages and/or a + Man-in-the-middle (MiTM) attack, despite the attacker not + having gained access to the server’s private key + itself. <url + href="https://nvd.nist.gov/vuln/detail/CVE-2017-1000385">CVE-2017-1000385</url> + </p> <p> Exploiting this vulnerability to perform + plaintext recovery of encrypted messages will, in most + practical cases, allow an attacker to read the plaintext + only after the session has completed. Only TLS sessions + established using RSA key exchange are vulnerable to this + attack. </p> <p> Exploiting this vulnerability to conduct + a MiTM attack requires the attacker to complete the + initial attack, which may require thousands of server + requests, during the handshake phase of the targeted + session within the window of the configured handshake + timeout. This attack may be conducted against any TLS + session using RSA signatures, but only if cipher suites + using RSA key exchange are also enabled on the server. + The limited window of opportunity, limitations in + bandwidth, and latency make this attack significantly + more difficult to execute. </p> <p> RSA key exchange is + enabled by default although least prioritized if server + order is honored. For such a cipher suite to be chosen it + must also be supported by the client and probably the + only shared cipher suite. </p> <p> Captured TLS sessions + encrypted with ephemeral cipher suites (DHE or ECDHE) are + not at risk for subsequent decryption due to this + vulnerability. </p> <p> As a workaround if default cipher + suite configuration was used you can configure the server + to not use vulnerable suites with the ciphers option like + this: </p> <c> {ciphers, [Suite || Suite <- + ssl:cipher_suites(), element(1,Suite) =/= rsa]} </c> <p> + that is your code will look somethingh like this: </p> + <c> ssl:listen(Port, [{ciphers, [Suite || Suite <- + ssl:cipher_suites(), element(1,S) =/= rsa]} | Options]). + </c> <p> Thanks to Hanno Böck, Juraj Somorovsky and + Craig Young for reporting this vulnerability. </p> + <p> + Own Id: OTP-14748</p> + </item> + </list> + </section> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + If no SNI is available and the hostname is an IP-address + also check for IP-address match. This check is not as + good as a DNS hostname check and certificates using + IP-address are not recommended.</p> + <p> + Own Id: OTP-14655</p> + </item> + </list> + </section> + +</section> <section><title>SSL 8.2.1</title> @@ -175,9 +250,59 @@ </item> </list> </section> - </section> +<section><title>SSL 8.1.3.1</title> + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> An erlang TLS server configured with cipher suites + using rsa key exchange, may be vulnerable to an Adaptive + Chosen Ciphertext attack (AKA Bleichenbacher attack) + against RSA, which when exploited, may result in + plaintext recovery of encrypted messages and/or a + Man-in-the-middle (MiTM) attack, despite the attacker not + having gained access to the server’s private key + itself. <url + href="https://nvd.nist.gov/vuln/detail/CVE-2017-1000385">CVE-2017-1000385</url> + </p> <p> Exploiting this vulnerability to perform + plaintext recovery of encrypted messages will, in most + practical cases, allow an attacker to read the plaintext + only after the session has completed. Only TLS sessions + established using RSA key exchange are vulnerable to this + attack. </p> <p> Exploiting this vulnerability to conduct + a MiTM attack requires the attacker to complete the + initial attack, which may require thousands of server + requests, during the handshake phase of the targeted + session within the window of the configured handshake + timeout. This attack may be conducted against any TLS + session using RSA signatures, but only if cipher suites + using RSA key exchange are also enabled on the server. + The limited window of opportunity, limitations in + bandwidth, and latency make this attack significantly + more difficult to execute. </p> <p> RSA key exchange is + enabled by default although least prioritized if server + order is honored. For such a cipher suite to be chosen it + must also be supported by the client and probably the + only shared cipher suite. </p> <p> Captured TLS sessions + encrypted with ephemeral cipher suites (DHE or ECDHE) are + not at risk for subsequent decryption due to this + vulnerability. </p> <p> As a workaround if default cipher + suite configuration was used you can configure the server + to not use vulnerable suites with the ciphers option like + this: </p> <c> {ciphers, [Suite || Suite <- + ssl:cipher_suites(), element(1,Suite) =/= rsa]} </c> <p> + that is your code will look somethingh like this: </p> + <c> ssl:listen(Port, [{ciphers, [Suite || Suite <- + ssl:cipher_suites(), element(1,S) =/= rsa]} | Options]). + </c> <p> Thanks to Hanno Böck, Juraj Somorovsky and + Craig Young for reporting this vulnerability. </p> + <p> + Own Id: OTP-14748</p> + </item> + </list> + </section> +</section> <section><title>SSL 8.1.3</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -556,6 +681,60 @@ </section> + <section><title>SSL 7.3.3.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> An erlang TLS server configured with cipher suites + using rsa key exchange, may be vulnerable to an Adaptive + Chosen Ciphertext attack (AKA Bleichenbacher attack) + against RSA, which when exploited, may result in + plaintext recovery of encrypted messages and/or a + Man-in-the-middle (MiTM) attack, despite the attacker not + having gained access to the server’s private key + itself. <url + href="https://nvd.nist.gov/vuln/detail/CVE-2017-1000385">CVE-2017-1000385</url> + </p> <p> Exploiting this vulnerability to perform + plaintext recovery of encrypted messages will, in most + practical cases, allow an attacker to read the plaintext + only after the session has completed. Only TLS sessions + established using RSA key exchange are vulnerable to this + attack. </p> <p> Exploiting this vulnerability to conduct + a MiTM attack requires the attacker to complete the + initial attack, which may require thousands of server + requests, during the handshake phase of the targeted + session within the window of the configured handshake + timeout. This attack may be conducted against any TLS + session using RSA signatures, but only if cipher suites + using RSA key exchange are also enabled on the server. + The limited window of opportunity, limitations in + bandwidth, and latency make this attack significantly + more difficult to execute. </p> <p> RSA key exchange is + enabled by default although least prioritized if server + order is honored. For such a cipher suite to be chosen it + must also be supported by the client and probably the + only shared cipher suite. </p> <p> Captured TLS sessions + encrypted with ephemeral cipher suites (DHE or ECDHE) are + not at risk for subsequent decryption due to this + vulnerability. </p> <p> As a workaround if default cipher + suite configuration was used you can configure the server + to not use vulnerable suites with the ciphers option like + this: </p> <c> {ciphers, [Suite || Suite <- + ssl:cipher_suites(), element(1,Suite) =/= rsa]} </c> <p> + that is your code will look somethingh like this: </p> + <c> ssl:listen(Port, [{ciphers, [Suite || Suite <- + ssl:cipher_suites(), element(1,S) =/= rsa]} | Options]). + </c> <p> Thanks to Hanno Böck, Juraj Somorovsky and + Craig Young for reporting this vulnerability. </p> + <p> + Own Id: OTP-14748</p> + </item> + </list> + </section> + + </section> + <section><title>SSL 7.3.3</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -585,7 +764,59 @@ </list> </section> + <section><title>SSL 7.3.3.0.1</title> + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> An erlang TLS server configured with cipher suites + using rsa key exchange, may be vulnerable to an Adaptive + Chosen Ciphertext attack (AKA Bleichenbacher attack) + against RSA, which when exploited, may result in + plaintext recovery of encrypted messages and/or a + Man-in-the-middle (MiTM) attack, despite the attacker not + having gained access to the server’s private key + itself. <url + href="https://nvd.nist.gov/vuln/detail/CVE-2017-1000385">CVE-2017-1000385</url> + </p> <p> Exploiting this vulnerability to perform + plaintext recovery of encrypted messages will, in most + practical cases, allow an attacker to read the plaintext + only after the session has completed. Only TLS sessions + established using RSA key exchange are vulnerable to this + attack. </p> <p> Exploiting this vulnerability to conduct + a MiTM attack requires the attacker to complete the + initial attack, which may require thousands of server + requests, during the handshake phase of the targeted + session within the window of the configured handshake + timeout. This attack may be conducted against any TLS + session using RSA signatures, but only if cipher suites + using RSA key exchange are also enabled on the server. + The limited window of opportunity, limitations in + bandwidth, and latency make this attack significantly + more difficult to execute. </p> <p> RSA key exchange is + enabled by default although least prioritized if server + order is honored. For such a cipher suite to be chosen it + must also be supported by the client and probably the + only shared cipher suite. </p> <p> Captured TLS sessions + encrypted with ephemeral cipher suites (DHE or ECDHE) are + not at risk for subsequent decryption due to this + vulnerability. </p> <p> As a workaround if default cipher + suite configuration was used you can configure the server + to not use vulnerable suites with the ciphers option like + this: </p> <c> {ciphers, [Suite || Suite <- + ssl:cipher_suites(), element(1,Suite) =/= rsa]} </c> <p> + that is your code will look somethingh like this: </p> + <c> ssl:listen(Port, [{ciphers, [Suite || Suite <- + ssl:cipher_suites(), element(1,S) =/= rsa]} | Options]). + </c> <p> Thanks to Hanno Böck, Juraj Somorovsky and + Craig Young for reporting this vulnerability. </p> + <p> + Own Id: OTP-14748</p> + </item> + </list> + </section> + + </section> <section><title>Improvements and New Features</title> <list> <item> diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index ac5a69c69b..8fcda78ed5 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -203,7 +203,9 @@ <tag><c>{certfile, path()}</c></tag> <item><p>Path to a file containing the user certificate.</p></item> - <tag><c>{key, {'RSAPrivateKey'| 'DSAPrivateKey' | 'ECPrivateKey' + <tag> + <marker id="key_option_def"/> + <c>{key, {'RSAPrivateKey'| 'DSAPrivateKey' | 'ECPrivateKey' |'PrivateKeyInfo', public_key:der_encoded()} | #{algorithm := rsa | dss | ecdsa, engine := crypto:engine_ref(), key_id := crypto:key_id(), password => crypto:password()}</c></tag> <item><p>The DER-encoded user's private key or a map refering to a crypto diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl index eed49ca090..90df298c4a 100644 --- a/lib/ssl/src/dtls_connection.erl +++ b/lib/ssl/src/dtls_connection.erl @@ -461,7 +461,6 @@ init({call, _} = Type, Event, #state{role = server} = State) -> ssl_connection:?FUNCTION_NAME(Type, Event, State#state{flight_state = reliable}, ?MODULE); init(Type, Event, State) -> ssl_connection:?FUNCTION_NAME(Type, Event, State, ?MODULE). - %%-------------------------------------------------------------------- -spec error(gen_statem:event_type(), {start, timeout()} | term(), #state{}) -> @@ -596,6 +595,7 @@ abbreviated(state_timeout, Event, State) -> handle_state_timeout(Event, ?FUNCTION_NAME, State); abbreviated(Type, Event, State) -> ssl_connection:?FUNCTION_NAME(Type, Event, State, ?MODULE). + %%-------------------------------------------------------------------- -spec certify(gen_statem:event_type(), term(), #state{}) -> gen_statem:state_function_result(). @@ -807,6 +807,7 @@ handle_client_hello(#client_hello{client_version = ClientVersion} = Hello, State = prepare_flight(State0#state{connection_states = ConnectionStates, negotiated_version = Version, hashsign_algorithm = HashSign, + client_hello_version = ClientVersion, session = Session, negotiated_protocol = Protocol}), diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index 79485833e0..414b181078 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -1541,8 +1541,25 @@ server_certify_and_key_exchange(State0, Connection) -> request_client_cert(State2, Connection). certify_client_key_exchange(#encrypted_premaster_secret{premaster_secret= EncPMS}, - #state{private_key = Key} = State, Connection) -> - PremasterSecret = ssl_handshake:premaster_secret(EncPMS, Key), + #state{private_key = Key, client_hello_version = {Major, Minor} = Version} = State, Connection) -> + + %% Countermeasure for Bleichenbacher attack always provide some kind of premaster secret + %% and fail handshake later.RFC 5246 section 7.4.7.1. + PremasterSecret = + try ssl_handshake:premaster_secret(EncPMS, Key) of + Secret when erlang:byte_size(Secret) == ?NUM_OF_PREMASTERSECRET_BYTES -> + case Secret of + <<?BYTE(Major), ?BYTE(Minor), _/binary>> -> %% Correct + Secret; + <<?BYTE(_), ?BYTE(_), Rest/binary>> -> %% Version mismatch + <<?BYTE(Major), ?BYTE(Minor), Rest/binary>> + end; + _ -> %% erlang:byte_size(Secret) =/= ?NUM_OF_PREMASTERSECRET_BYTES + make_premaster_secret(Version, rsa) + catch + #alert{description = ?DECRYPT_ERROR} -> + make_premaster_secret(Version, rsa) + end, calculate_master_secret(PremasterSecret, State, Connection, certify, cipher); certify_client_key_exchange(#client_diffie_hellman_public{dh_public = ClientPublicDhKey}, #state{diffie_hellman_params = #'DHParameter'{} = Params, diff --git a/lib/ssl/src/ssl_connection.hrl b/lib/ssl/src/ssl_connection.hrl index 3e26f67de1..f9d2149170 100644 --- a/lib/ssl/src/ssl_connection.hrl +++ b/lib/ssl/src/ssl_connection.hrl @@ -57,6 +57,7 @@ session_cache_cb :: atom(), crl_db :: term(), negotiated_version :: ssl_record:ssl_version() | 'undefined', + client_hello_version :: ssl_record:ssl_version() | 'undefined', client_certificate_requested = false :: boolean(), key_algorithm :: ssl_cipher:key_algo(), hashsign_algorithm = {undefined, undefined}, diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index 1560340ccf..24f3a97b9b 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -1224,7 +1224,6 @@ certificate_authorities_from_db(_CertDbHandle, {extracted, CertDbData}) -> [], CertDbData). %%-------------Handle handshake messages -------------------------------- - validation_fun_and_state({Fun, UserState0}, Role, CertDbHandle, CertDbRef, ServerNameIndication, CRLCheck, CRLDbHandle, CertPath) -> {fun(OtpCert, {extension, _} = Extension, {SslState, UserState}) -> diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index 43422fab8d..a43e0ac56e 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -476,6 +476,7 @@ hello(internal, #client_hello{client_version = ClientVersion} = Hello, gen_handshake(ssl_connection, hello, internal, {common_client_hello, Type, ServerHelloExt}, State#state{connection_states = ConnectionStates, negotiated_version = Version, + client_hello_version = ClientVersion, hashsign_algorithm = HashSign, session = Session, negotiated_protocol = Protocol}) diff --git a/lib/ssl/vsn.mk b/lib/ssl/vsn.mk index bb77326751..cf6481d14c 100644 --- a/lib/ssl/vsn.mk +++ b/lib/ssl/vsn.mk @@ -1 +1 @@ -SSL_VSN = 8.2.1 +SSL_VSN = 8.2.2 |