diff options
Diffstat (limited to 'lib')
53 files changed, 1072 insertions, 1198 deletions
diff --git a/lib/compiler/src/beam_type.erl b/lib/compiler/src/beam_type.erl index 2b5d558ee4..7e9a243ada 100644 --- a/lib/compiler/src/beam_type.erl +++ b/lib/compiler/src/beam_type.erl @@ -26,6 +26,8 @@ -import(lists, [filter/2,foldl/3,keyfind/3,member/2, reverse/1,reverse/2,sort/1]). +-define(UNICODE_INT, {integer,{0,16#10FFFF}}). + -spec module(beam_utils:module_code(), [compile:option()]) -> {'ok',beam_utils:module_code()}. @@ -494,6 +496,10 @@ update({test,test_arity,_Fail,[Src,Arity]}, Ts0) -> tdb_update([{Src,{tuple,Arity,[]}}], Ts0); update({test,is_map,_Fail,[Src]}, Ts0) -> tdb_update([{Src,map}], Ts0); +update({get_map_elements,_,Src,{list,Elems0}}, Ts0) -> + {_Ss,Ds} = beam_utils:split_even(Elems0), + Elems = [{Dst,kill} || Dst <- Ds], + tdb_update([{Src,map}|Elems], Ts0); update({test,is_nonempty_list,_Fail,[Src]}, Ts0) -> tdb_update([{Src,nonempty_list}], Ts0); update({test,is_eq_exact,_,[Reg,{atom,_}=Atom]}, Ts) -> @@ -507,10 +513,39 @@ update({test,is_eq_exact,_,[Reg,{atom,_}=Atom]}, Ts) -> end; update({test,is_record,_Fail,[Src,Tag,{integer,Arity}]}, Ts) -> tdb_update([{Src,{tuple,Arity,[Tag]}}], Ts); -update({test,_Test,_Fail,_Other}, Ts) -> - Ts; + +%% Binary matching + update({test,bs_get_integer2,_,_,Args,Dst}, Ts) -> tdb_update([{Dst,get_bs_integer_type(Args)}], Ts); +update({test,bs_get_utf8,_,_,_,Dst}, Ts) -> + tdb_update([{Dst,?UNICODE_INT}], Ts); +update({test,bs_get_utf16,_,_,_,Dst}, Ts) -> + tdb_update([{Dst,?UNICODE_INT}], Ts); +update({test,bs_get_utf32,_,_,_,Dst}, Ts) -> + tdb_update([{Dst,?UNICODE_INT}], Ts); +update({bs_init,_,_,_,_,Dst}, Ts) -> + tdb_update([{Dst,kill}], Ts); +update({bs_put,_,_,_}, Ts) -> + Ts; +update({bs_save2,_,_}, Ts) -> + Ts; +update({bs_restore2,_,_}, Ts) -> + Ts; +update({bs_context_to_binary,Dst}, Ts) -> + tdb_update([{Dst,kill}], Ts); +update({test,bs_start_match2,_,_,_,Dst}, Ts) -> + tdb_update([{Dst,kill}], Ts); +update({test,bs_get_binary2,_,_,_,Dst}, Ts) -> + tdb_update([{Dst,kill}], Ts); +update({test,bs_get_float2,_,_,_,Dst}, Ts) -> + tdb_update([{Dst,float}], Ts); + +update({test,_Test,_Fail,_Other}, Ts) -> + Ts; + +%% Calls + update({call_ext,Ar,{extfunc,math,Math,Ar}}, Ts) -> case is_math_bif(Math, Ar) of true -> tdb_update([{{x,0},float}], Ts); @@ -537,9 +572,10 @@ update({call_ext,3,{extfunc,erlang,setelement,3}}, Ts0) -> update({call,_Arity,_Func}, Ts) -> tdb_kill_xregs(Ts); update({call_ext,_Arity,_Func}, Ts) -> tdb_kill_xregs(Ts); update({make_fun2,_,_,_,_}, Ts) -> tdb_kill_xregs(Ts); +update({call_fun, _}, Ts) -> tdb_kill_xregs(Ts); +update({apply, _}, Ts) -> tdb_kill_xregs(Ts); + update({line,_}, Ts) -> Ts; -update({bs_save2,_,_}, Ts) -> Ts; -update({bs_restore2,_,_}, Ts) -> Ts; %% The instruction is unknown. Kill all information. update(_I, _Ts) -> tdb_new(). diff --git a/lib/crypto/doc/src/crypto.xml b/lib/crypto/doc/src/crypto.xml index 552d95d7dc..2718ee9055 100644 --- a/lib/crypto/doc/src/crypto.xml +++ b/lib/crypto/doc/src/crypto.xml @@ -521,7 +521,7 @@ scheme. <c>VerStr</c> contains a text variant of the version.</p> <pre> > <input>info_lib().</input> -[{<<"OpenSSL">>,9469983,<<"OpenSSL 0.9.8a 11 Oct 2005">>}] +[{<<"OpenSSL">>,269484095,<<"OpenSSL 1.1.0c 10 Nov 2016"">>}] </pre> <note><p> From OTP R16 the <em>numeric version</em> represents the version of the OpenSSL diff --git a/lib/crypto/doc/src/crypto_app.xml b/lib/crypto/doc/src/crypto_app.xml index a958bdfcb7..6950dfeec3 100644 --- a/lib/crypto/doc/src/crypto_app.xml +++ b/lib/crypto/doc/src/crypto_app.xml @@ -42,9 +42,12 @@ <title>DEPENDENCIES</title> <p>The current crypto implementation uses nifs to interface - OpenSSLs crypto library and requires <em>OpenSSL</em> package - version 0.9.8 or higher. FIPS mode support requires at least - version 1.0.1 and a FIPS capable OpenSSL installation.</p> + OpenSSLs crypto library and may work with limited functionality + with as old versions as <em>OpenSSL</em> 0.9.8c. + FIPS mode support requires at least + version 1.0.1 and a FIPS capable OpenSSL installation. We recommend using a + version that is officially supported by the OpenSSL project. API compatible backends like + LibreSSL should also work.</p> <p>Source releases of OpenSSL can be downloaded from the <url href="http://www.openssl.org">OpenSSL</url> project home page, or mirror sites listed there. diff --git a/lib/dialyzer/doc/src/dialyzer.xml b/lib/dialyzer/doc/src/dialyzer.xml index 4b7eb4ad68..e34ffd6def 100644 --- a/lib/dialyzer/doc/src/dialyzer.xml +++ b/lib/dialyzer/doc/src/dialyzer.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2006</year><year>2016</year> + <year>2006</year><year>2017</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -457,11 +457,6 @@ dialyzer --plts plt_1 ... plt_n -- files_to_analyze</code> <c>gui/1</c></seealso> below (<c>WarnOpts</c>).</p> <note> - <p>Attribute <c>-dialyzer()</c> is not checked by the Erlang - compiler, but by Dialyzer itself.</p> - </note> - - <note> <p>Warning option <c>-Wrace_conditions</c> has no effect when set in source files.</p> </note> diff --git a/lib/dialyzer/src/dialyzer_options.erl b/lib/dialyzer/src/dialyzer_options.erl index 616e8834f5..ec3f41311d 100644 --- a/lib/dialyzer/src/dialyzer_options.erl +++ b/lib/dialyzer/src/dialyzer_options.erl @@ -277,6 +277,9 @@ assert_solvers([Term|_]) -> -spec build_warnings([atom()], dial_warn_tags()) -> dial_warn_tags(). +%% The warning options are checked by the code linter. +%% The function erl_lint:is_module_dialyzer_option/1 must +%% be updated if options are added or removed. build_warnings([Opt|Opts], Warnings) -> NewWarnings = case Opt of diff --git a/lib/dialyzer/test/plt_SUITE.erl b/lib/dialyzer/test/plt_SUITE.erl index ba153c1c27..92c63bdb0c 100644 --- a/lib/dialyzer/test/plt_SUITE.erl +++ b/lib/dialyzer/test/plt_SUITE.erl @@ -259,26 +259,41 @@ remove_plt(Config) -> {init_plt, Plt}] ++ Opts), ok. +%% ERL-283, OTP-13979. As of OTP-14323 this test no longer does what +%% it is designed to do--the linter stops every attempt to run the +%% checks of Dialyzer's on bad dialyzer attributes. For the time +%% being, the linter's error message are checked instead. The test +%% needs to be updated when/if the Dialyzer can analyze Core Erlang +%% without compiling abstract code. bad_dialyzer_attr(Config) -> PrivDir = ?config(priv_dir, Config), - Plt = filename:join(PrivDir, "plt_bad_dialyzer_attr.plt"), + Source = lists:concat([dial, ".erl"]), + Filename = filename:join(PrivDir, Source), + ok = dialyzer_common:check_plt(PrivDir), + PltFilename = dialyzer_common:plt_file(PrivDir), + Opts = [{files, [Filename]}, + {check_plt, false}, + {from, src_code}, + {init_plt, PltFilename}], + Prog1 = <<"-module(dial). -dialyzer({no_return, [undef/0]}).">>, - {ok, Beam1} = compile(Config, Prog1, dial, []), + ok = file:write_file(Filename, Prog1), {dialyzer_error, - "Analysis failed with error:\n" - "Could not scan the following file(s):\n" - " Unknown function undef/0 in line " ++ _} = - (catch run_dialyzer(plt_build, [Beam1], [{output_plt, Plt}])), + "Analysis failed with error:\n" ++ Str1} = + (catch dialyzer:run(Opts)), + P1 = string:str(Str1, "dial.erl:2: function undef/0 undefined"), + true = P1 > 0, Prog2 = <<"-module(dial). -dialyzer({no_return, [{undef,1,2}]}).">>, - {ok, Beam2} = compile(Config, Prog2, dial, []), + ok = file:write_file(Filename, Prog2), {dialyzer_error, - "Analysis failed with error:\n" - "Could not scan the following file(s):\n" - " Bad function {undef,1,2} in line " ++ _} = - (catch run_dialyzer(plt_build, [Beam2], [{output_plt, Plt}])), + "Analysis failed with error:\n" ++ Str2} = + (catch dialyzer:run(Opts)), + P2 = string:str(Str2, "dial.erl:2: badly formed dialyzer " + "attribute: {no_return,{undef,1,2}}"), + true = P2 > 0, ok. diff --git a/lib/erl_interface/doc/src/ei.xml b/lib/erl_interface/doc/src/ei.xml index ddfb4d88a8..c3c776296c 100644 --- a/lib/erl_interface/doc/src/ei.xml +++ b/lib/erl_interface/doc/src/ei.xml @@ -421,22 +421,16 @@ typedef enum { <name><ret>int</ret><nametext>ei_x_encode_atom_len_as(ei_x_buff* x, const char *p, int len, erlang_char_encoding from_enc, erlang_char_encoding to_enc)</nametext></name> <fsummary>Encode an atom.</fsummary> <desc> - <p>Encodes an atom in the binary format with character encoding - <seealso marker="#erlang_char_encoding"><c>to_enc</c></seealso> - (Latin-1 or UTF-8). Parameter <c>p</c> is the name of the atom with + <p>Encodes an atom in the binary format. Parameter <c>p</c> is the name of the atom with character encoding <seealso marker="#erlang_char_encoding"><c>from_enc</c></seealso> (ASCII, Latin-1, or UTF-8). The name must either be <c>NULL</c>-terminated or - a function variant with a <c>len</c> parameter must be used. - If <c>to_enc</c> is set to the bitwise OR'd combination - <c>(ERLANG_LATIN1|ERLANG_UTF8)</c>, UTF-8 encoding is only used if the - atom string cannot be represented in Latin-1 encoding.</p> - <p>The encoding fails if <c>p</c> is an invalid string in encoding - <c>from_enc</c>, if the string is too long, or if it cannot be - represented with character encoding <c>to_enc</c>.</p> - <p>These functions were introduced in Erlang/OTP R16 as part of a first - step to support UTF-8 atoms. Atoms encoded with <c>ERLANG_UTF8</c> - cannot be decoded by earlier releases than R16.</p> + a function variant with a <c>len</c> parameter must be used.</p> + <p>The encoding fails if <c>p</c> is not a valid string in encoding + <c>from_enc</c>.</p> + + <p>Argument <c>to_enc</c> is ignored. As from Erlang/OTP 20 the encoding is always + done in UTF-8 which is readable by nodes as old as Erlang/OTP R16.</p> </desc> </func> diff --git a/lib/erl_interface/src/encode/encode_atom.c b/lib/erl_interface/src/encode/encode_atom.c index c1817628e5..1fd7811a0e 100644 --- a/lib/erl_interface/src/encode/encode_atom.c +++ b/lib/erl_interface/src/encode/encode_atom.c @@ -26,7 +26,6 @@ static int verify_ascii_atom(const char* src, int slen); static int verify_utf8_atom(const char* src, int slen); -static int is_latin1_as_utf8(const char *p, int len); int ei_encode_atom(char *buf, int *index, const char *p) { @@ -34,7 +33,7 @@ int ei_encode_atom(char *buf, int *index, const char *p) if (len >= MAXATOMLEN) len = MAXATOMLEN - 1; - return ei_encode_atom_len_as(buf, index, p, len, ERLANG_LATIN1, ERLANG_LATIN1); + return ei_encode_atom_len_as(buf, index, p, len, ERLANG_LATIN1, 0); } int ei_encode_atom_len(char *buf, int *index, const char *p, int len) @@ -42,7 +41,7 @@ int ei_encode_atom_len(char *buf, int *index, const char *p, int len) /* This function is documented to truncate at MAXATOMLEN (256) */ if (len >= MAXATOMLEN) len = MAXATOMLEN - 1; - return ei_encode_atom_len_as(buf, index, p, len, ERLANG_LATIN1, ERLANG_LATIN1); + return ei_encode_atom_len_as(buf, index, p, len, ERLANG_LATIN1, 0); } int ei_encode_atom_as(char *buf, int *index, const char *p, @@ -64,46 +63,11 @@ int ei_encode_atom_len_as(char *buf, int *index, const char *p, int len, return -1; } - if (to_enc == (ERLANG_LATIN1 | ERLANG_UTF8)) { - if (from_enc == ERLANG_UTF8) { - to_enc = is_latin1_as_utf8(p, len) ? ERLANG_LATIN1 : ERLANG_UTF8; - } - else { - to_enc = from_enc; - } - } - switch(to_enc) { - case ERLANG_LATIN1: - if (buf) { - put8(s,ERL_ATOM_EXT); - switch (from_enc) { - case ERLANG_UTF8: - len = utf8_to_latin1(s+2, p, len, MAXATOMLEN-1, NULL); - if (len < 0) return -1; - break; - case ERLANG_ASCII: - if (verify_ascii_atom(p, len) < 0) return -1; - memcpy(s+2, p, len); - break; - case ERLANG_LATIN1: - memcpy(s+2, p, len); - break; - default: - return -1; - } - put16be(s,len); - } - else { - s += 3; - if (from_enc == ERLANG_UTF8) { - len = utf8_to_latin1(NULL, p, len, MAXATOMLEN-1, NULL); - if (len < 0) return -1; - } else if (from_enc == ERLANG_ASCII) - if (verify_ascii_atom(p, len) < 0) return -1; - } - break; - - case ERLANG_UTF8: + /* + * Since OTP 20 we totally ignore 'to_enc' + * and alway encode as UTF8. + */ + { offs = 1 + 1; switch (from_enc) { case ERLANG_LATIN1: @@ -133,10 +97,6 @@ int ei_encode_atom_len_as(char *buf, int *index, const char *p, int len, } } else s+= offs; - break; - - default: - return -1; } s += len; @@ -197,13 +157,3 @@ static int verify_utf8_atom(const char* src, int slen) return 0; } -/* Only latin1 code points in utf8 string? - */ -static int is_latin1_as_utf8(const char *p, int len) -{ - int i; - for (i=0; i<len; i++) { - if ((unsigned char)p[i] > 0xC3) return 0; - } - return 1; -} diff --git a/lib/erl_interface/src/encode/encode_boolean.c b/lib/erl_interface/src/encode/encode_boolean.c index 61e7e5e6e7..053029af05 100644 --- a/lib/erl_interface/src/encode/encode_boolean.c +++ b/lib/erl_interface/src/encode/encode_boolean.c @@ -32,12 +32,12 @@ int ei_encode_boolean(char *buf, int *index, int p) val = p ? "true" : "false"; len = strlen(val); - if (!buf) s += 3; + if (!buf) s += 2; else { - put8(s,ERL_ATOM_EXT); - put16be(s,len); + put8(s, ERL_SMALL_ATOM_UTF8_EXT); + put8(s, len); - memmove(s,val,len); /* unterminated string */ + memcpy(s,val,len); /* unterminated string */ } s += len; diff --git a/lib/erl_interface/src/legacy/erl_eterm.c b/lib/erl_interface/src/legacy/erl_eterm.c index e4b3b49c7d..5153d0f2e7 100644 --- a/lib/erl_interface/src/legacy/erl_eterm.c +++ b/lib/erl_interface/src/legacy/erl_eterm.c @@ -188,14 +188,20 @@ char* erl_atom_ptr_latin1(Erl_Atom_data* a) char* erl_atom_ptr_utf8(Erl_Atom_data* a) { if (a->utf8 == NULL) { - int dlen = a->lenL * 2; /* over estimation */ - a->utf8 = malloc(dlen + 1); - a->lenU = latin1_to_utf8(a->utf8, a->latin1, a->lenL, dlen, NULL); - a->utf8[a->lenU] = '\0'; + erlang_char_encoding enc; + a->lenU = latin1_to_utf8(NULL, a->latin1, a->lenL, a->lenL*2, &enc); + if (enc == ERLANG_ASCII) { + a->utf8 = a->latin1; + } + else { + a->utf8 = malloc(a->lenU + 1); + latin1_to_utf8(a->utf8, a->latin1, a->lenL, a->lenU, NULL); + a->utf8[a->lenU] = '\0'; + } } return a->utf8; - } + int erl_atom_size_latin1(Erl_Atom_data* a) { if (a->latin1 == NULL) { diff --git a/lib/erl_interface/src/legacy/erl_marshal.c b/lib/erl_interface/src/legacy/erl_marshal.c index 527ae0ef8f..b7a8455313 100644 --- a/lib/erl_interface/src/legacy/erl_marshal.c +++ b/lib/erl_interface/src/legacy/erl_marshal.c @@ -175,10 +175,9 @@ static void encode_atom(Erl_Atom_data* a, unsigned char **ext) int ix = 0; if (a->latin1) { ei_encode_atom_len_as((char*)*ext, &ix, a->latin1, a->lenL, - ERLANG_LATIN1, ERLANG_LATIN1); + ERLANG_LATIN1, ERLANG_UTF8); } - else if (ei_encode_atom_len_as((char*)*ext, &ix, a->utf8, a->lenU, - ERLANG_UTF8, ERLANG_LATIN1) < 0) { + else { ei_encode_atom_len_as((char*)*ext, &ix, a->utf8, a->lenU, ERLANG_UTF8, ERLANG_UTF8); } @@ -542,12 +541,8 @@ int erl_term_len(ETERM *ep) static int atom_len_helper(Erl_Atom_data* a) { - if (erl_atom_ptr_latin1(a)) { - return 1 + 2 + a->lenL; /* ERL_ATOM_EXT */ - } - else { - return 1 + 1 + (a->lenU > 255) + a->lenU; - } + (void) erl_atom_ptr_utf8(a); + return 1 + 1 + (a->lenU > 255) + a->lenU; } static int erl_term_len_helper(ETERM *ep, int dist) diff --git a/lib/erl_interface/test/ei_decode_SUITE.erl b/lib/erl_interface/test/ei_decode_SUITE.erl index 10e90685c8..8612450692 100644 --- a/lib/erl_interface/test/ei_decode_SUITE.erl +++ b/lib/erl_interface/test/ei_decode_SUITE.erl @@ -179,7 +179,8 @@ test_ei_decode_misc(Config) when is_list(Config) -> send_term_as_binary(P,foo), send_term_as_binary(P,''), - send_term_as_binary(P,'ÅÄÖåäö'), + %%send_term_as_binary(P,'ÅÄÖåäö'), + send_latin1_atom_as_binary(P, "ÅÄÖåäö"), send_term_as_binary(P,"foo"), send_term_as_binary(P,""), @@ -200,18 +201,19 @@ test_ei_decode_misc(Config) when is_list(Config) -> test_ei_decode_utf8_atom(Config) -> P = runner:start(?test_ei_decode_utf8_atom), - send_utf8_atom_as_binary(P,"å"), - send_utf8_atom_as_binary(P,"ä"), - send_term_as_binary(P,'ö'), - send_term_as_binary(P,'õ'), + send_latin1_atom_as_binary(P,"å"), + send_latin1_atom_as_binary(P,"ä"), + send_latin1_atom_as_binary(P,"ö"), + send_latin1_atom_as_binary(P,"õ"), send_utf8_atom_as_binary(P,[1758]), send_utf8_atom_as_binary(P,[1758,1758]), send_utf8_atom_as_binary(P,[1758,1758,1758]), send_utf8_atom_as_binary(P,[1758,1758,1758,1758]), - send_utf8_atom_as_binary(P,"a"), - send_utf8_atom_as_binary(P,"b"), + send_latin1_atom_as_binary(P,"a"), + send_latin1_atom_as_binary(P,"b"), + send_term_as_binary(P,'c'), send_term_as_binary(P,'d'), @@ -230,6 +232,9 @@ send_raw(Port, Bin) when is_port(Port) -> send_utf8_atom_as_binary(Port, String) -> Port ! {self(), {command, term_to_binary(uc_atup(String))}}. +send_latin1_atom_as_binary(Port, String) -> + Port ! {self(), {command, encode_latin1_atom(String)}}. + send_integers(P) -> send_term_as_binary(P,0), % SMALL_INTEGER_EXT smallest send_term_as_binary(P,255), % SMALL_INTEGER_EXT largest @@ -304,6 +309,12 @@ send_integers2(P) -> send_term_as_binary(P, []), % illegal type ok. +encode_latin1_atom(String) -> + Len = length(String), + %% Use ATOM_EXT (not SMALL_*) to simulate old term_to_binary + TagLen = [$d, Len bsr 8, Len band 16#ff], + list_to_binary([131, TagLen, String]). + uc_atup(ATxt) -> string_to_atom(ATxt). diff --git a/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c b/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c index cfe9083065..649dc9a677 100644 --- a/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c +++ b/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c @@ -102,7 +102,7 @@ int ei_decode_my_string(const char *buf, int *index, char *to, } \ \ if (size1 != SIZE) { \ - fail("size of encoded data is incorrect"); \ + fail1("size of encoded data (%d) is incorrect", size1); \ return; \ } \ } \ @@ -614,11 +614,11 @@ TESTCASE(test_ei_decode_misc) EI_DECODE_2(decode_double, 9, double, -1.0); EI_DECODE_2(decode_double, 9, double, 1.0); - EI_DECODE_2(decode_boolean, 8, int, 0); - EI_DECODE_2(decode_boolean, 7, int, 1); + EI_DECODE_2(decode_boolean, 7, int, 0); + EI_DECODE_2(decode_boolean, 6, int, 1); - EI_DECODE_STRING(decode_my_atom, 6, "foo"); - EI_DECODE_STRING(decode_my_atom, 3, ""); + EI_DECODE_STRING(decode_my_atom, 5, "foo"); + EI_DECODE_STRING(decode_my_atom, 2, ""); EI_DECODE_STRING(decode_my_atom, 9, "������"); EI_DECODE_STRING(decode_my_string, 6, "foo"); @@ -665,10 +665,10 @@ TESTCASE(test_ei_decode_utf8_atom) P99({ERLANG_ANY,ERLANG_LATIN1,ERLANG_ASCII})); EI_DECODE_STRING_4(decode_my_atom_as, 4, "b", P99({ERLANG_UTF8,ERLANG_LATIN1,ERLANG_ASCII})); - EI_DECODE_STRING_4(decode_my_atom_as, 4, "c", - P99({ERLANG_LATIN1,ERLANG_LATIN1,ERLANG_ASCII})); - EI_DECODE_STRING_4(decode_my_atom_as, 4, "d", - P99({ERLANG_ASCII,ERLANG_LATIN1,ERLANG_ASCII})); + EI_DECODE_STRING_4(decode_my_atom_as, 3, "c", + P99({ERLANG_LATIN1,ERLANG_UTF8,ERLANG_ASCII})); + EI_DECODE_STRING_4(decode_my_atom_as, 3, "d", + P99({ERLANG_ASCII,ERLANG_UTF8,ERLANG_ASCII})); report(1); } diff --git a/lib/erl_interface/test/ei_decode_encode_SUITE.erl b/lib/erl_interface/test/ei_decode_encode_SUITE.erl index 570a91e2da..108a1f5142 100644 --- a/lib/erl_interface/test/ei_decode_encode_SUITE.erl +++ b/lib/erl_interface/test/ei_decode_encode_SUITE.erl @@ -170,7 +170,6 @@ get_binary(P) -> -define(VERSION_MAGIC, 131). --define(ATOM_EXT, 100). -define(REFERENCE_EXT, 101). -define(PORT_EXT, 102). -define(PID_EXT, 103). diff --git a/lib/erl_interface/test/ei_encode_SUITE.erl b/lib/erl_interface/test/ei_encode_SUITE.erl index ac6ec9cf4e..43484a1319 100644 --- a/lib/erl_interface/test/ei_encode_SUITE.erl +++ b/lib/erl_interface/test/ei_encode_SUITE.erl @@ -184,17 +184,17 @@ test_ei_encode_misc(Config) when is_list(Config) -> {<<70,_:8/binary>>,Fp1} = get_buf_and_term(P), true = match_float(Fp1, 1.0), - {<<100,0,5,"false">>,false} = get_buf_and_term(P), - {<<100,0,4,"true">> ,true} = get_buf_and_term(P), - {<<100,0,4,"true">> ,true} = get_buf_and_term(P), - {<<100,0,4,"true">> ,true} = get_buf_and_term(P), - - {<<100,0,3,"foo">>,foo} = get_buf_and_term(P), - {<<100,0,3,"foo">>,foo} = get_buf_and_term(P), - {<<100,0,0,"">>,''} = get_buf_and_term(P), - {<<100,0,0,"">>,''} = get_buf_and_term(P), - {<<100,0,6,"ÅÄÖåäö">>,'ÅÄÖåäö'} = get_buf_and_term(P), - {<<100,0,6,"ÅÄÖåäö">>,'ÅÄÖåäö'} = get_buf_and_term(P), + {<<$w,5,"false">>,false} = get_buf_and_term(P), + {<<$w,4,"true">> ,true} = get_buf_and_term(P), + {<<$w,4,"true">> ,true} = get_buf_and_term(P), + {<<$w,4,"true">> ,true} = get_buf_and_term(P), + + {<<$w,3,"foo">>,foo} = get_buf_and_term(P), + {<<$w,3,"foo">>,foo} = get_buf_and_term(P), + {<<$w,0,"">>,''} = get_buf_and_term(P), + {<<$w,0,"">>,''} = get_buf_and_term(P), + {<<$w,12,"ÅÄÖåäö"/utf8>>,'ÅÄÖåäö'} = get_buf_and_term(P), + {<<$w,12,"ÅÄÖåäö"/utf8>>,'ÅÄÖåäö'} = get_buf_and_term(P), {<<107,0,3,"foo">>,"foo"} = get_buf_and_term(P), {<<107,0,3,"foo">>,"foo"} = get_buf_and_term(P), @@ -239,12 +239,12 @@ test_ei_encode_utf8_atom(Config) -> P = runner:start(?test_ei_encode_utf8_atom), {<<119,2,195,133>>,'Å'} = get_buf_and_term(P), - {<<100,0,1,197>>,'Å'} = get_buf_and_term(P), - {<<100,0,1,197>>,'Å'} = get_buf_and_term(P), + {<<119,2,195,133>>,'Å'} = get_buf_and_term(P), + {<<119,2,195,133>>,'Å'} = get_buf_and_term(P), {<<119,2,195,133>>,'Å'} = get_buf_and_term(P), {<<119,1,$A>>,'A'} = get_buf_and_term(P), - {<<100,0,1,$A>>,'A'} = get_buf_and_term(P), + {<<119,1,$A>>,'A'} = get_buf_and_term(P), runner:recv_eot(P), ok. @@ -254,13 +254,13 @@ test_ei_encode_utf8_atom_len(Config) -> P = runner:start(?test_ei_encode_utf8_atom_len), {<<119,2,195,133>>,'Å'} = get_buf_and_term(P), - {<<100,0,2,197,196>>,'ÅÄ'} = get_buf_and_term(P), - {<<100,0,1,197>>,'Å'} = get_buf_and_term(P), + {<<119,4,195,133,195,132>>,'ÅÄ'} = get_buf_and_term(P), + {<<119,2,195,133>>,'Å'} = get_buf_and_term(P), {<<119,4,195,133,195,132>>,'ÅÄ'} = get_buf_and_term(P), {<<119,1,$A>>,'A'} = get_buf_and_term(P), - {<<100,0,2,$A,$B>>,'AB'} = get_buf_and_term(P), - {<<100,0,255,_:(255*8)>>,_} = get_buf_and_term(P), + {<<119,2,$A,$B>>,'AB'} = get_buf_and_term(P), + {<<119,255,_:(255*8)>>,_} = get_buf_and_term(P), runner:recv_eot(P), ok. diff --git a/lib/erl_interface/test/erl_ext_SUITE_data/ext_test.c b/lib/erl_interface/test/erl_ext_SUITE_data/ext_test.c index 36cf086ed2..afac5485e8 100644 --- a/lib/erl_interface/test/erl_ext_SUITE_data/ext_test.c +++ b/lib/erl_interface/test/erl_ext_SUITE_data/ext_test.c @@ -407,7 +407,7 @@ test_compare_ext(char *test_desc, } -#define ATOM_EXT (100) +#define SMALL_ATOM_UTF8_EXT (119) #define REFERENCE_EXT (101) #define PORT_EXT (102) #define PID_EXT (103) @@ -429,13 +429,13 @@ write_atom(unsigned char *buf, char *atom) len = 0; while(atom[len]) { - buf[len + 3] = atom[len]; + buf[len + 2] = atom[len]; len++; } - buf[0] = ATOM_EXT; - PUT_UINT16(&buf[1], len); + buf[0] = SMALL_ATOM_UTF8_EXT; + buf[1] = len; - return buf + 3 + len; + return buf + 2 + len; } static unsigned char * diff --git a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java index dca2eb7c51..e1718f8380 100644 --- a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java +++ b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java @@ -403,7 +403,6 @@ public class OtpOutputStream extends ByteArrayOutputStream { public void write_atom(final String atom) { String enc_atom; byte[] bytes; - boolean isLatin1 = true; if (atom.codePointCount(0, atom.length()) <= OtpExternal.maxAtomLength) { enc_atom = atom; @@ -416,29 +415,15 @@ public class OtpOutputStream extends ByteArrayOutputStream { OtpExternal.maxAtomLength); } - for (int offset = 0; offset < enc_atom.length();) { - final int cp = enc_atom.codePointAt(offset); - if ((cp & ~0xFF) != 0) { - isLatin1 = false; - break; - } - offset += Character.charCount(cp); - } try { - if (isLatin1) { - bytes = enc_atom.getBytes("ISO-8859-1"); - write1(OtpExternal.atomTag); - write2BE(bytes.length); + bytes = enc_atom.getBytes("UTF-8"); + final int length = bytes.length; + if (length < 256) { + write1(OtpExternal.smallAtomUtf8Tag); + write1(length); } else { - bytes = enc_atom.getBytes("UTF-8"); - final int length = bytes.length; - if (length < 256) { - write1(OtpExternal.smallAtomUtf8Tag); - write1(length); - } else { - write1(OtpExternal.atomUtf8Tag); - write2BE(length); - } + write1(OtpExternal.atomUtf8Tag); + write2BE(length); } writeN(bytes); } catch (final java.io.UnsupportedEncodingException e) { diff --git a/lib/jinterface/test/jinterface_SUITE_data/Maps.java b/lib/jinterface/test/jinterface_SUITE_data/Maps.java index e8a05245da..a1b6fa73c9 100644 --- a/lib/jinterface/test/jinterface_SUITE_data/Maps.java +++ b/lib/jinterface/test/jinterface_SUITE_data/Maps.java @@ -40,19 +40,19 @@ class Maps { public static void main(final String argv[]) { runTest(new byte[] { (byte) 131, 116, 0, 0, 0, 0 }, "#{}", 1); - runTest(new byte[] { (byte) 131, 116, 0, 0, 0, 1, 100, 0, 1, 97, 100, - 0, 1, 98 }, "#{a => b}", 2); + runTest(new byte[] { (byte) 131, 116, 0, 0, 0, 1, 119, 1, 97, 119, + 1, 98 }, "#{a => b}", 2); // make sure keys are sorted here, jinterface doesn't reorder them runTest(new byte[] { (byte) 131, 116, 0, 0, 0, 2, 97, 2, 106, - 100, 0, 1, 97, 97, 1 }, "#{2 => [],a => 1}", 3); + 119, 1, 97, 97, 1 }, "#{2 => [],a => 1}", 3); runTest(new byte[] { (byte) 131, 116, 0, 0, 0, 1, 104, 1, 97, 3, 108, - 0, 0, 0, 1, 100, 0, 1, 114, 106 }, "#{{3} => [r]}", 4); + 0, 0, 0, 1, 119, 1, 114, 106 }, "#{{3} => [r]}", 4); try { // #{2 => [],a => 1} final OtpErlangMap map = new OtpErlangMap(new OtpInputStream( new byte[] { (byte) 131, 116, 0, 0, 0, 2, 97, 2, 106, - 100, 0, 1, 97, 97, 1 })); + 119, 1, 97, 97, 1 })); if (map.arity() != 2) { fail(5); diff --git a/lib/kernel/src/dist_util.erl b/lib/kernel/src/dist_util.erl index 8d2fc4d4b7..d929179715 100644 --- a/lib/kernel/src/dist_util.erl +++ b/lib/kernel/src/dist_util.erl @@ -131,7 +131,7 @@ handshake_other_started(#hs_data{request_type=ReqType}=HSData0) -> other_version=Version, other_node=Node, other_started=true}, - check_dflag_xnc(HSData), + check_dflags(HSData), is_allowed(HSData), ?debug({"MD5 connection from ~p (V~p)~n", [Node, HSData#hs_data.other_version]}), @@ -168,27 +168,24 @@ is_allowed(#hs_data{other_node = Node, %% Check that both nodes can handle the same types of extended %% node containers. If they can not, abort the connection. %% -check_dflag_xnc(#hs_data{other_node = Node, - other_flags = OtherFlags, - other_started = OtherStarted} = HSData) -> - XRFlg = ?DFLAG_EXTENDED_REFERENCES, - XPPFlg = case erlang:system_info(compat_rel) of - R when R >= 10 -> - ?DFLAG_EXTENDED_PIDS_PORTS; - _ -> - 0 - end, - ReqXncFlags = XRFlg bor XPPFlg, - case OtherFlags band ReqXncFlags =:= ReqXncFlags of - true -> - ok; - false -> - What = case {OtherFlags band XRFlg =:= XRFlg, - OtherFlags band XPPFlg =:= XPPFlg} of - {false, false} -> "references, pids and ports"; - {true, false} -> "pids and ports"; - {false, true} -> "references" - end, +check_dflags(#hs_data{other_node = Node, + other_flags = OtherFlags, + other_started = OtherStarted} = HSData) -> + + Mandatory = [{?DFLAG_EXTENDED_REFERENCES, "EXTENDED_REFERENCES"}, + {?DFLAG_EXTENDED_PIDS_PORTS, "EXTENDED_PIDS_PORTS"}, + {?DFLAG_UTF8_ATOMS, "UTF8_ATOMS"}], + Missing = lists:filtermap(fun({Bit, Str}) -> + case Bit band OtherFlags of + Bit -> false; + 0 -> {true, Str} + end + end, + Mandatory), + case Missing of + [] -> + ok; + _ -> case OtherStarted of true -> send_status(HSData, not_allowed), @@ -199,9 +196,9 @@ check_dflag_xnc(#hs_data{other_node = Node, How = "aborted" end, error_msg("** ~w: Connection attempt ~s node ~w ~s " - "since it cannot handle extended ~s. " - "**~n", [node(), Dir, Node, How, What]), - ?shutdown2(Node, {check_dflag_xnc_failed, What}) + "since it cannot handle ~p." + "**~n", [node(), Dir, Node, How, Missing]), + ?shutdown2(Node, {check_dflags_failed, Missing}) end. @@ -327,7 +324,7 @@ handshake_we_started(#hs_data{request_type=ReqType, NewHSData = HSData#hs_data{this_flags = ThisFlags, other_flags = OtherFlags, other_started = false}, - check_dflag_xnc(NewHSData), + check_dflags(NewHSData), MyChallenge = gen_challenge(), {MyCookie,HisCookie} = get_cookies(Node), send_challenge_reply(NewHSData,MyChallenge, diff --git a/lib/kernel/test/disk_log_SUITE.erl b/lib/kernel/test/disk_log_SUITE.erl index 23fe975ef7..079cc2f90f 100644 --- a/lib/kernel/test/disk_log_SUITE.erl +++ b/lib/kernel/test/disk_log_SUITE.erl @@ -481,7 +481,7 @@ halt_ro_crash(Conf) when is_list(Conf) -> %% This is how it was before R6B: %% {C1,T1,15} = disk_log:chunk(a,start), %% {C2,T2} = disk_log:chunk(a,C1), - {C1,_OneItem,7478} = disk_log:chunk(a,start), + {C1,_OneItem,7476} = disk_log:chunk(a,start), {C2, [], 7} = disk_log:chunk(a,C1), eof = disk_log:chunk(a,C2), ok = disk_log:close(a), @@ -2502,8 +2502,8 @@ error_repair(Conf) when is_list(Conf) -> ok = disk_log:close(n), BadFile = add_ext(File, 2), % current file set_opened(BadFile), - crash(BadFile, 28), % the binary is now invalid - {repaired,n,{recovered,0},{badbytes,26}} = + crash(BadFile, 26), % the binary is now invalid + {repaired,n,{recovered,0},{badbytes,24}} = disk_log:open([{name, n}, {file, File}, {type, wrap}, {format, internal}, {size, {40,No}}]), ok = disk_log:close(n), @@ -2518,8 +2518,8 @@ error_repair(Conf) when is_list(Conf) -> ok = disk_log:close(n), BadFile2 = add_ext(File, 1), % current file set_opened(BadFile2), - crash(BadFile2, 51), % the second binary is now invalid - {repaired,n,{recovered,1},{badbytes,26}} = + crash(BadFile2, 47), % the second binary is now invalid + {repaired,n,{recovered,1},{badbytes,24}} = disk_log:open([{name, n}, {file, File}, {type, wrap}, {format, internal}, {size, {4000,No}}]), ok = disk_log:close(n), @@ -2571,7 +2571,7 @@ error_repair(Conf) when is_list(Conf) -> ok = disk_log:close(n), set_opened(File), crash(File, 30), - {repaired,n,{recovered,3},{badbytes,16}} = + {repaired,n,{recovered,3},{badbytes,15}} = disk_log:open([{name, n}, {file, File}, {type, halt}, {format, internal},{repair,true}, {head_func, {?MODULE, head_fun, [{ok,"head"}]}}]), @@ -2797,7 +2797,7 @@ chunk(Conf) when is_list(Conf) -> ok = disk_log:log_terms(n, [{some,terms}]), % second file full 2 = curf(n), BadFile = add_ext(File, 1), - crash(BadFile, 28), % the _binary_ is now invalid + crash(BadFile, 26), % the _binary_ is now invalid {error, {corrupt_log_file, BFile}} = disk_log:chunk(n, start, 1), BadFile = BFile, ok = disk_log:close(n), @@ -2807,7 +2807,7 @@ chunk(Conf) when is_list(Conf) -> {format, internal}]), ok = disk_log:log_terms(n, [{this,is}]), ok = disk_log:sync(n), - crash(File, 28), % the _binary_ is now invalid + crash(File, 26), % the _binary_ is now invalid {error, {corrupt_log_file, File2}} = disk_log:chunk(n, start, 1), crash(File, 10), {error,{corrupt_log_file,_}} = disk_log:bchunk(n, start, 1), @@ -2901,8 +2901,8 @@ chunk(Conf) when is_list(Conf) -> {ok, n} = disk_log:open([{name, n}, {file, File}, {type, wrap}, {format, internal}, {mode, read_only}]), CrashFile = add_ext(File, 1), - crash(CrashFile, 51), % the binary term {some,terms} is now bad - {H1, [{this,is}], 18} = disk_log:chunk(n, start, 10), + crash(CrashFile, 46), % the binary term {some,terms} is now bad + {H1, [{this,is}], 16} = disk_log:chunk(n, start, 10), {H2, [{on,a},{wrap,file}]} = disk_log:chunk(n, H1), eof = disk_log:chunk(n, H2), ok = disk_log:close(n), @@ -2916,8 +2916,8 @@ chunk(Conf) when is_list(Conf) -> ok = disk_log:close(n), {ok, n} = disk_log:open([{name, n}, {file, File}, {type, halt}, {format, internal}, {mode, read_only}]), - crash(File, 51), % the binary term {some,terms} is now bad - {J1, [{this,is}], 18} = disk_log:chunk(n, start, 10), + crash(File, 46), % the binary term {some,terms} is now bad + {J1, [{this,is}], 16} = disk_log:chunk(n, start, 10), {J2, [{on,a},{halt,file}]} = disk_log:chunk(n, J1), eof = disk_log:chunk(n, J2), ok = disk_log:close(n), @@ -2932,8 +2932,8 @@ chunk(Conf) when is_list(Conf) -> ok = disk_log:close(n), {ok, n} = disk_log:open([{name, n}, {file, File}, {type, halt}, {format, internal}, {mode, read_only}]), - crash(File, 44), % the binary term {s} is now bad - {J11, [{this,is}], 7} = disk_log:chunk(n, start, 10), + crash(File, 40), % the binary term {s} is now bad + {J11, [{this,is}], 6} = disk_log:chunk(n, start, 10), {J21, [{on,a},{halt,file}]} = disk_log:chunk(n, J11), eof = disk_log:chunk(n, J21), ok = disk_log:close(n), @@ -3052,7 +3052,7 @@ truncate(Conf) when is_list(Conf) -> ok = disk_log:truncate(n, apa), rec(1, {disk_log, node(), n, {truncated, 6}}), {0, 0} = no_overflows(n), - 23 = curb(n), + 22 = curb(n), 1 = curf(n), 1 = cur_cnt(n), true = (Size == sz(n)), @@ -3072,7 +3072,7 @@ truncate(Conf) when is_list(Conf) -> ok = disk_log:truncate(n, apa), rec(1, {disk_log, node(), n, {truncated, 3}}), {0, 0} = no_overflows(n), - 23 = curb(n), + 22 = curb(n), 1 = curf(n), 1 = cur_cnt(n), true = (Size == sz(n)), @@ -3181,45 +3181,45 @@ info_current(Conf) when is_list(Conf) -> %% Internal with header. {ok, n} = disk_log:open([{name, n}, {file, File}, {type, wrap}, {head, header}, {size, {100,No}}]), - {26, 1} = {curb(n), cur_cnt(n)}, + {25, 1} = {curb(n), cur_cnt(n)}, {1, 1} = {no_written_items(n), no_items(n)}, ok = disk_log:log(n, B), - {94, 2} = {curb(n), cur_cnt(n)}, + {93, 2} = {curb(n), cur_cnt(n)}, {2, 2} = {no_written_items(n), no_items(n)}, ok = disk_log:close(n), {ok, n} = disk_log:open([{name, n}, {file, File}, {type, wrap}, {notify, true}, {head, header}, {size, {100,No}}]), - {94, 2} = {curb(n), cur_cnt(n)}, + {93, 2} = {curb(n), cur_cnt(n)}, {0, 2} = {no_written_items(n), no_items(n)}, ok = disk_log:log(n, B), rec(1, {disk_log, node(), n, {wrap, 0}}), - {94, 2} = {curb(n), cur_cnt(n)}, + {93, 2} = {curb(n), cur_cnt(n)}, {2, 4} = {no_written_items(n), no_items(n)}, disk_log:inc_wrap_file(n), rec(1, {disk_log, node(), n, {wrap, 0}}), - {26, 1} = {curb(n), cur_cnt(n)}, + {25, 1} = {curb(n), cur_cnt(n)}, {3, 4} = {no_written_items(n), no_items(n)}, ok = disk_log:log_terms(n, [B,B,B]), %% Used to be one message, but now one per wrapped file. rec(1, {disk_log, node(), n, {wrap, 0}}), rec(1, {disk_log, node(), n, {wrap, 2}}), - {94, 2} = {curb(n), cur_cnt(n)}, + {93, 2} = {curb(n), cur_cnt(n)}, {8, 7} = {no_written_items(n), no_items(n)}, ok = disk_log:log_terms(n, [B]), rec(1, {disk_log, node(), n, {wrap, 2}}), ok = disk_log:log_terms(n, [B]), rec(1, {disk_log, node(), n, {wrap, 2}}), - {94, 2} = {curb(n), cur_cnt(n)}, + {93, 2} = {curb(n), cur_cnt(n)}, {12, 7} = {no_written_items(n), no_items(n)}, ok = disk_log:log_terms(n, [BB,BB]), %% Used to be one message, but now one per wrapped file. rec(2, {disk_log, node(), n, {wrap, 2}}), - {194, 2} = {curb(n), cur_cnt(n)}, + {193, 2} = {curb(n), cur_cnt(n)}, {16, 7} = {no_written_items(n), no_items(n)}, ok = disk_log:log_terms(n, [SB,SB,SB]), rec(1, {disk_log, node(), n, {wrap, 2}}), - {80, 4} = {curb(n), cur_cnt(n)}, + {79, 4} = {curb(n), cur_cnt(n)}, {20, 9} = {no_written_items(n), no_items(n)}, ok = disk_log:close(n), del(File, No), diff --git a/lib/kernel/test/erl_distribution_wb_SUITE.erl b/lib/kernel/test/erl_distribution_wb_SUITE.erl index 61aa3b32ee..c1dc208cc1 100644 --- a/lib/kernel/test/erl_distribution_wb_SUITE.erl +++ b/lib/kernel/test/erl_distribution_wb_SUITE.erl @@ -56,10 +56,14 @@ -define(DFLAG_HIDDEN_ATOM_CACHE,16#40). -define(DFLAG_NEW_FUN_TAGS,16#80). -define(DFLAG_EXTENDED_PIDS_PORTS,16#100). +-define(DFLAG_UTF8_ATOMS, 16#10000). %% From R9 and forward extended references is compulsory %% From R10 and forward extended pids and ports are compulsory --define(COMPULSORY_DFLAGS, (?DFLAG_EXTENDED_REFERENCES bor ?DFLAG_EXTENDED_PIDS_PORTS)). +%% From R20 and forward UTF8 atoms are compulsory +-define(COMPULSORY_DFLAGS, (?DFLAG_EXTENDED_REFERENCES bor + ?DFLAG_EXTENDED_PIDS_PORTS bor + ?DFLAG_UTF8_ATOMS)). -define(shutdown(X), exit(X)). diff --git a/lib/kernel/test/pg2_SUITE.erl b/lib/kernel/test/pg2_SUITE.erl index fdc268cb5a..9460608a3e 100644 --- a/lib/kernel/test/pg2_SUITE.erl +++ b/lib/kernel/test/pg2_SUITE.erl @@ -31,7 +31,7 @@ -export([ otp_7277/1, otp_8259/1, otp_8653/1, - compat/1, basic/1]). + basic/1]). -define(TESTCASE, testcase_name). -define(testcase, proplists:get_value(?TESTCASE, Config)). @@ -56,7 +56,7 @@ all() -> groups() -> [{tickets, [], - [otp_7277, otp_8259, otp_8653, compat, basic]}]. + [otp_7277, otp_8259, otp_8653, basic]}]. init_per_suite(Config) -> Config. @@ -218,29 +218,6 @@ loop() -> exit(normal) end. -%% OTP-8259. Check that 'exchange' and 'del_member' work. -compat(Config) when is_list(Config) -> - case test_server:is_release_available("r13b") of - true -> - Pid = spawn(forever()), - G = a, - ok = pg2:create(G), - ok = pg2:join(G, Pid), - ok = pg2:join(G, Pid), - {ok, A} = start_node_rel(r13, r13b, slave), - pong = net_adm:ping(A), - wait_for_ready_net(Config), - {ok, _} = rpc:call(A, pg2, start, []), - ?UNTIL([Pid,Pid] =:= rpc:call(A, pg2, get_members, [a])), - true = exit(Pid, kill), - ?UNTIL([] =:= pg2:get_members(a)), - ?UNTIL([] =:= rpc:call(A, pg2, get_members, [a])), - test_server:stop_node(A), - ok; - false -> - {skipped, "No support for old node"} - end. - %% OTP-8259. Some basic tests. basic(Config) when is_list(Config) -> _ = [pg2:delete(G) || G <- pg2:which_groups()], diff --git a/lib/kernel/test/prim_file_SUITE.erl b/lib/kernel/test/prim_file_SUITE.erl index 8be94f1e57..0c84a7d1cc 100644 --- a/lib/kernel/test/prim_file_SUITE.erl +++ b/lib/kernel/test/prim_file_SUITE.erl @@ -19,8 +19,8 @@ %% -module(prim_file_SUITE). -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, - init_per_group/2,end_per_group/2, - read_write_file/1]). + init_per_group/2,end_per_group/2, init_per_testcase/2, end_per_testcase/2, + read_write_file/1, free_memory/0]). -export([cur_dir_0a/1, cur_dir_0b/1, cur_dir_1a/1, cur_dir_1b/1, make_del_dir_a/1, make_del_dir_b/1, @@ -115,6 +115,18 @@ groups() -> [make_link_a, make_link_b, read_link_info_for_non_link, symlinks_a, symlinks_b, list_dir_error]}]. +init_per_testcase(large_write, Config) -> + {ok, Started} = application:ensure_all_started(os_mon), + [{started, Started}|Config]; +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(large_write, Config) -> + [application:stop(App) || App <- lists:reverse(proplists:get_value(started, Config))], + ok; +end_per_testcase(_, _Config) -> + ok. + init_per_group(_GroupName, Config) -> Config. @@ -2022,11 +2034,13 @@ run_large_file_test(Config, Run, Name) -> {{unix,sunos},OsVersion} when OsVersion < {5,5,1} -> {skip,"Only supported on Win32, Unix or SunOS >= 5.5.1"}; {{unix,_},_} -> - N = unix_free(proplists:get_value(priv_dir, Config)), - io:format("Free disk: ~w KByte~n", [N]), - if N < 5 bsl 20 -> + DiscFree = unix_free(proplists:get_value(priv_dir, Config)), + MemFree = free_memory(), + io:format("Free disk: ~w KByte~n", [DiscFree]), + io:format("Free mem: ~w MByte~n", [MemFree]), + if DiscFree < 5 bsl 20; MemFree < 5 bsl 10 -> %% Less than 5 GByte free - {skip,"Less than 5 GByte free disk"}; + {skip,"Less than 5 GByte free disk/mem"}; true -> do_run_large_file_test(Config, Run, Name) end; @@ -2079,6 +2093,27 @@ zip_data([], Bs) -> zip_data(As, []) -> As. +%% Stolen from emulator -> alloc_SUITE +free_memory() -> + %% Free memory in MB. + try + SMD = memsup:get_system_memory_data(), + {value, {free_memory, Free}} = lists:keysearch(free_memory, 1, SMD), + TotFree = (Free + + case lists:keysearch(cached_memory, 1, SMD) of + {value, {cached_memory, Cached}} -> Cached; + false -> 0 + end + + case lists:keysearch(buffered_memory, 1, SMD) of + {value, {buffered_memory, Buffed}} -> Buffed; + false -> 0 + end), + TotFree div (1024*1024) + catch + error : undef -> + ct:fail({"os_mon not built"}) + end. + %%%----------------------------------------------------------------- %%% Utilities rm_rf(Mod,Dir) -> diff --git a/lib/orber/COSS/CosNaming/CosNaming_NamingContextExt_impl.erl b/lib/orber/COSS/CosNaming/CosNaming_NamingContextExt_impl.erl index 8f7da2425b..620c91d406 100644 --- a/lib/orber/COSS/CosNaming/CosNaming_NamingContextExt_impl.erl +++ b/lib/orber/COSS/CosNaming/CosNaming_NamingContextExt_impl.erl @@ -610,11 +610,16 @@ convert_list([{N, T, _O}|Rest], HowMany, Counter, Acc) -> %% Returns : %%---------------------------------------------------------------------- destroy(OE_THIS, OE_State) -> - case corba:get_subobject_key(OE_THIS) of - <<131,100,0,9,117,110,100,101,102,105,110,101,100>> -> - %% undefined binary. - corba:raise(#'NO_PERMISSION'{completion_status=?COMPLETED_NO}); - SubobjKey -> + SubobjKey = corba:get_subobject_key(OE_THIS), + try begin + true = (byte_size(SubobjKey) < 20), + undefined = binary_to_term(SubobjKey) + end + of + _ -> + corba:raise(#'NO_PERMISSION'{completion_status=?COMPLETED_NO}) + catch + error:_ -> %% Not atom 'undefined', carry on... _DF = fun() -> case mnesia:wread({orber_CosNaming, SubobjKey}) of diff --git a/lib/sasl/test/systools_SUITE.erl b/lib/sasl/test/systools_SUITE.erl index 0c98232467..1a8bd3f607 100644 --- a/lib/sasl/test/systools_SUITE.erl +++ b/lib/sasl/test/systools_SUITE.erl @@ -87,25 +87,25 @@ end_per_group(_GroupName, Config) -> init_per_suite(Config) when is_list(Config) -> + %% To use in end_per_testcase + Path = code:get_path(), + {ok,Cwd} = file:get_cwd(), + %% Make of copy of the data directory. DataDir = ?datadir, PrivDir = ?privdir, CopyDir = fname(PrivDir, "datacopy"), + ok = file:make_dir(CopyDir), TarFile = fname(PrivDir, "datacopy.tgz"), - {ok, Tar} = erl_tar:open(TarFile, [write, compressed]), - ok = erl_tar:add(Tar, DataDir, CopyDir, [compressed]), - ok = erl_tar:close(Tar), - ok = erl_tar:extract(TarFile, [compressed]), + ok = file:set_cwd(DataDir), + ok = erl_tar:create(TarFile, ["."], [compressed]), + ok = erl_tar:extract(TarFile, [compressed, {cwd,CopyDir}]), ok = file:delete(TarFile), %% Compile source files in the copy directory. Sources = filelib:wildcard(fname([CopyDir,'*','*','*','*','*.erl'])), lists:foreach(fun compile_source/1, Sources), - %% To use in end_per_testcase - Path = code:get_path(), - {ok,Cwd} = file:get_cwd(), - [{copy_dir, CopyDir}, {cwd,Cwd}, {path,Path} | Config]. compile_source(File) -> diff --git a/lib/ssh/src/ssh_connection_handler.erl b/lib/ssh/src/ssh_connection_handler.erl index 84adf952e6..84bb7dc23f 100644 --- a/lib/ssh/src/ssh_connection_handler.erl +++ b/lib/ssh/src/ssh_connection_handler.erl @@ -60,7 +60,7 @@ ]). %%% Behaviour callbacks --export([callback_mode/0, handle_event/4, terminate/3, +-export([init/1, callback_mode/0, handle_event/4, terminate/3, format_status/2, code_change/4]). %%% Exports not intended to be used :). They are used for spawning and tests @@ -337,8 +337,7 @@ renegotiate_data(ConnectionHandler) -> transport_protocol :: atom(), % ex: tcp transport_cb :: atom(), % ex: gen_tcp transport_close_tag :: atom(), % ex: tcp_closed - ssh_params :: #ssh{} - | undefined, + ssh_params :: #ssh{}, socket :: inet:socket(), decrypted_data_buffer = <<>> :: binary(), encrypted_data_buffer = <<>> :: binary(), @@ -362,71 +361,78 @@ renegotiate_data(ConnectionHandler) -> ) -> no_return(). %% . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . init_connection_handler(Role, Socket, Opts) -> - process_flag(trap_exit, true), - S0 = init_process_state(Role, Socket, Opts), - try - {Protocol, Callback, CloseTag} = ?GET_OPT(transport, Opts), - S0#data{ssh_params = init_ssh_record(Role, Socket, Opts), - transport_protocol = Protocol, - transport_cb = Callback, - transport_close_tag = CloseTag - } - of - S -> - gen_statem:enter_loop(?MODULE, - [], %%[{debug,[trace,log,statistics,debug]} || Role==server], - {hello,Role}, - S) - catch - _:Error -> - gen_statem:enter_loop(?MODULE, - [], - {init_error,Error}, - S0) - end. - - -init_process_state(Role, Socket, Opts) -> - D = #data{connection_state = - C = #connection{channel_cache = ssh_channel:cache_create(), - channel_id_seed = 0, - port_bindings = [], - requests = [], - options = Opts}, - starter = ?GET_INTERNAL_OPT(user_pid, Opts), - socket = Socket, - opts = Opts - }, - case Role of - client -> - %% Start the renegotiation timers - timer:apply_after(?REKEY_TIMOUT, gen_statem, cast, [self(), renegotiate]), - timer:apply_after(?REKEY_DATA_TIMOUT, gen_statem, cast, [self(), data_size]), - cache_init_idle_timer(D); - server -> - cache_init_idle_timer( - D#data{connection_state = init_connection(Role, C, Opts)} - ) + case init([Role, Socket, Opts]) of + {ok, StartState, D} -> + process_flag(trap_exit, true), + gen_statem:enter_loop(?MODULE, + [], %%[{debug,[trace,log,statistics,debug]} || Role==server], + StartState, + D); + + {stop, enotconn} -> + %% Handles the abnormal sequence: + %% SYN-> + %% <-SYNACK + %% ACK-> + %% RST-> + exit({shutdown, "TCP connection to server was prematurely closed by the client"}); + + {stop, OtherError} -> + exit({shutdown, {init,OtherError}}) end. -init_connection(server, C = #connection{}, Opts) -> - Sups = ?GET_INTERNAL_OPT(supervisors, Opts), - SystemSup = proplists:get_value(system_sup, Sups), - SubSystemSup = proplists:get_value(subsystem_sup, Sups), - ConnectionSup = proplists:get_value(connection_sup, Sups), +init([Role,Socket,Opts]) -> + case inet:peername(Socket) of + {ok, PeerAddr} -> + {Protocol, Callback, CloseTag} = ?GET_OPT(transport, Opts), + C = #connection{channel_cache = ssh_channel:cache_create(), + channel_id_seed = 0, + port_bindings = [], + requests = [], + options = Opts}, + D0 = #data{starter = ?GET_INTERNAL_OPT(user_pid, Opts), + connection_state = C, + socket = Socket, + transport_protocol = Protocol, + transport_cb = Callback, + transport_close_tag = CloseTag, + ssh_params = init_ssh_record(Role, Socket, PeerAddr, Opts), + opts = Opts + }, + D = case Role of + client -> + %% Start the renegotiation timers + timer:apply_after(?REKEY_TIMOUT, gen_statem, cast, [self(), renegotiate]), + timer:apply_after(?REKEY_DATA_TIMOUT, gen_statem, cast, [self(), data_size]), + cache_init_idle_timer(D0); + server -> + Sups = ?GET_INTERNAL_OPT(supervisors, Opts), + cache_init_idle_timer( + D0#data{connection_state = + C#connection{cli_spec = ?GET_OPT(ssh_cli, Opts, {ssh_cli,[?GET_OPT(shell, Opts)]}), + exec = ?GET_OPT(exec, Opts), + system_supervisor = proplists:get_value(system_sup, Sups), + sub_system_supervisor = proplists:get_value(subsystem_sup, Sups), + connection_supervisor = proplists:get_value(connection_sup, Sups) + }}) + end, + {ok, {hello,Role}, D}; + + {error,Error} -> + {stop, Error} + end. - C#connection{cli_spec = ?GET_OPT(ssh_cli, Opts, {ssh_cli,[?GET_OPT(shell, Opts)]}), - exec = ?GET_OPT(exec, Opts), - system_supervisor = SystemSup, - sub_system_supervisor = SubSystemSup, - connection_supervisor = ConnectionSup - }. init_ssh_record(Role, Socket, Opts) -> - {ok, PeerAddr} = inet:peername(Socket), + %% Export of this internal function is + %% intended for low-level protocol test suites + {ok,PeerAddr} = inet:peername(Socket), + init_ssh_record(Role, Socket, PeerAddr, Opts). + +init_ssh_record(Role, _Socket, PeerAddr, Opts) -> KeyCb = ?GET_OPT(key_cb, Opts), AuthMethods = case Role of @@ -481,8 +487,7 @@ init_ssh_record(Role, Socket, Opts) -> -type renegotiate_flag() :: init | renegotiate. -type state_name() :: - {init_error,any()} - | {hello, role()} + {hello, role()} | {kexinit, role(), renegotiate_flag()} | {key_exchange, role(), renegotiate_flag()} | {key_exchange_dh_gex_init, server, renegotiate_flag()} @@ -504,26 +509,9 @@ init_ssh_record(Role, Socket, Opts) -> %% . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . -%%% ######## Error in the initialisation #### - callback_mode() -> handle_event_function. -handle_event(_, _Event, {init_error,Error}, _) -> - case Error of - {badmatch,{error,enotconn}} -> - %% Handles the abnormal sequence: - %% SYN-> - %% <-SYNACK - %% ACK-> - %% RST-> - {stop, {shutdown,"TCP connenction to server was prematurely closed by the client"}}; - - OtherError -> - {stop, {shutdown,{init,OtherError}}} - end; - - %%% ######## {hello, client|server} #### %% The very first event that is sent when the we are set as controlling process of Socket handle_event(_, socket_control, {hello,_}, D) -> @@ -813,7 +801,7 @@ handle_event(_, #ssh_msg_userauth_info_request{} = Msg, {userauth_keyboard_inter send_bytes(Reply, D), {next_state, {userauth_keyboard_interactive_info_response,client}, D#data{ssh_params = Ssh}}; not_ok -> - {next_state, {userauth,client}, D, [{next_event, internal, Msg}]} + {next_state, {userauth,client}, D, [postpone]} end; handle_event(_, #ssh_msg_userauth_info_response{} = Msg, {userauth_keyboard_interactive, server}, D) -> @@ -842,14 +830,14 @@ handle_event(_, #ssh_msg_userauth_info_response{} = Msg, {userauth_keyboard_inte {next_state, {connected,server}, D#data{auth_user = User, ssh_params = Ssh#ssh{authenticated = true}}}; -handle_event(_, Msg = #ssh_msg_userauth_failure{}, {userauth_keyboard_interactive, client}, +handle_event(_, #ssh_msg_userauth_failure{}, {userauth_keyboard_interactive, client}, #data{ssh_params = Ssh0} = D0) -> Prefs = [{Method,M,F,A} || {Method,M,F,A} <- Ssh0#ssh.userauth_preference, Method =/= "keyboard-interactive"], D = D0#data{ssh_params = Ssh0#ssh{userauth_preference=Prefs}}, - {next_state, {userauth,client}, D, [{next_event, internal, Msg}]}; + {next_state, {userauth,client}, D, [postpone]}; -handle_event(_, Msg=#ssh_msg_userauth_failure{}, {userauth_keyboard_interactive_info_response, client}, +handle_event(_, #ssh_msg_userauth_failure{}, {userauth_keyboard_interactive_info_response, client}, #data{ssh_params = Ssh0} = D0) -> Opts = Ssh0#ssh.opts, D = case ?GET_OPT(password, Opts) of @@ -859,23 +847,23 @@ handle_event(_, Msg=#ssh_msg_userauth_failure{}, {userauth_keyboard_interactive_ D0#data{ssh_params = Ssh0#ssh{opts = ?PUT_OPT({password,not_ok}, Opts)}} % FIXME:intermodule dependency end, - {next_state, {userauth,client}, D, [{next_event, internal, Msg}]}; + {next_state, {userauth,client}, D, [postpone]}; -handle_event(_, Msg=#ssh_msg_userauth_success{}, {userauth_keyboard_interactive_info_response, client}, D) -> - {next_state, {userauth,client}, D, [{next_event, internal, Msg}]}; +handle_event(_, #ssh_msg_userauth_success{}, {userauth_keyboard_interactive_info_response, client}, D) -> + {next_state, {userauth,client}, D, [postpone]}; -handle_event(_, Msg=#ssh_msg_userauth_info_request{}, {userauth_keyboard_interactive_info_response, client}, D) -> - {next_state, {userauth_keyboard_interactive,client}, D, [{next_event, internal, Msg}]}; +handle_event(_, #ssh_msg_userauth_info_request{}, {userauth_keyboard_interactive_info_response, client}, D) -> + {next_state, {userauth_keyboard_interactive,client}, D, [postpone]}; %%% ######## {connected, client|server} #### -handle_event(_, {#ssh_msg_kexinit{},_} = Event, {connected,Role}, D0) -> +handle_event(_, {#ssh_msg_kexinit{},_}, {connected,Role}, D0) -> {KeyInitMsg, SshPacket, Ssh} = ssh_transport:key_exchange_init_msg(D0#data.ssh_params), D = D0#data{ssh_params = Ssh, key_exchange_init_msg = KeyInitMsg}, send_bytes(SshPacket, D), - {next_state, {kexinit,Role,renegotiate}, D, [{next_event, internal, Event}]}; + {next_state, {kexinit,Role,renegotiate}, D, [postpone]}; handle_event(_, #ssh_msg_disconnect{description=Desc} = Msg, StateName, D0) -> {disconnect, _, {{replies,Replies}, _}} = diff --git a/lib/ssh/src/ssh_transport.erl b/lib/ssh/src/ssh_transport.erl index 54ea80c727..6b47868d5c 100644 --- a/lib/ssh/src/ssh_transport.erl +++ b/lib/ssh/src/ssh_transport.erl @@ -181,7 +181,7 @@ ssh_vsn() -> end. random_id(Nlo, Nup) -> - [crypto:rand_uniform($a,$z+1) || _<- lists:duplicate(crypto:rand_uniform(Nlo,Nup+1),x) ]. + [$a + rand:uniform($z-$a+1) - 1 || _<- lists:duplicate(Nlo + rand:uniform(Nup-Nlo+1) - 1, x)]. hello_version_msg(Data) -> [Data,"\r\n"]. @@ -1041,7 +1041,7 @@ padding_length(Size, #ssh{encrypt_block_size = BlockSize, end, PadBlockSize = max(BlockSize,4), MaxExtraBlocks = (max(RandomLengthPadding,MinPaddingLen) - MinPaddingLen) div PadBlockSize, - ExtraPaddingLen = try crypto:rand_uniform(0,MaxExtraBlocks)*PadBlockSize + ExtraPaddingLen = try (rand:uniform(MaxExtraBlocks+1) - 1) * PadBlockSize catch _:_ -> 0 end, MinPaddingLen + ExtraPaddingLen. diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl index 16e0df2101..440607e99d 100644 --- a/lib/ssl/src/dtls_connection.erl +++ b/lib/ssl/src/dtls_connection.erl @@ -464,28 +464,42 @@ handle_info({Protocol, _, _, _, Data}, StateName, {stop, {shutdown, own_alert}} end; handle_info({CloseTag, Socket}, StateName, - #state{socket = Socket, close_tag = CloseTag, + #state{socket = Socket, + socket_options = #socket_options{active = Active}, + protocol_buffers = #protocol_buffers{dtls_cipher_texts = CTs}, + close_tag = CloseTag, negotiated_version = Version} = State) -> %% Note that as of DTLS 1.2 (TLS 1.1), %% failure to properly close a connection no longer requires that a %% session not be resumed. This is a change from DTLS 1.0 to conform %% with widespread implementation practice. - case Version of - {254, N} when N =< 253 -> - ok; - _ -> - %% As invalidate_sessions here causes performance issues, - %% we will conform to the widespread implementation - %% practice and go aginst the spec - %%invalidate_session(Role, Host, Port, Session) - ok - end, - ssl_connection:handle_normal_shutdown(?ALERT_REC(?FATAL, ?CLOSE_NOTIFY), StateName, State), - {stop, {shutdown, transport_closed}}; -handle_info(new_cookie_secret, StateName, #state{protocol_specific = #{cookie_secret := Secret} = CookieInfo} = State) -> + case (Active == false) andalso (CTs =/= []) of + false -> + case Version of + {254, N} when N =< 253 -> + ok; + _ -> + %% As invalidate_sessions here causes performance issues, + %% we will conform to the widespread implementation + %% practice and go aginst the spec + %%invalidate_session(Role, Host, Port, Session) + ok + end, + ssl_connection:handle_normal_shutdown(?ALERT_REC(?FATAL, ?CLOSE_NOTIFY), StateName, State), + {stop, {shutdown, transport_closed}}; + true -> + %% Fixes non-delivery of final DTLS record in {active, once}. + %% Basically allows the application the opportunity to set {active, once} again + %% and then receive the final message. + next_event(StateName, no_record, State) + end; + +handle_info(new_cookie_secret, StateName, + #state{protocol_specific = #{current_cookie_secret := Secret} = CookieInfo} = State) -> erlang:send_after(dtls_v1:cookie_timeout(), self(), new_cookie_secret), - {next_state, StateName, State#state{protocol_specific = CookieInfo#{cookie_secret => dtls_v1:cookie_secret(), - previous_cookie_secret => Secret}}}; + {next_state, StateName, State#state{protocol_specific = + CookieInfo#{current_cookie_secret => dtls_v1:cookie_secret(), + previous_cookie_secret => Secret}}}; handle_info(Msg, StateName, State) -> ssl_connection:handle_info(Msg, StateName, State). diff --git a/lib/ssl/src/dtls_record.erl b/lib/ssl/src/dtls_record.erl index 0ee51c24b6..049f83e49e 100644 --- a/lib/ssl/src/dtls_record.erl +++ b/lib/ssl/src/dtls_record.erl @@ -439,43 +439,59 @@ encode_dtls_cipher_text(Type, {MajVer, MinVer}, Fragment, encode_plain_text(Type, Version, Data, #{compression_state := CompS0, epoch := Epoch, sequence_number := Seq, + cipher_state := CipherS0, security_parameters := #security_parameters{ cipher_type = ?AEAD, + bulk_cipher_algorithm = + BulkCipherAlgo, compression_algorithm = CompAlg} } = WriteState0) -> {Comp, CompS1} = ssl_record:compress(CompAlg, Data, CompS0), - WriteState1 = WriteState0#{compression_state => CompS1}, AAD = calc_aad(Type, Version, Epoch, Seq), - ssl_record:cipher_aead(dtls_v1:corresponding_tls_version(Version), Comp, WriteState1, AAD); -encode_plain_text(Type, Version, Data, #{compression_state := CompS0, + TLSVersion = dtls_v1:corresponding_tls_version(Version), + {CipherFragment, CipherS1} = + ssl_cipher:cipher_aead(BulkCipherAlgo, CipherS0, Seq, AAD, Comp, TLSVersion), + {CipherFragment, WriteState0#{compression_state => CompS1, + cipher_state => CipherS1}}; +encode_plain_text(Type, Version, Fragment, #{compression_state := CompS0, epoch := Epoch, sequence_number := Seq, + cipher_state := CipherS0, security_parameters := - #security_parameters{compression_algorithm = CompAlg} + #security_parameters{compression_algorithm = CompAlg, + bulk_cipher_algorithm = + BulkCipherAlgo} }= WriteState0) -> - {Comp, CompS1} = ssl_record:compress(CompAlg, Data, CompS0), + {Comp, CompS1} = ssl_record:compress(CompAlg, Fragment, CompS0), WriteState1 = WriteState0#{compression_state => CompS1}, - MacHash = calc_mac_hash(Type, Version, WriteState1, Epoch, Seq, Comp), - ssl_record:cipher(dtls_v1:corresponding_tls_version(Version), Comp, WriteState1, MacHash). + MAC = calc_mac_hash(Type, Version, WriteState1, Epoch, Seq, Comp), + TLSVersion = dtls_v1:corresponding_tls_version(Version), + {CipherFragment, CipherS1} = + ssl_cipher:cipher(BulkCipherAlgo, CipherS0, MAC, Fragment, TLSVersion), + {CipherFragment, WriteState0#{cipher_state => CipherS1}}. decode_cipher_text(#ssl_tls{type = Type, version = Version, epoch = Epoch, sequence_number = Seq, fragment = CipherFragment} = CipherText, #{compression_state := CompressionS0, + cipher_state := CipherS0, security_parameters := #security_parameters{ cipher_type = ?AEAD, + bulk_cipher_algorithm = + BulkCipherAlgo, compression_algorithm = CompAlg}} = ReadState0, ConnnectionStates0) -> AAD = calc_aad(Type, Version, Epoch, Seq), - case ssl_record:decipher_aead(dtls_v1:corresponding_tls_version(Version), - CipherFragment, ReadState0, AAD) of - {PlainFragment, ReadState1} -> + TLSVersion = dtls_v1:corresponding_tls_version(Version), + case ssl_cipher:decipher_aead(BulkCipherAlgo, CipherS0, Seq, AAD, CipherFragment, TLSVersion) of + {PlainFragment, CipherState} -> {Plain, CompressionS1} = ssl_record:uncompress(CompAlg, PlainFragment, CompressionS0), - ReadState = ReadState1#{compression_state => CompressionS1}, + ReadState = ReadState0#{compression_state => CompressionS1, + cipher_state => CipherState}, ConnnectionStates = set_connection_state_by_epoch(ReadState, Epoch, ConnnectionStates0, read), {CipherText#ssl_tls{fragment = Plain}, ConnnectionStates}; #alert{} = Alert -> @@ -528,5 +544,4 @@ mac_hash(Version, MacAlg, MacSecret, SeqNo, Type, Length, Fragment) -> Length, Fragment). calc_aad(Type, {MajVer, MinVer}, Epoch, SeqNo) -> - NewSeq = (Epoch bsl 48) + SeqNo, - <<NewSeq:64/integer, ?BYTE(Type), ?BYTE(MajVer), ?BYTE(MinVer)>>. + <<?UINT16(Epoch), ?UINT48(SeqNo), ?BYTE(Type), ?BYTE(MajVer), ?BYTE(MinVer)>>. diff --git a/lib/ssl/src/ssl_cipher.erl b/lib/ssl/src/ssl_cipher.erl index 8e6860e9dc..d04f09efdc 100644 --- a/lib/ssl/src/ssl_cipher.erl +++ b/lib/ssl/src/ssl_cipher.erl @@ -40,7 +40,7 @@ ec_keyed_suites/0, anonymous_suites/1, psk_suites/1, srp_suites/0, rc4_suites/1, des_suites/1, openssl_suite/1, openssl_suite_name/1, filter/2, filter_suites/1, hash_algorithm/1, sign_algorithm/1, is_acceptable_hash/2, is_fallback/1, - random_bytes/1, calc_aad/3, calc_mac_hash/4, + random_bytes/1, calc_mac_hash/4, is_stream_ciphersuite/1]). -export_type([cipher_suite/0, @@ -157,7 +157,7 @@ cipher_aead(?CHACHA20_POLY1305, CipherState, SeqNo, AAD, Fragment, Version) -> aead_cipher(chacha20_poly1305, #cipher_state{key=Key} = CipherState, SeqNo, AAD0, Fragment, _Version) -> CipherLen = erlang:iolist_size(Fragment), AAD = <<AAD0/binary, ?UINT16(CipherLen)>>, - Nonce = <<SeqNo:64/integer>>, + Nonce = ?uint64(SeqNo), {Content, CipherTag} = crypto:block_encrypt(chacha20_poly1305, Key, Nonce, {AAD, Fragment}), {<<Content/binary, CipherTag/binary>>, CipherState}; aead_cipher(Type, #cipher_state{key=Key, iv = IV0, nonce = Nonce} = CipherState, _SeqNo, AAD0, Fragment, _Version) -> @@ -280,7 +280,7 @@ aead_ciphertext_to_state(chacha20_poly1305, SeqNo, _IV, AAD0, Fragment, _Version CipherLen = size(Fragment) - 16, <<CipherText:CipherLen/bytes, CipherTag:16/bytes>> = Fragment, AAD = <<AAD0/binary, ?UINT16(CipherLen)>>, - Nonce = <<SeqNo:64/integer>>, + Nonce = ?uint64(SeqNo), {Nonce, AAD, CipherText, CipherTag}; aead_ciphertext_to_state(_, _SeqNo, <<Salt:4/bytes, _/binary>>, AAD0, Fragment, _Version) -> CipherLen = size(Fragment) - 24, @@ -1531,10 +1531,6 @@ is_fallback(CipherSuites)-> random_bytes(N) -> crypto:strong_rand_bytes(N). -calc_aad(Type, {MajVer, MinVer}, - #{sequence_number := SeqNo}) -> - <<SeqNo:64/integer, ?BYTE(Type), ?BYTE(MajVer), ?BYTE(MinVer)>>. - calc_mac_hash(Type, Version, PlainFragment, #{sequence_number := SeqNo, mac_secret := MacSecret, diff --git a/lib/ssl/src/ssl_record.erl b/lib/ssl/src/ssl_record.erl index 539e189c4f..24e52655b0 100644 --- a/lib/ssl/src/ssl_record.erl +++ b/lib/ssl/src/ssl_record.erl @@ -45,11 +45,7 @@ -export([compress/3, uncompress/3, compressions/0]). %% Payload encryption/decryption --export([cipher/4, decipher/4, is_correct_mac/2, - cipher_aead/4, decipher_aead/4]). - -%% Encoding --export([encode_plain_text/4]). +-export([cipher/4, decipher/4, cipher_aead/4, is_correct_mac/2]). -export_type([ssl_version/0, ssl_atom_version/0, connection_states/0, connection_state/0]). @@ -271,26 +267,6 @@ set_pending_cipher_state(#{pending_read := Read, pending_read => Read#{cipher_state => ServerState}, pending_write => Write#{cipher_state => ClientState}}. -encode_plain_text(Type, Version, Data, #{compression_state := CompS0, - security_parameters := - #security_parameters{ - cipher_type = ?AEAD, - compression_algorithm = CompAlg} - } = WriteState0) -> - {Comp, CompS1} = ssl_record:compress(CompAlg, Data, CompS0), - WriteState1 = WriteState0#{compression_state => CompS1}, - AAD = ssl_cipher:calc_aad(Type, Version, WriteState1), - ssl_record:cipher_aead(Version, Comp, WriteState1, AAD); -encode_plain_text(Type, Version, Data, #{compression_state := CompS0, - security_parameters := - #security_parameters{compression_algorithm = CompAlg} - }= WriteState0) -> - {Comp, CompS1} = ssl_record:compress(CompAlg, Data, CompS0), - WriteState1 = WriteState0#{compression_state => CompS1}, - MacHash = ssl_cipher:calc_mac_hash(Type, Version, Comp, WriteState1), - ssl_record:cipher(Version, Comp, WriteState1, MacHash); -encode_plain_text(_,_,_,CS) -> - exit({cs, CS}). uncompress(?NULL, Data, CS) -> {Data, CS}. @@ -322,12 +298,12 @@ cipher(Version, Fragment, {CipherFragment, CipherS1} = ssl_cipher:cipher(BulkCipherAlgo, CipherS0, MacHash, Fragment, Version), {CipherFragment, WriteState0#{cipher_state => CipherS1}}. -%%-------------------------------------------------------------------- --spec cipher_aead(ssl_version(), iodata(), connection_state(), MacHash::binary()) -> - {CipherFragment::binary(), connection_state()}. -%% -%% Description: Payload encryption -%%-------------------------------------------------------------------- +%% %%-------------------------------------------------------------------- +%% -spec cipher_aead(ssl_version(), iodata(), connection_state(), MacHash::binary()) -> +%% {CipherFragment::binary(), connection_state()}. +%% %% +%% %% Description: Payload encryption +%% %%-------------------------------------------------------------------- cipher_aead(Version, Fragment, #{cipher_state := CipherS0, sequence_number := SeqNo, @@ -341,7 +317,8 @@ cipher_aead(Version, Fragment, {CipherFragment, WriteState0#{cipher_state => CipherS1}}. %%-------------------------------------------------------------------- --spec decipher(ssl_version(), binary(), connection_state(), boolean()) -> {binary(), binary(), connection_state} | #alert{}. +-spec decipher(ssl_version(), binary(), connection_state(), boolean()) -> + {binary(), binary(), connection_state} | #alert{}. %% %% Description: Payload decryption %%-------------------------------------------------------------------- @@ -359,26 +336,7 @@ decipher(Version, CipherFragment, #alert{} = Alert -> Alert end. -%%-------------------------------------------------------------------- --spec decipher_aead(ssl_version(), binary(), connection_state(), binary()) -> - {binary(), binary(), connection_state()} | #alert{}. -%% -%% Description: Payload decryption -%%-------------------------------------------------------------------- -decipher_aead(Version, CipherFragment, - #{sequence_number := SeqNo, - security_parameters := - #security_parameters{bulk_cipher_algorithm = - BulkCipherAlgo}, - cipher_state := CipherS0 - } = ReadState, AAD) -> - case ssl_cipher:decipher_aead(BulkCipherAlgo, CipherS0, SeqNo, AAD, CipherFragment, Version) of - {PlainFragment, CipherS1} -> - CS1 = ReadState#{cipher_state => CipherS1}, - {PlainFragment, CS1}; - #alert{} = Alert -> - Alert - end. + %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- diff --git a/lib/ssl/src/tls_record.erl b/lib/ssl/src/tls_record.erl index 993a1622fe..065c6dc8a7 100644 --- a/lib/ssl/src/tls_record.erl +++ b/lib/ssl/src/tls_record.erl @@ -372,7 +372,7 @@ get_tls_records_aux(Data, Acc) -> end. encode_plain_text(Type, Version, Data, #{current_write := Write0} = ConnectionStates) -> - {CipherFragment, Write1} = ssl_record:encode_plain_text(Type, Version, Data, Write0), + {CipherFragment, Write1} = do_encode_plain_text(Type, Version, Data, Write0), {CipherText, Write} = encode_tls_cipher_text(Type, Version, CipherFragment, Write1), {CipherText, ConnectionStates#{current_write => Write}}. @@ -446,19 +446,24 @@ decode_cipher_text(#ssl_tls{type = Type, version = Version, #{current_read := #{compression_state := CompressionS0, sequence_number := Seq, + cipher_state := CipherS0, security_parameters := #security_parameters{ cipher_type = ?AEAD, + bulk_cipher_algorithm = + BulkCipherAlgo, compression_algorithm = CompAlg} } = ReadState0} = ConnnectionStates0, _) -> - AAD = ssl_cipher:calc_aad(Type, Version, ReadState0), - case ssl_record:decipher_aead(Version, CipherFragment, ReadState0, AAD) of - {PlainFragment, ReadState1} -> + AAD = calc_aad(Type, Version, ReadState0), + case ssl_cipher:decipher_aead(BulkCipherAlgo, CipherS0, Seq, AAD, CipherFragment, Version) of + {PlainFragment, CipherS1} -> {Plain, CompressionS1} = ssl_record:uncompress(CompAlg, PlainFragment, CompressionS0), ConnnectionStates = ConnnectionStates0#{ - current_read => ReadState1#{sequence_number => Seq + 1, - compression_state => CompressionS1}}, + current_read => ReadState0#{ + cipher_state => CipherS1, + sequence_number => Seq + 1, + compression_state => CompressionS1}}, {CipherText#ssl_tls{fragment = Plain}, ConnnectionStates}; #alert{} = Alert -> Alert @@ -489,4 +494,29 @@ decode_cipher_text(#ssl_tls{type = Type, version = Version, end; #alert{} = Alert -> Alert - end. + end. + +do_encode_plain_text(Type, Version, Data, #{compression_state := CompS0, + security_parameters := + #security_parameters{ + cipher_type = ?AEAD, + compression_algorithm = CompAlg} + } = WriteState0) -> + {Comp, CompS1} = ssl_record:compress(CompAlg, Data, CompS0), + WriteState1 = WriteState0#{compression_state => CompS1}, + AAD = calc_aad(Type, Version, WriteState1), + ssl_record:cipher_aead(Version, Comp, WriteState1, AAD); +do_encode_plain_text(Type, Version, Data, #{compression_state := CompS0, + security_parameters := + #security_parameters{compression_algorithm = CompAlg} + }= WriteState0) -> + {Comp, CompS1} = ssl_record:compress(CompAlg, Data, CompS0), + WriteState1 = WriteState0#{compression_state => CompS1}, + MacHash = ssl_cipher:calc_mac_hash(Type, Version, Comp, WriteState1), + ssl_record:cipher(Version, Comp, WriteState1, MacHash); +do_encode_plain_text(_,_,_,CS) -> + exit({cs, CS}). + +calc_aad(Type, {MajVer, MinVer}, + #{sequence_number := SeqNo}) -> + <<?UINT64(SeqNo), ?BYTE(Type), ?BYTE(MajVer), ?BYTE(MinVer)>>. diff --git a/lib/ssl/test/ssl_ECC_SUITE.erl b/lib/ssl/test/ssl_ECC_SUITE.erl index b77f909dfa..b05e2c74db 100644 --- a/lib/ssl/test/ssl_ECC_SUITE.erl +++ b/lib/ssl/test/ssl_ECC_SUITE.erl @@ -91,11 +91,7 @@ init_per_suite(Config0) -> end_per_suite(Config0), try crypto:start() of ok -> - %% make rsa certs using oppenssl - Config1 = ssl_test_lib:make_rsa_cert(Config0), - Config2 = ssl_test_lib:make_ecdsa_cert(Config1), - Config = ssl_test_lib:make_ecdh_rsa_cert(Config2), - ssl_test_lib:cert_options(Config) + Config0 catch _:_ -> {skip, "Crypto did not start"} end. diff --git a/lib/ssl/test/ssl_certificate_verify_SUITE.erl b/lib/ssl/test/ssl_certificate_verify_SUITE.erl index 66b0c09b73..45bcdf1f78 100644 --- a/lib/ssl/test/ssl_certificate_verify_SUITE.erl +++ b/lib/ssl/test/ssl_certificate_verify_SUITE.erl @@ -74,7 +74,7 @@ tests() -> cert_expired, invalid_signature_client, invalid_signature_server, - extended_key_usage_verify_client, + extended_key_usage_verify_both, extended_key_usage_verify_server, critical_extension_verify_client, critical_extension_verify_server, @@ -88,18 +88,14 @@ error_handling_tests()-> unknown_server_ca_accept_verify_peer, unknown_server_ca_accept_backwardscompatibility, no_authority_key_identifier, - no_authority_key_identifier_and_nonstandard_encoding]. + no_authority_key_identifier_keyEncipherment]. -init_per_suite(Config0) -> +init_per_suite(Config) -> catch crypto:stop(), try crypto:start() of ok -> - ssl_test_lib:clean_start(), - %% make rsa certs using oppenssl - {ok, _} = make_certs:all(proplists:get_value(data_dir, Config0), - proplists:get_value(priv_dir, Config0)), - Config = ssl_test_lib:make_dsa_cert(Config0), - ssl_test_lib:cert_options(Config) + ssl_test_lib:clean_start(), + ssl_test_lib:make_rsa_cert(Config) catch _:_ -> {skip, "Crypto did not start"} end. @@ -108,49 +104,39 @@ end_per_suite(_Config) -> ssl:stop(), application:stop(crypto). -init_per_group(tls, Config) -> +init_per_group(tls, Config0) -> Version = tls_record:protocol_version(tls_record:highest_protocol_version([])), ssl:stop(), application:load(ssl), application:set_env(ssl, protocol_version, Version), - application:set_env(ssl, bypass_pem_cache, Version), ssl:start(), - NewConfig = proplists:delete(protocol, Config), - [{protocol, tls}, {version, tls_record:protocol_version(Version)} | NewConfig]; + Config = proplists:delete(protocol, Config0), + [{protocol, tls}, {version, tls_record:protocol_version(Version)} | Config]; -init_per_group(dtls, Config) -> +init_per_group(dtls, Config0) -> Version = dtls_record:protocol_version(dtls_record:highest_protocol_version([])), ssl:stop(), application:load(ssl), application:set_env(ssl, protocol_version, Version), - application:set_env(ssl, bypass_pem_cache, Version), ssl:start(), - NewConfig = proplists:delete(protocol_opts, proplists:delete(protocol, Config)), - [{protocol, dtls}, {protocol_opts, [{protocol, dtls}]}, {version, dtls_record:protocol_version(Version)} | NewConfig]; + Config = proplists:delete(protocol_opts, proplists:delete(protocol, Config0)), + [{protocol, dtls}, {protocol_opts, [{protocol, dtls}]}, {version, dtls_record:protocol_version(Version)} | Config]; init_per_group(active, Config) -> - [{active, true}, {receive_function, send_recv_result_active} | Config]; + [{active, true}, {receive_function, send_recv_result_active} | Config]; init_per_group(active_once, Config) -> - [{active, once}, {receive_function, send_recv_result_active_once} | Config]; + [{active, once}, {receive_function, send_recv_result_active_once} | Config]; init_per_group(passive, Config) -> - [{active, false}, {receive_function, send_recv_result} | Config]; + [{active, false}, {receive_function, send_recv_result} | Config]; +init_per_group(error_handling, Config) -> + [{active, false}, {receive_function, send_recv_result} | Config]; + init_per_group(_, Config) -> Config. end_per_group(_GroupName, Config) -> Config. -init_per_testcase(TestCase, Config) when TestCase == cert_expired; - TestCase == invalid_signature_client; - TestCase == invalid_signature_server; - TestCase == extended_key_usage_verify_none; - TestCase == extended_key_usage_verify_peer; - TestCase == critical_extension_verify_none; - TestCase == critical_extension_verify_peer; - TestCase == no_authority_key_identifier; - TestCase == no_authority_key_identifier_and_nonstandard_encoding-> - ssl:clear_pem_cache(), - init_per_testcase(common, Config); init_per_testcase(_TestCase, Config) -> ssl:stop(), ssl:start(), @@ -168,23 +154,23 @@ end_per_testcase(_TestCase, Config) -> verify_peer() -> [{doc,"Test option verify_peer"}]. verify_peer(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), Active = proplists:get_value(active, Config), ReceiveFunction = proplists:get_value(receive_function, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, - {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{active, Active}, {verify, verify_peer} - | ServerOpts]}]), + {mfa, {ssl_test_lib, ReceiveFunction, []}}, + {options, [{active, Active}, {verify, verify_peer} + | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, - {from, self()}, - {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{active, Active} | ClientOpts]}]), - + {from, self()}, + {mfa, {ssl_test_lib, ReceiveFunction, []}}, + {options, [{active, Active}, {verify, verify_peer} | ClientOpts]}]), + ssl_test_lib:check_result(Server, ok, Client, ok), ssl_test_lib:close(Server), ssl_test_lib:close(Client). @@ -194,23 +180,24 @@ verify_none() -> [{doc,"Test option verify_none"}]. verify_none(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), Active = proplists:get_value(active, Config), ReceiveFunction = proplists:get_value(receive_function, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, - {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{active, Active}, {verify, verify_none} - | ServerOpts]}]), + {mfa, {ssl_test_lib, ReceiveFunction, []}}, + {options, [{active, Active}, {verify, verify_none} + | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, - {from, self()}, - {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{active, Active} | ClientOpts]}]), + {from, self()}, + {mfa, {ssl_test_lib, ReceiveFunction, []}}, + {options, [{active, Active}, + {verify, verify_none} | ClientOpts]}]), ssl_test_lib:check_result(Server, ok, Client, ok), ssl_test_lib:close(Server), @@ -222,8 +209,8 @@ server_verify_client_once() -> [{doc,"Test server option verify_client_once"}]. server_verify_client_once(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_opts, []), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, []), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), Active = proplists:get_value(active, Config), ReceiveFunction = proplists:get_value(receive_function, Config), @@ -239,7 +226,7 @@ server_verify_client_once(Config) when is_list(Config) -> {host, Hostname}, {from, self()}, {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{active, Active} | ClientOpts]}]), + {options, [{active, Active} | ClientOpts]}]), ssl_test_lib:check_result(Server, ok, Client0, ok), Server ! {listen, {mfa, {ssl_test_lib, no_result, []}}}, @@ -261,8 +248,8 @@ server_require_peer_cert_ok() -> server_require_peer_cert_ok(Config) when is_list(Config) -> ServerOpts = [{verify, verify_peer}, {fail_if_no_peer_cert, true} - | ssl_test_lib:ssl_options(server_verification_opts, Config)], - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + | ssl_test_lib:ssl_options(server_rsa_opts, Config)], + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), Active = proplists:get_value(active, Config), ReceiveFunction = proplists:get_value(receive_function, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), @@ -290,20 +277,21 @@ server_require_peer_cert_fail() -> server_require_peer_cert_fail(Config) when is_list(Config) -> ServerOpts = [{verify, verify_peer}, {fail_if_no_peer_cert, true} - | ssl_test_lib:ssl_options(server_verification_opts, Config)], + | ssl_test_lib:ssl_options(server_rsa_opts, Config)], BadClientOpts = ssl_test_lib:ssl_options(empty_client_opts, Config), + Active = proplists:get_value(active, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, {from, self()}, - {options, [{active, false} | ServerOpts]}]), + {options, [{active, Active} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, - {options, [{active, false} | BadClientOpts]}]), + {options, [{active, Active} | BadClientOpts]}]), receive {Server, {error, {tls_alert, "handshake failure"}}} -> receive @@ -321,24 +309,25 @@ server_require_peer_cert_partial_chain() -> server_require_peer_cert_partial_chain(Config) when is_list(Config) -> ServerOpts = [{verify, verify_peer}, {fail_if_no_peer_cert, true} - | ssl_test_lib:ssl_options(server_verification_opts, Config)], - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + | ssl_test_lib:ssl_options(server_rsa_opts, Config)], + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + Active = proplists:get_value(active, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), {ok, ClientCAs} = file:read_file(proplists:get_value(cacertfile, ClientOpts)), - [{_,RootCA,_}, {_, _, _}] = public_key:pem_decode(ClientCAs), + [{_,RootCA,_} | _] = public_key:pem_decode(ClientCAs), Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {ssl_test_lib, no_result, []}}, - {options, [{active, false} | ServerOpts]}]), + {options, [{active, Active} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, {mfa, {ssl_test_lib, no_result, []}}, - {options, [{active, false}, + {options, [{active, Active}, {cacerts, [RootCA]} | proplists:delete(cacertfile, ClientOpts)]}]), receive @@ -356,14 +345,14 @@ server_require_peer_cert_allow_partial_chain() -> server_require_peer_cert_allow_partial_chain(Config) when is_list(Config) -> ServerOpts = [{verify, verify_peer}, {fail_if_no_peer_cert, true} - | ssl_test_lib:ssl_options(server_verification_opts, Config)], - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + | ssl_test_lib:ssl_options(server_rsa_opts, Config)], + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Active = proplists:get_value(active, Config), ReceiveFunction = proplists:get_value(receive_function, Config), {ok, ClientCAs} = file:read_file(proplists:get_value(cacertfile, ClientOpts)), - [{_,_,_}, {_, IntermidiateCA, _}] = public_key:pem_decode(ClientCAs), + [{_,_,_}, {_, IntermidiateCA, _} | _] = public_key:pem_decode(ClientCAs), PartialChain = fun(CertChain) -> case lists:member(IntermidiateCA, CertChain) of @@ -398,12 +387,12 @@ server_require_peer_cert_do_not_allow_partial_chain() -> server_require_peer_cert_do_not_allow_partial_chain(Config) when is_list(Config) -> ServerOpts = [{verify, verify_peer}, {fail_if_no_peer_cert, true} - | ssl_test_lib:ssl_options(server_verification_opts, Config)], - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + | ssl_test_lib:ssl_options(server_rsa_opts, Config)], + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), {ok, ServerCAs} = file:read_file(proplists:get_value(cacertfile, ServerOpts)), - [{_,_,_}, {_, IntermidiateCA, _}] = public_key:pem_decode(ServerCAs), + [{_,_,_}, {_, IntermidiateCA, _} | _] = public_key:pem_decode(ServerCAs), PartialChain = fun(_CertChain) -> unknown_ca @@ -439,12 +428,12 @@ server_require_peer_cert_partial_chain_fun_fail() -> server_require_peer_cert_partial_chain_fun_fail(Config) when is_list(Config) -> ServerOpts = [{verify, verify_peer}, {fail_if_no_peer_cert, true} - | ssl_test_lib:ssl_options(server_verification_opts, Config)], - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + | ssl_test_lib:ssl_options(server_rsa_opts, Config)], + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), {ok, ServerCAs} = file:read_file(proplists:get_value(cacertfile, ServerOpts)), - [{_,_,_}, {_, IntermidiateCA, _}] = public_key:pem_decode(ServerCAs), + [{_,_,_}, {_, IntermidiateCA, _} | _] = public_key:pem_decode(ServerCAs), PartialChain = fun(_CertChain) -> ture = false %% crash on purpose @@ -479,8 +468,8 @@ verify_fun_always_run_client() -> [{doc,"Verify that user verify_fun is always run (for valid and valid_peer not only unknown_extension)"}]. verify_fun_always_run_client(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, {from, self()}, @@ -524,8 +513,8 @@ verify_fun_always_run_client(Config) when is_list(Config) -> verify_fun_always_run_server() -> [{doc,"Verify that user verify_fun is always run (for valid and valid_peer not only unknown_extension)"}]. verify_fun_always_run_server(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), %% If user verify fun is called correctly we fail the connection. @@ -573,63 +562,28 @@ cert_expired() -> [{doc,"Test server with expired certificate"}]. cert_expired(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), - PrivDir = proplists:get_value(priv_dir, Config), - - KeyFile = filename:join(PrivDir, "otpCA/private/key.pem"), - [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), - Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), - - ServerCertFile = proplists:get_value(certfile, ServerOpts), - NewServerCertFile = filename:join(PrivDir, "server/expired_cert.pem"), - [{'Certificate', DerCert, _}] = ssl_test_lib:pem_to_der(ServerCertFile), - OTPCert = public_key:pkix_decode_cert(DerCert, otp), - OTPTbsCert = OTPCert#'OTPCertificate'.tbsCertificate, - {Year, Month, Day} = date(), - {Hours, Min, Sec} = time(), - NotBeforeStr = lists:flatten(io_lib:format("~p~s~s~s~s~sZ",[Year-2, - two_digits_str(Month), - two_digits_str(Day), - two_digits_str(Hours), - two_digits_str(Min), - two_digits_str(Sec)])), - NotAfterStr = lists:flatten(io_lib:format("~p~s~s~s~s~sZ",[Year-1, - two_digits_str(Month), - two_digits_str(Day), - two_digits_str(Hours), - two_digits_str(Min), - two_digits_str(Sec)])), - NewValidity = {'Validity', {generalTime, NotBeforeStr}, {generalTime, NotAfterStr}}, - - ct:log("Validity: ~p ~n NewValidity: ~p ~n", - [OTPTbsCert#'OTPTBSCertificate'.validity, NewValidity]), - - NewOTPTbsCert = OTPTbsCert#'OTPTBSCertificate'{validity = NewValidity}, - NewServerDerCert = public_key:pkix_sign(NewOTPTbsCert, Key), - ssl_test_lib:der_to_pem(NewServerCertFile, [{'Certificate', NewServerDerCert, not_encrypted}]), - NewServerOpts = [{certfile, NewServerCertFile} | proplists:delete(certfile, ServerOpts)], - + Active = proplists:get_value(active, Config), + {ClientOpts0, ServerOpts0} = ssl_test_lib:make_rsa_cert_chains([{server_ca_0, + [{validity, {{Year-2, Month, Day}, + {Year-1, Month, Day}}}]}], + Config, "_expired"), + ClientOpts = ssl_test_lib:ssl_options(ClientOpts0, Config), + ServerOpts = ssl_test_lib:ssl_options(ServerOpts0, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, {from, self()}, - {options, NewServerOpts}]), + {options, [{active, Active}| ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, - {options, [{verify, verify_peer} | ClientOpts]}]), - receive - {Client, {error, {tls_alert, "certificate expired"}}} -> - receive - {Server, {error, {tls_alert, "certificate expired"}}} -> - ok; - {Server, {error, closed}} -> - ok - end - end. + {options, [{verify, verify_peer}, {active, Active} | ClientOpts]}]), + + tcp_delivery_workaround(Server, {error, {tls_alert, "certificate expired"}}, + Client, {error, {tls_alert, "certificate expired"}}). two_digits_str(N) when N < 10 -> lists:flatten(io_lib:format("0~p", [N])); @@ -638,60 +592,32 @@ two_digits_str(N) -> %%-------------------------------------------------------------------- extended_key_usage_verify_server() -> - [{doc,"Test cert that has a critical extended_key_usage extension in verify_peer mode for server"}]. - -extended_key_usage_verify_server(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), - PrivDir = proplists:get_value(priv_dir, Config), + [{doc,"Test cert that has a critical extended_key_usage extension in server cert"}]. + +extended_key_usage_verify_server(Config) when is_list(Config) -> + {ClientOpts0, ServerOpts0} = ssl_test_lib:make_rsa_cert_chains([{server_peer_opts, + [{extensions, + [{?'id-ce-extKeyUsage', + [?'id-kp-serverAuth'], true}] + }]}], Config, "_keyusage_server"), + ClientOpts = ssl_test_lib:ssl_options(ClientOpts0, Config), + ServerOpts = ssl_test_lib:ssl_options(ServerOpts0, Config), Active = proplists:get_value(active, Config), ReceiveFunction = proplists:get_value(receive_function, Config), - KeyFile = filename:join(PrivDir, "otpCA/private/key.pem"), - [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), - Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), - - ServerCertFile = proplists:get_value(certfile, ServerOpts), - NewServerCertFile = filename:join(PrivDir, "server/new_cert.pem"), - [{'Certificate', ServerDerCert, _}] = ssl_test_lib:pem_to_der(ServerCertFile), - ServerOTPCert = public_key:pkix_decode_cert(ServerDerCert, otp), - ServerExtKeyUsageExt = {'Extension', ?'id-ce-extKeyUsage', true, [?'id-kp-serverAuth']}, - ServerOTPTbsCert = ServerOTPCert#'OTPCertificate'.tbsCertificate, - ServerExtensions = ServerOTPTbsCert#'OTPTBSCertificate'.extensions, - NewServerOTPTbsCert = ServerOTPTbsCert#'OTPTBSCertificate'{extensions = - [ServerExtKeyUsageExt | - ServerExtensions]}, - NewServerDerCert = public_key:pkix_sign(NewServerOTPTbsCert, Key), - ssl_test_lib:der_to_pem(NewServerCertFile, [{'Certificate', NewServerDerCert, not_encrypted}]), - NewServerOpts = [{certfile, NewServerCertFile} | proplists:delete(certfile, ServerOpts)], - - ClientCertFile = proplists:get_value(certfile, ClientOpts), - NewClientCertFile = filename:join(PrivDir, "client/new_cert.pem"), - [{'Certificate', ClientDerCert, _}] = ssl_test_lib:pem_to_der(ClientCertFile), - ClientOTPCert = public_key:pkix_decode_cert(ClientDerCert, otp), - ClientExtKeyUsageExt = {'Extension', ?'id-ce-extKeyUsage', true, [?'id-kp-clientAuth']}, - ClientOTPTbsCert = ClientOTPCert#'OTPCertificate'.tbsCertificate, - ClientExtensions = ClientOTPTbsCert#'OTPTBSCertificate'.extensions, - NewClientOTPTbsCert = ClientOTPTbsCert#'OTPTBSCertificate'{extensions = - [ClientExtKeyUsageExt | - ClientExtensions]}, - NewClientDerCert = public_key:pkix_sign(NewClientOTPTbsCert, Key), - ssl_test_lib:der_to_pem(NewClientCertFile, [{'Certificate', NewClientDerCert, not_encrypted}]), - NewClientOpts = [{certfile, NewClientCertFile} | proplists:delete(certfile, ClientOpts)], - {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{verify, verify_peer}, {active, Active} | NewServerOpts]}]), + {options, [{verify, verify_none}, {active, Active} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{verify, verify_none}, {active, Active} | - NewClientOpts]}]), + {options, [{verify, verify_peer}, {active, Active} | + ClientOpts]}]), ssl_test_lib:check_result(Server, ok, Client, ok), @@ -699,60 +625,35 @@ extended_key_usage_verify_server(Config) when is_list(Config) -> ssl_test_lib:close(Client). %%-------------------------------------------------------------------- -extended_key_usage_verify_client() -> +extended_key_usage_verify_both() -> [{doc,"Test cert that has a critical extended_key_usage extension in client verify_peer mode"}]. -extended_key_usage_verify_client(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), - PrivDir = proplists:get_value(priv_dir, Config), +extended_key_usage_verify_both(Config) when is_list(Config) -> + {ClientOpts0, ServerOpts0} = ssl_test_lib:make_rsa_cert_chains([{server_peer_opts, + [{extensions, [{?'id-ce-extKeyUsage', + [?'id-kp-serverAuth'], true}] + }]}, + {client_peer_opts, + [{extensions, [{?'id-ce-extKeyUsage', + [?'id-kp-clientAuth'], true}] + }]}], Config, "_keyusage_both"), + ClientOpts = ssl_test_lib:ssl_options(ClientOpts0, Config), + ServerOpts = ssl_test_lib:ssl_options(ServerOpts0, Config), Active = proplists:get_value(active, Config), ReceiveFunction = proplists:get_value(receive_function, Config), - KeyFile = filename:join(PrivDir, "otpCA/private/key.pem"), - [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), - Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), - - ServerCertFile = proplists:get_value(certfile, ServerOpts), - NewServerCertFile = filename:join(PrivDir, "server/new_cert.pem"), - [{'Certificate', ServerDerCert, _}] = ssl_test_lib:pem_to_der(ServerCertFile), - ServerOTPCert = public_key:pkix_decode_cert(ServerDerCert, otp), - ServerExtKeyUsageExt = {'Extension', ?'id-ce-extKeyUsage', true, [?'id-kp-serverAuth']}, - ServerOTPTbsCert = ServerOTPCert#'OTPCertificate'.tbsCertificate, - ServerExtensions = ServerOTPTbsCert#'OTPTBSCertificate'.extensions, - NewServerOTPTbsCert = ServerOTPTbsCert#'OTPTBSCertificate'{extensions = - [ServerExtKeyUsageExt | - ServerExtensions]}, - NewServerDerCert = public_key:pkix_sign(NewServerOTPTbsCert, Key), - ssl_test_lib:der_to_pem(NewServerCertFile, [{'Certificate', NewServerDerCert, not_encrypted}]), - NewServerOpts = [{certfile, NewServerCertFile} | proplists:delete(certfile, ServerOpts)], - - ClientCertFile = proplists:get_value(certfile, ClientOpts), - NewClientCertFile = filename:join(PrivDir, "client/new_cert.pem"), - [{'Certificate', ClientDerCert, _}] = ssl_test_lib:pem_to_der(ClientCertFile), - ClientOTPCert = public_key:pkix_decode_cert(ClientDerCert, otp), - ClientExtKeyUsageExt = {'Extension', ?'id-ce-extKeyUsage', true, [?'id-kp-clientAuth']}, - ClientOTPTbsCert = ClientOTPCert#'OTPCertificate'.tbsCertificate, - ClientExtensions = ClientOTPTbsCert#'OTPTBSCertificate'.extensions, - NewClientOTPTbsCert = ClientOTPTbsCert#'OTPTBSCertificate'{extensions = - [ClientExtKeyUsageExt | - ClientExtensions]}, - NewClientDerCert = public_key:pkix_sign(NewClientOTPTbsCert, Key), - ssl_test_lib:der_to_pem(NewClientCertFile, [{'Certificate', NewClientDerCert, not_encrypted}]), - NewClientOpts = [{certfile, NewClientCertFile} | proplists:delete(certfile, ClientOpts)], - {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{verify, verify_none}, {active, Active} | NewServerOpts]}]), + {options, [{verify, verify_peer}, {active, Active} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{verify, verify_none}, {active, Active} | NewClientOpts]}]), + {options, [{verify, verify_peer}, {active, Active} | ClientOpts]}]), ssl_test_lib:check_result(Server, ok, Client, ok), @@ -764,132 +665,103 @@ critical_extension_verify_server() -> [{doc,"Test cert that has a critical unknown extension in verify_peer mode"}]. critical_extension_verify_server(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), - PrivDir = proplists:get_value(priv_dir, Config), + {ClientOpts0, ServerOpts0} = ssl_test_lib:make_rsa_cert_chains([{client_peer_opts, + [{extensions, [{{2,16,840,1,113730,1,1}, + <<3,2,6,192>>, true}] + }]}], Config, "_client_unknown_extension"), + ClientOpts = ssl_test_lib:ssl_options(ClientOpts0, Config), + ServerOpts = ssl_test_lib:ssl_options(ServerOpts0, Config), Active = proplists:get_value(active, Config), ReceiveFunction = proplists:get_value(receive_function, Config), - KeyFile = filename:join(PrivDir, "otpCA/private/key.pem"), - NewCertName = integer_to_list(erlang:unique_integer()) ++ ".pem", - - ServerCertFile = proplists:get_value(certfile, ServerOpts), - NewServerCertFile = filename:join([PrivDir, "server", NewCertName]), - add_critical_netscape_cert_type(ServerCertFile, NewServerCertFile, KeyFile), - NewServerOpts = [{certfile, NewServerCertFile} | proplists:delete(certfile, ServerOpts)], - - ClientCertFile = proplists:get_value(certfile, ClientOpts), - NewClientCertFile = filename:join([PrivDir, "client", NewCertName]), - add_critical_netscape_cert_type(ClientCertFile, NewClientCertFile, KeyFile), - NewClientOpts = [{certfile, NewClientCertFile} | proplists:delete(certfile, ClientOpts)], - {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server_error( [{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{verify, verify_peer}, {active, Active} | NewServerOpts]}]), + {options, [{verify, verify_peer}, {active, Active} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client_error( [{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{verify, verify_none}, {active, Active} | NewClientOpts]}]), + {options, [{verify, verify_none}, {active, Active} | ClientOpts]}]), %% This certificate has a critical extension that we don't - %% understand. Therefore, verification should fail. - tcp_delivery_workaround(Server, {error, {tls_alert, "unsupported certificate"}}, - Client, {error, {tls_alert, "unsupported certificate"}}), + %% understand. Therefore, verification should fail. - ssl_test_lib:close(Server), - ok. + tcp_delivery_workaround(Server, {error, {tls_alert, "unsupported certificate"}}, + Client, {error, {tls_alert, "unsupported certificate"}}), + + ssl_test_lib:close(Server). %%-------------------------------------------------------------------- critical_extension_verify_client() -> [{doc,"Test cert that has a critical unknown extension in verify_peer mode"}]. critical_extension_verify_client(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), - PrivDir = proplists:get_value(priv_dir, Config), + {ClientOpts0, ServerOpts0} = ssl_test_lib:make_rsa_cert_chains([{server_peer_opts, + [{extensions, [{{2,16,840,1,113730,1,1}, + <<3,2,6,192>>, true}] + }]}], Config, "_server_unknown_extensions"), + ClientOpts = ssl_test_lib:ssl_options(ClientOpts0, Config), + ServerOpts = ssl_test_lib:ssl_options(ServerOpts0, Config), Active = proplists:get_value(active, Config), ReceiveFunction = proplists:get_value(receive_function, Config), - KeyFile = filename:join(PrivDir, "otpCA/private/key.pem"), - NewCertName = integer_to_list(erlang:unique_integer()) ++ ".pem", - - ServerCertFile = proplists:get_value(certfile, ServerOpts), - NewServerCertFile = filename:join([PrivDir, "server", NewCertName]), - add_critical_netscape_cert_type(ServerCertFile, NewServerCertFile, KeyFile), - NewServerOpts = [{certfile, NewServerCertFile} | proplists:delete(certfile, ServerOpts)], - - ClientCertFile = proplists:get_value(certfile, ClientOpts), - NewClientCertFile = filename:join([PrivDir, "client", NewCertName]), - add_critical_netscape_cert_type(ClientCertFile, NewClientCertFile, KeyFile), - NewClientOpts = [{certfile, NewClientCertFile} | proplists:delete(certfile, ClientOpts)], - {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server_error( [{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{verify, verify_none}, {active, Active} | NewServerOpts]}]), + {options, [{verify, verify_none}, {active, Active} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client_error( [{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{verify, verify_peer}, {active, Active} | NewClientOpts]}]), + {options, [{verify, verify_peer}, {active, Active} | ClientOpts]}]), %% This certificate has a critical extension that we don't %% understand. Therefore, verification should fail. - tcp_delivery_workaround(Server, {error, {tls_alert, "unsupported certificate"}}, - Client, {error, {tls_alert, "unsupported certificate"}}), + ssl_test_lib:check_result(Server, {error, {tls_alert, "unsupported certificate"}}, + Client, {error, {tls_alert, "unsupported certificate"}}), + + ssl_test_lib:close(Server). - ssl_test_lib:close(Server), - ok. %%-------------------------------------------------------------------- critical_extension_verify_none() -> [{doc,"Test cert that has a critical unknown extension in verify_none mode"}]. critical_extension_verify_none(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), - PrivDir = proplists:get_value(priv_dir, Config), + {ClientOpts0, ServerOpts0} = ssl_test_lib:make_rsa_cert_chains([{client_peer_opts, + [{extensions, + [{{2,16,840,1,113730,1,1}, + <<3,2,6,192>>, true}] + }]}], Config, "_unknown_extensions"), + ClientOpts = ssl_test_lib:ssl_options(ClientOpts0, Config), + ServerOpts = ssl_test_lib:ssl_options(ServerOpts0, Config), Active = proplists:get_value(active, Config), ReceiveFunction = proplists:get_value(receive_function, Config), - KeyFile = filename:join(PrivDir, "otpCA/private/key.pem"), - NewCertName = integer_to_list(erlang:unique_integer()) ++ ".pem", - - ServerCertFile = proplists:get_value(certfile, ServerOpts), - NewServerCertFile = filename:join([PrivDir, "server", NewCertName]), - add_critical_netscape_cert_type(ServerCertFile, NewServerCertFile, KeyFile), - NewServerOpts = [{certfile, NewServerCertFile} | proplists:delete(certfile, ServerOpts)], - - ClientCertFile = proplists:get_value(certfile, ClientOpts), - NewClientCertFile = filename:join([PrivDir, "client", NewCertName]), - add_critical_netscape_cert_type(ClientCertFile, NewClientCertFile, KeyFile), - NewClientOpts = [{certfile, NewClientCertFile} | proplists:delete(certfile, ClientOpts)], - {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server( [{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{verify, verify_none}, {active, Active} | NewServerOpts]}]), + {options, [{verify, verify_none}, {active, Active} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client( [{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{verify, verify_none}, {active, Active} | NewClientOpts]}]), + {options, [{verify, verify_none}, {active, Active} | ClientOpts]}]), %% This certificate has a critical extension that we don't %% understand. But we're using `verify_none', so verification @@ -897,28 +769,7 @@ critical_extension_verify_none(Config) when is_list(Config) -> ssl_test_lib:check_result(Server, ok, Client, ok), ssl_test_lib:close(Server), - ssl_test_lib:close(Client), - ok. - -add_critical_netscape_cert_type(CertFile, NewCertFile, KeyFile) -> - [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), - Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), - - [{'Certificate', DerCert, _}] = ssl_test_lib:pem_to_der(CertFile), - OTPCert = public_key:pkix_decode_cert(DerCert, otp), - %% This is the "Netscape Cert Type" extension, telling us that the - %% certificate can be used for SSL clients and SSL servers. - NetscapeCertTypeExt = #'Extension'{ - extnID = {2,16,840,1,113730,1,1}, - critical = true, - extnValue = <<3,2,6,192>>}, - OTPTbsCert = OTPCert#'OTPCertificate'.tbsCertificate, - Extensions = OTPTbsCert#'OTPTBSCertificate'.extensions, - NewOTPTbsCert = OTPTbsCert#'OTPTBSCertificate'{ - extensions = [NetscapeCertTypeExt] ++ Extensions}, - NewDerCert = public_key:pkix_sign(NewOTPTbsCert, Key), - ssl_test_lib:der_to_pem(NewCertFile, [{'Certificate', NewDerCert, not_encrypted}]), - ok. + ssl_test_lib:close(Client). %%-------------------------------------------------------------------- no_authority_key_identifier() -> @@ -926,35 +777,21 @@ no_authority_key_identifier() -> " but are present in trusted certs db."}]. no_authority_key_identifier(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), - PrivDir = proplists:get_value(priv_dir, Config), - - KeyFile = filename:join(PrivDir, "otpCA/private/key.pem"), - [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), - Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), - - CertFile = proplists:get_value(certfile, ServerOpts), - NewCertFile = filename:join(PrivDir, "server/new_cert.pem"), - [{'Certificate', DerCert, _}] = ssl_test_lib:pem_to_der(CertFile), - OTPCert = public_key:pkix_decode_cert(DerCert, otp), - OTPTbsCert = OTPCert#'OTPCertificate'.tbsCertificate, - Extensions = OTPTbsCert#'OTPTBSCertificate'.extensions, - NewExtensions = delete_authority_key_extension(Extensions, []), - NewOTPTbsCert = OTPTbsCert#'OTPTBSCertificate'{extensions = NewExtensions}, - - ct:log("Extensions ~p~n, NewExtensions: ~p~n", [Extensions, NewExtensions]), - - NewDerCert = public_key:pkix_sign(NewOTPTbsCert, Key), - ssl_test_lib:der_to_pem(NewCertFile, [{'Certificate', NewDerCert, not_encrypted}]), - NewServerOpts = [{certfile, NewCertFile} | proplists:delete(certfile, ServerOpts)], + {ClientOpts0, ServerOpts0} = ssl_test_lib:make_rsa_cert_chains([{server_peer_opts, + [{extensions, [{auth_key_id, undefined}] + }]}, + {client_peer_opts, + [{extensions, [{auth_key_id, undefined}] + }]}], Config, "_peer_no_auth_key_id"), + ClientOpts = ssl_test_lib:ssl_options(ClientOpts0, Config), + ServerOpts = ssl_test_lib:ssl_options(ServerOpts0, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {ssl_test_lib, send_recv_result_active, []}}, - {options, NewServerOpts}]), + {options, ServerOpts}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, @@ -970,53 +807,35 @@ no_authority_key_identifier(Config) when is_list(Config) -> delete_authority_key_extension([], Acc) -> lists:reverse(Acc); delete_authority_key_extension([#'Extension'{extnID = ?'id-ce-authorityKeyIdentifier'} | Rest], - Acc) -> + Acc) -> delete_authority_key_extension(Rest, Acc); delete_authority_key_extension([Head | Rest], Acc) -> delete_authority_key_extension(Rest, [Head | Acc]). %%-------------------------------------------------------------------- -no_authority_key_identifier_and_nonstandard_encoding() -> - [{doc, "Test cert with nonstandard encoding that does not have" - " authorityKeyIdentifier extension but are present in trusted certs db."}]. - -no_authority_key_identifier_and_nonstandard_encoding(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), - PrivDir = proplists:get_value(priv_dir, Config), - - KeyFile = filename:join(PrivDir, "otpCA/private/key.pem"), - [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), - Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), - - CertFile = proplists:get_value(certfile, ServerOpts), - NewCertFile = filename:join(PrivDir, "server/new_cert.pem"), - [{'Certificate', DerCert, _}] = ssl_test_lib:pem_to_der(CertFile), - ServerCert = public_key:pkix_decode_cert(DerCert, plain), - ServerTbsCert = ServerCert#'Certificate'.tbsCertificate, - Extensions0 = ServerTbsCert#'TBSCertificate'.extensions, - %% need to remove authorityKeyIdentifier extension to cause DB lookup by signature - Extensions = delete_authority_key_extension(Extensions0, []), - NewExtensions = replace_key_usage_extension(Extensions, []), - NewServerTbsCert = ServerTbsCert#'TBSCertificate'{extensions = NewExtensions}, - - ct:log("Extensions ~p~n, NewExtensions: ~p~n", [Extensions, NewExtensions]), - - TbsDer = public_key:pkix_encode('TBSCertificate', NewServerTbsCert, plain), - Sig = public_key:sign(TbsDer, md5, Key), - NewServerCert = ServerCert#'Certificate'{tbsCertificate = NewServerTbsCert, signature = Sig}, - NewDerCert = public_key:pkix_encode('Certificate', NewServerCert, plain), - ssl_test_lib:der_to_pem(NewCertFile, [{'Certificate', NewDerCert, not_encrypted}]), - NewServerOpts = [{certfile, NewCertFile} | proplists:delete(certfile, ServerOpts)], - +no_authority_key_identifier_keyEncipherment() -> + [{doc, "Test cert with keyEncipherment key_usage an no" + " authorityKeyIdentifier extension, but are present in trusted certs db."}]. + +no_authority_key_identifier_keyEncipherment(Config) when is_list(Config) -> + {ClientOpts0, ServerOpts0} = ssl_test_lib:make_rsa_cert_chains([{server_peer_opts, + [{extensions, [{auth_key_id, undefined}, + {key_usage, [digitalSignature, + keyEncipherment]}] + }]}, + {client_peer_opts, + [{extensions, [{auth_key_id, undefined}] + }]}], Config, "_peer_keyEncipherment"), + ClientOpts = ssl_test_lib:ssl_options(ClientOpts0, Config), + ServerOpts = ssl_test_lib:ssl_options(ServerOpts0, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {ssl_test_lib, send_recv_result_active, []}}, - {options, [{active, true} | NewServerOpts]}]), + {options, [{active, true} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, @@ -1028,14 +847,6 @@ no_authority_key_identifier_and_nonstandard_encoding(Config) when is_list(Config ssl_test_lib:close(Server), ssl_test_lib:close(Client). -replace_key_usage_extension([], Acc) -> - lists:reverse(Acc); -replace_key_usage_extension([#'Extension'{extnID = ?'id-ce-keyUsage'} = E | Rest], Acc) -> - %% A nonstandard DER encoding of [digitalSignature, keyEncipherment] - Val = <<3, 2, 0, 16#A0>>, - replace_key_usage_extension(Rest, [E#'Extension'{extnValue = Val} | Acc]); -replace_key_usage_extension([Head | Rest], Acc) -> - replace_key_usage_extension(Rest, [Head | Acc]). %%-------------------------------------------------------------------- @@ -1043,16 +854,16 @@ invalid_signature_server() -> [{doc,"Test client with invalid signature"}]. invalid_signature_server(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), PrivDir = proplists:get_value(priv_dir, Config), - KeyFile = filename:join(PrivDir, "server/key.pem"), + KeyFile = proplists:get_value(keyfile, ServerOpts), [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), ServerCertFile = proplists:get_value(certfile, ServerOpts), - NewServerCertFile = filename:join(PrivDir, "server/invalid_cert.pem"), + NewServerCertFile = filename:join(PrivDir, "server_invalid_cert.pem"), [{'Certificate', ServerDerCert, _}] = ssl_test_lib:pem_to_der(ServerCertFile), ServerOTPCert = public_key:pkix_decode_cert(ServerDerCert, otp), ServerOTPTbsCert = ServerOTPCert#'OTPCertificate'.tbsCertificate, @@ -1071,8 +882,8 @@ invalid_signature_server(Config) when is_list(Config) -> {from, self()}, {options, [{verify, verify_peer} | ClientOpts]}]), - tcp_delivery_workaround(Server, {error, {tls_alert, "bad certificate"}}, - Client, {error, {tls_alert, "bad certificate"}}). + tcp_delivery_workaround(Server, {error, {tls_alert, "unknown ca"}}, + Client, {error, {tls_alert, "unknown ca"}}). %%-------------------------------------------------------------------- @@ -1080,16 +891,16 @@ invalid_signature_client() -> [{doc,"Test server with invalid signature"}]. invalid_signature_client(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), PrivDir = proplists:get_value(priv_dir, Config), - KeyFile = filename:join(PrivDir, "client/key.pem"), + KeyFile = proplists:get_value(keyfile, ClientOpts), [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), ClientCertFile = proplists:get_value(certfile, ClientOpts), - NewClientCertFile = filename:join(PrivDir, "client/invalid_cert.pem"), + NewClientCertFile = filename:join(PrivDir, "client_invalid_cert.pem"), [{'Certificate', ClientDerCert, _}] = ssl_test_lib:pem_to_der(ClientCertFile), ClientOTPCert = public_key:pkix_decode_cert(ClientDerCert, otp), ClientOTPTbsCert = ClientOTPCert#'OTPCertificate'.tbsCertificate, @@ -1108,8 +919,8 @@ invalid_signature_client(Config) when is_list(Config) -> {from, self()}, {options, NewClientOpts}]), - tcp_delivery_workaround(Server, {error, {tls_alert, "bad certificate"}}, - Client, {error, {tls_alert, "bad certificate"}}). + tcp_delivery_workaround(Server, {error, {tls_alert, "unknown ca"}}, + Client, {error, {tls_alert, "unknown ca"}}). %%-------------------------------------------------------------------- @@ -1118,8 +929,14 @@ client_with_cert_cipher_suites_handshake() -> [{doc, "Test that client with a certificate without keyEncipherment usage " " extension can connect to a server with restricted cipher suites "}]. client_with_cert_cipher_suites_handshake(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts_digital_signature_only, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + {ClientOpts0, ServerOpts0} = ssl_test_lib:make_rsa_cert_chains([{client_peer_opts, + [{extensions, + [{key_usage, [digitalSignature]}] + }]}], Config, "_sign_only_extensions"), + + + ClientOpts = ssl_test_lib:ssl_options(ClientOpts0, Config), + ServerOpts = ssl_test_lib:ssl_options(ServerOpts0, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, @@ -1148,7 +965,7 @@ client_with_cert_cipher_suites_handshake(Config) when is_list(Config) -> server_verify_no_cacerts() -> [{doc,"Test server must have cacerts if it wants to verify client"}]. server_verify_no_cacerts(Config) when is_list(Config) -> - ServerOpts = proplists:delete(cacertfile, ssl_test_lib:ssl_options(server_opts, Config)), + ServerOpts = proplists:delete(cacertfile, ssl_test_lib:ssl_options(server_rsa_opts, Config)), {_, ServerNode, _} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, {from, self()}, @@ -1163,7 +980,7 @@ unknown_server_ca_fail() -> [{doc,"Test that the client fails if the ca is unknown in verify_peer mode"}]. unknown_server_ca_fail(Config) when is_list(Config) -> ClientOpts = ssl_test_lib:ssl_options(empty_client_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, {from, self()}, @@ -1207,7 +1024,7 @@ unknown_server_ca_accept_verify_none() -> [{doc,"Test that the client succeds if the ca is unknown in verify_none mode"}]. unknown_server_ca_accept_verify_none(Config) when is_list(Config) -> ClientOpts = ssl_test_lib:ssl_options(empty_client_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, @@ -1232,7 +1049,7 @@ unknown_server_ca_accept_verify_peer() -> " with a verify_fun that accepts the unknown ca error"}]. unknown_server_ca_accept_verify_peer(Config) when is_list(Config) -> ClientOpts = ssl_test_lib:ssl_options(empty_client_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, @@ -1271,7 +1088,7 @@ unknown_server_ca_accept_backwardscompatibility() -> [{doc,"Test that old style verify_funs will work"}]. unknown_server_ca_accept_backwardscompatibility(Config) when is_list(Config) -> ClientOpts = ssl_test_lib:ssl_options(empty_client_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index 302b5178a5..b8fd5dc975 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -485,6 +485,18 @@ make_dsa_cert(Config) -> {certfile, ClientCertFile}, {keyfile, ClientKeyFile}]} | Config]. +make_rsa_cert_chains(ChainConf, Config, Suffix) -> + CryptoSupport = crypto:supports(), + KeyGenSpec = key_gen_info(rsa, rsa), + ClientFileBase = filename:join([proplists:get_value(priv_dir, Config), "rsa" ++ Suffix]), + ServerFileBase = filename:join([proplists:get_value(priv_dir, Config), "rsa" ++ Suffix]), + GenCertData = x509_test:gen_test_certs([{digest, appropriate_sha(CryptoSupport)} | KeyGenSpec] ++ ChainConf), + [{server_config, ServerConf}, + {client_config, ClientConf}] = + x509_test:gen_pem_config_files(GenCertData, ClientFileBase, ServerFileBase), + {[{verify, verify_peer} | ClientConf], + [{reuseaddr, true}, {verify, verify_peer} | ServerConf] + }. make_ec_cert_chains(ClientChainType, ServerChainType, Config) -> CryptoSupport = crypto:supports(), @@ -524,6 +536,11 @@ key_gen_spec(Role, ecdhe_rsa) -> [{list_to_atom(Role ++ "_key_gen"), hardcode_rsa_key(1)}, {list_to_atom(Role ++ "_key_gen_chain"), [hardcode_rsa_key(2), hardcode_rsa_key(3)]} + ]; +key_gen_spec(Role, rsa) -> + [{list_to_atom(Role ++ "_key_gen"), hardcode_rsa_key(1)}, + {list_to_atom(Role ++ "_key_gen_chain"), [hardcode_rsa_key(2), + hardcode_rsa_key(3)]} ]. make_ecdsa_cert(Config) -> CryptoSupport = crypto:supports(), @@ -571,7 +588,8 @@ make_rsa_cert(Config) -> {server_rsa_verify_opts, [{ssl_imp, new}, {reuseaddr, true}, {verify, verify_peer} | ServerConf]}, - {client_rsa_opts, ClientConf} + {client_rsa_opts, ClientConf}, + {client_rsa_verify_opts, [{verify, verify_peer} |ClientConf]} | Config]; false -> Config @@ -935,9 +953,9 @@ available_suites(Version) -> rsa_non_signed_suites(Version) -> lists:filter(fun({rsa, _, _}) -> - true; + false; (_) -> - false + true end, available_suites(Version)). @@ -1398,10 +1416,13 @@ do_supports_ssl_tls_version(Port) -> true end. -ssl_options(Option, Config) -> +ssl_options(Option, Config) when is_atom(Option) -> ProtocolOpts = proplists:get_value(protocol_opts, Config, []), Opts = proplists:get_value(Option, Config, []), - Opts ++ ProtocolOpts. + Opts ++ ProtocolOpts; +ssl_options(Options, Config) -> + ProtocolOpts = proplists:get_value(protocol_opts, Config, []), + Options ++ ProtocolOpts. protocol_version(Config) -> protocol_version(Config, atom). diff --git a/lib/ssl/test/x509_test.erl b/lib/ssl/test/x509_test.erl index 13f8dfdaa9..c36e96013b 100644 --- a/lib/ssl/test/x509_test.erl +++ b/lib/ssl/test/x509_test.erl @@ -96,7 +96,7 @@ gen_pem_config_files(GenCertData, ClientBase, ServerBase) -> public_key:generate_key(KeyGen) end. - root_cert(Role, PrivKey, Opts) -> +root_cert(Role, PrivKey, Opts) -> TBS = cert_template(), Issuer = issuer("root", Role, " ROOT CA"), OTPTBS = TBS#'OTPTBSCertificate'{ @@ -105,7 +105,7 @@ gen_pem_config_files(GenCertData, ClientBase, ServerBase) -> validity = validity(Opts), subject = Issuer, subjectPublicKeyInfo = public_key(PrivKey), - extensions = extensions(Opts) + extensions = extensions(ca, Opts) }, public_key:pkix_sign(OTPTBS, PrivKey). @@ -175,32 +175,31 @@ validity(Opts) -> #'Validity'{notBefore={generalTime, Format(DefFrom)}, notAfter ={generalTime, Format(DefTo)}}. -extensions(Opts) -> - case proplists:get_value(extensions, Opts, []) of - false -> - asn1_NOVALUE; - Exts -> - lists:flatten([extension(Ext) || Ext <- default_extensions(Exts)]) - end. +extensions(Type, Opts) -> + Exts = proplists:get_value(extensions, Opts, []), + lists:flatten([extension(Ext) || Ext <- default_extensions(Type, Exts)]). + +%% Common extension: name_constraints, policy_constraints, ext_key_usage, inhibit_any, +%% auth_key_id, subject_key_id, policy_mapping, + +default_extensions(ca, Exts) -> + Def = [{key_usage, [keyCertSign, cRLSign]}, + {basic_constraints, default}], + add_default_extensions(Def, Exts); -default_extensions(Exts) -> - Def = [{key_usage,undefined}, - {subject_altname, undefined}, - {issuer_altname, undefined}, - {basic_constraints, default}, - {name_constraints, undefined}, - {policy_constraints, undefined}, - {ext_key_usage, undefined}, - {inhibit_any, undefined}, - {auth_key_id, undefined}, - {subject_key_id, undefined}, - {policy_mapping, undefined}], +default_extensions(peer, Exts) -> + Def = [{key_usage, [digitalSignature, keyAgreement]}], + add_default_extensions(Def, Exts). + +add_default_extensions(Def, Exts) -> Filter = fun({Key, _}, D) -> - lists:keydelete(Key, 1, D) + lists:keydelete(Key, 1, D); + ({Key, _, _}, D) -> + lists:keydelete(Key, 1, D) end, Exts ++ lists:foldl(Filter, Def, Exts). - -extension({_, undefined}) -> + +extension({_, undefined}) -> []; extension({basic_constraints, Data}) -> case Data of @@ -218,6 +217,17 @@ extension({basic_constraints, Data}) -> #'Extension'{extnID = ?'id-ce-basicConstraints', extnValue = Data} end; +extension({auth_key_id, {Oid, Issuer, SNr}}) -> + #'Extension'{extnID = ?'id-ce-authorityKeyIdentifier', + extnValue = #'AuthorityKeyIdentifier'{ + keyIdentifier = Oid, + authorityCertIssuer = Issuer, + authorityCertSerialNumber = SNr}, + critical = false}; +extension({key_usage, Value}) -> + #'Extension'{extnID = ?'id-ce-keyUsage', + extnValue = Value, + critical = false}; extension({Id, Data, Critical}) -> #'Extension'{extnID = Id, extnValue = Data, critical = Critical}. @@ -277,24 +287,31 @@ cert_chain(Role, Root, RootKey, Opts, Keys) -> cert_chain(Role, Root, RootKey, Opts, Keys, 0, []). cert_chain(Role, IssuerCert, IssuerKey, Opts, [Key], _, Acc) -> + PeerOpts = list_to_atom(atom_to_list(Role) ++ "_peer_opts"), Cert = cert(Role, public_key:pkix_decode_cert(IssuerCert, otp), - IssuerKey, Key, "admin", " Peer cert", Opts), + IssuerKey, Key, "admin", " Peer cert", Opts, PeerOpts, peer), [{Cert, Key}, {IssuerCert, IssuerKey} | Acc]; cert_chain(Role, IssuerCert, IssuerKey, Opts, [Key | Keys], N, Acc) -> + CAOpts = list_to_atom(atom_to_list(Role) ++ "_ca_" ++ integer_to_list(N)), Cert = cert(Role, public_key:pkix_decode_cert(IssuerCert, otp), IssuerKey, Key, "webadmin", - " Intermidiate CA " ++ integer_to_list(N), Opts), + " Intermidiate CA " ++ integer_to_list(N), Opts, CAOpts, ca), cert_chain(Role, Cert, Key, Opts, Keys, N+1, [{IssuerCert, IssuerKey} | Acc]). -cert(Role, #'OTPCertificate'{tbsCertificate = #'OTPTBSCertificate'{subject = Issuer}}, - PrivKey, Key, Contact, Name, Opts) -> +cert(Role, #'OTPCertificate'{tbsCertificate = #'OTPTBSCertificate'{subject = Issuer, + serialNumber = SNr + }}, + PrivKey, Key, Contact, Name, Opts, CertOptsName, Type) -> + CertOpts = proplists:get_value(CertOptsName, Opts, []), TBS = cert_template(), OTPTBS = TBS#'OTPTBSCertificate'{ signature = sign_algorithm(PrivKey, Opts), issuer = Issuer, - validity = validity(Opts), + validity = validity(CertOpts), subject = subject(Contact, atom_to_list(Role) ++ Name), subjectPublicKeyInfo = public_key(Key), - extensions = extensions(Opts) + extensions = extensions(Type, + add_default_extensions([{auth_key_id, {auth_key_oid(Role), Issuer, SNr}}], + CertOpts)) }, public_key:pkix_sign(OTPTBS, PrivKey). @@ -319,3 +336,8 @@ default_key_gen() -> [{namedCurve, hd(tls_v1:ecc_curves(0))}, {namedCurve, hd(tls_v1:ecc_curves(0))}] end. + +auth_key_oid(server) -> + ?'id-kp-serverAuth'; +auth_key_oid(client) -> + ?'id-kp-clientAuth'. diff --git a/lib/stdlib/doc/src/erl_tar.xml b/lib/stdlib/doc/src/erl_tar.xml index f28d8b425b..fab7c832d5 100644 --- a/lib/stdlib/doc/src/erl_tar.xml +++ b/lib/stdlib/doc/src/erl_tar.xml @@ -292,6 +292,10 @@ <c>Fd</c> is assumed to be a file descriptor returned from function <c>file:open/2</c>.</p> <p>Otherwise, <c>Name</c> is to be a filename.</p> + <note><p>Leading slashes in tar member names will be removed before + writing the file. That is, absolute paths will be turned into + relative paths. There will be an info message written to the error + logger when paths are changed in this way.</p></note> </desc> </func> diff --git a/lib/stdlib/doc/src/re.xml b/lib/stdlib/doc/src/re.xml index aadb5beaba..078ca0e38c 100644 --- a/lib/stdlib/doc/src/re.xml +++ b/lib/stdlib/doc/src/re.xml @@ -5,7 +5,7 @@ <header> <copyright> <year>2007</year> - <year>2016</year> + <year>2017</year> <holder>Ericsson AB, All Rights Reserved</holder> </copyright> <legalnotice> @@ -79,6 +79,14 @@ <funcs> <func> + <name name="version" arity="0"/> + <fsummary>Gives the PCRE version of the system in a string format</fsummary> + <desc> + <p>The return of this function is a string with the PCRE version of the system that was used in the Erlang/OTP compilation.</p> + </desc> + </func> + + <func> <name name="compile" arity="1"/> <fsummary>Compile a regular expression into a match program</fsummary> <desc> diff --git a/lib/stdlib/src/erl_lint.erl b/lib/stdlib/src/erl_lint.erl index 0789f5dfb7..78b7a0e751 100644 --- a/lib/stdlib/src/erl_lint.erl +++ b/lib/stdlib/src/erl_lint.erl @@ -404,6 +404,10 @@ format_error({not_exported_opaque, {TypeName, Arity}}) -> format_error({underspecified_opaque, {TypeName, Arity}}) -> io_lib:format("opaque type ~w~s is underspecified and therefore meaningless", [TypeName, gen_type_paren(Arity)]); +format_error({bad_dialyzer_attribute,Term}) -> + io_lib:format("badly formed dialyzer attribute: ~w", [Term]); +format_error({bad_dialyzer_option,Term}) -> + io_lib:format("unknown dialyzer warning option: ~w", [Term]); %% --- obsolete? unused? --- format_error({format_error, {Fmt, Args}}) -> io_lib:format(Fmt, Args). @@ -796,8 +800,7 @@ attribute_state(Form, St) -> %% State' %% Allow for record, type and opaque type definitions and spec %% declarations to be intersperced within function definitions. -%% Dialyzer attributes are also allowed everywhere, but are not -%% checked at all. +%% Dialyzer attributes are also allowed everywhere. function_state({attribute,L,record,{Name,Fields}}, St) -> record_def(L, Name, Fields, St); @@ -883,7 +886,8 @@ post_traversal_check(Forms, St0) -> StD = check_on_load(StC), StE = check_unused_records(Forms, StD), StF = check_local_opaque_types(StE), - check_callback_information(StF). + StG = check_dialyzer_attribute(Forms, StF), + check_callback_information(StG). %% check_behaviour(State0) -> State %% Check that the behaviour attribute is valid. @@ -3116,6 +3120,59 @@ check_local_opaque_types(St) -> end, dict:fold(FoldFun, St, Ts). +check_dialyzer_attribute(Forms, St0) -> + Vals = [{L,V} || + {attribute,L,dialyzer,Val} <- Forms, + V0 <- lists:flatten([Val]), + V <- case V0 of + {O,F} -> + [{A,B} || + A <- lists:flatten([O]), + B <- lists:flatten([F])]; + T -> [T] + end], + {Wellformed, Bad} = + lists:partition(fun ({_,{Option,FA}}) when is_atom(Option) -> + is_fa(FA); + ({_,Option}) when is_atom(Option) -> true; + (_) -> false + end, Vals), + St1 = foldl(fun ({L,Term}, St) -> + add_error(L, {bad_dialyzer_attribute,Term}, St) + end, St0, Bad), + DefFunctions = (gb_sets:to_list(St0#lint.defined) -- pseudolocals()), + Fun = fun ({L,{Option,FA}}, St) -> + case is_function_dialyzer_option(Option) of + true -> + case lists:member(FA, DefFunctions) of + true -> St; + false -> + add_error(L, {undefined_function,FA}, St) + end; + false -> + add_error(L, {bad_dialyzer_option,Option}, St) + end; + ({L,Option}, St) -> + case is_module_dialyzer_option(Option) of + true -> St; + false -> + add_error(L, {bad_dialyzer_option,Option}, St) + end + end, + foldl(Fun, St1, Wellformed). + +is_function_dialyzer_option(nowarn_function) -> true; +is_function_dialyzer_option(Option) -> + is_module_dialyzer_option(Option). + +is_module_dialyzer_option(Option) -> + lists:member(Option, + [no_return,no_unused,no_improper_lists,no_fun_app, + no_match,no_opaque,no_fail_call,no_contracts, + no_behaviours,no_undefined_callbacks,unmatched_returns, + error_handling,race_conditions,no_missing_calls, + specdiffs,overspecs,underspecs,unknown]). + %% icrt_clauses(Clauses, In, ImportVarTable, State) -> %% {UpdVt,State}. diff --git a/lib/stdlib/src/erl_tar.erl b/lib/stdlib/src/erl_tar.erl index a54df939bf..168ea4002c 100644 --- a/lib/stdlib/src/erl_tar.erl +++ b/lib/stdlib/src/erl_tar.erl @@ -69,6 +69,8 @@ format_error(invalid_gnu_1_0_sparsemap) -> "Invalid GNU sparse map (version 1.0)"; format_error({invalid_gnu_0_1_sparsemap, Format}) -> lists:flatten(io_lib:format("Invalid GNU sparse map (version ~s)", [Format])); +format_error(unsafe_path) -> + "The path points above the current working directory"; format_error({Name,Reason}) -> lists:flatten(io_lib:format("~ts: ~ts", [Name,format_error(Reason)])); format_error(Atom) when is_atom(Atom) -> @@ -120,26 +122,38 @@ do_extract(Handle, Opts) when is_list(Opts) -> extract1(eof, Reader, _, Acc) when is_list(Acc) -> {ok, {ok, lists:reverse(Acc)}, Reader}; +extract1(eof, Reader, _, leading_slash) -> + error_logger:info_msg("erl_tar: removed leading '/' from member names\n"), + {ok, ok, Reader}; extract1(eof, Reader, _, Acc) -> {ok, Acc, Reader}; -extract1(#tar_header{name=Name,size=Size}=Header, Reader, Opts, Acc) -> +extract1(#tar_header{name=Name,size=Size}=Header, Reader0, Opts, Acc0) -> case check_extract(Name, Opts) of true -> - case do_read(Reader, Size) of - {ok, Bin, Reader2} -> - case write_extracted_element(Header, Bin, Opts) of - ok -> - {ok, Acc, Reader2}; - {ok, NameBin} when is_list(Acc) -> - {ok, [NameBin | Acc], Reader2}; - {error, _} = Err -> - throw(Err) - end; + case do_read(Reader0, Size) of + {ok, Bin, Reader1} -> + Acc = extract2(Header, Bin, Opts, Acc0), + {ok, Acc, Reader1}; {error, _} = Err -> throw(Err) end; false -> - {ok, Acc, skip_file(Reader)} + {ok, Acc0, skip_file(Reader0)} + end. + +extract2(Header, Bin, Opts, Acc) -> + case write_extracted_element(Header, Bin, Opts) of + ok -> + case Header of + #tar_header{name="/"++_} -> + leading_slash; + #tar_header{} -> + Acc + end; + {ok, NameBin} when is_list(Acc) -> + [NameBin | Acc]; + {error, _} = Err -> + throw(Err) end. %% Checks if the file Name should be extracted. @@ -1052,14 +1066,11 @@ unpack_modern(Format, #header_v7{}=V7, Bin, #tar_header{}=Header0) safe_join_path([], Name) -> - strip_slashes(Name, both); + filename:join([Name]); safe_join_path(Prefix, []) -> - strip_slashes(Prefix, right); + filename:join([Prefix]); safe_join_path(Prefix, Name) -> - filename:join(strip_slashes(Prefix, right), strip_slashes(Name, both)). - -strip_slashes(Str, Direction) -> - string:strip(Str, Direction, $/). + filename:join(Prefix, Name). new_sparse_file_reader(Reader, Sparsemap, RealSize) -> true = validate_sparse_entries(Sparsemap, RealSize), @@ -1557,7 +1568,7 @@ write_extracted_element(#tar_header{name=Name,typeflag=Type}, ok end; write_extracted_element(#tar_header{name=Name0}=Header, Bin, Opts) -> - Name1 = filename:absname(Name0, Opts#read_opts.cwd), + Name1 = make_safe_path(Name0, Opts), Created = case typeflag(Header#tar_header.typeflag) of regular -> @@ -1585,6 +1596,16 @@ write_extracted_element(#tar_header{name=Name0}=Header, Bin, Opts) -> not_written -> ok end. +make_safe_path([$/|Path], Opts) -> + make_safe_path(Path, Opts); +make_safe_path(Path, #read_opts{cwd=Cwd}) -> + case filename:safe_relative_path(Path) of + unsafe -> + throw({error,{Path,unsafe_path}}); + SafePath -> + filename:absname(SafePath, Cwd) + end. + create_regular(Name, NameInArchive, Bin, Opts) -> case write_extracted_file(Name, Bin, Opts) of not_written -> diff --git a/lib/stdlib/src/re.erl b/lib/stdlib/src/re.erl index 52d3c35608..28aab7b590 100644 --- a/lib/stdlib/src/re.erl +++ b/lib/stdlib/src/re.erl @@ -33,7 +33,12 @@ %%% BIFs --export([compile/1, compile/2, run/2, run/3, inspect/2]). +-export([version/0, compile/1, compile/2, run/2, run/3, inspect/2]). + +-spec version() -> binary(). + +version() -> + erlang:nif_error(undef). -spec compile(Regexp) -> {ok, MP} | {error, ErrSpec} when Regexp :: iodata(), diff --git a/lib/stdlib/test/beam_lib_SUITE.erl b/lib/stdlib/test/beam_lib_SUITE.erl index 279e15f703..1baf7d0a94 100644 --- a/lib/stdlib/test/beam_lib_SUITE.erl +++ b/lib/stdlib/test/beam_lib_SUITE.erl @@ -240,7 +240,7 @@ do_error(BeamFile, ACopy) -> verify(missing_chunk, beam_lib:chunks(BF3, [imports])), BF4 = set_byte(ACopy, BeamFile, AbstractStart+10, 17), verify(invalid_chunk, beam_lib:chunks(BF4, [abstract_code])), - BF5 = set_byte(ACopy, BeamFile, AttributesStart+10, 17), + BF5 = set_byte(ACopy, BeamFile, AttributesStart+8, 17), verify(invalid_chunk, beam_lib:chunks(BF5, [attributes])), BF6 = set_byte(ACopy, BeamFile, 1, 17), @@ -251,7 +251,7 @@ do_error(BeamFile, ACopy) -> BF8 = set_byte(ACopy, BeamFile, 13, 17), verify(missing_chunk, beam_lib:chunks(BF8, ["AtU8"])), - BF9 = set_byte(ACopy, BeamFile, CompileInfoStart+10, 17), + BF9 = set_byte(ACopy, BeamFile, CompileInfoStart+8, 17), verify(invalid_chunk, beam_lib:chunks(BF9, [compile_info])). diff --git a/lib/stdlib/test/erl_lint_SUITE.erl b/lib/stdlib/test/erl_lint_SUITE.erl index fd7de65302..c469624fb4 100644 --- a/lib/stdlib/test/erl_lint_SUITE.erl +++ b/lib/stdlib/test/erl_lint_SUITE.erl @@ -64,8 +64,8 @@ predef/1, maps/1,maps_type/1,maps_parallel_match/1, otp_11851/1,otp_11879/1,otp_13230/1, - record_errors/1, otp_xxxxx/1, - non_latin1_module/1]). + record_errors/1, otp_11879_cont/1, + non_latin1_module/1, otp_14323/1]). suite() -> [{ct_hooks,[ts_install_cth]}, @@ -85,7 +85,7 @@ all() -> too_many_arguments, basic_errors, bin_syntax_errors, predef, maps, maps_type, maps_parallel_match, otp_11851, otp_11879, otp_13230, - record_errors, otp_xxxxx, non_latin1_module]. + record_errors, otp_11879_cont, non_latin1_module, otp_14323]. groups() -> [{unused_vars_warn, [], @@ -3875,7 +3875,7 @@ record_errors(Config) when is_list(Config) -> {3,erl_lint,{redefine_field,r,a}}],[]}}], run(Config, Ts). -otp_xxxxx(Config) -> +otp_11879_cont(Config) -> Ts = [{constraint1, <<"-export([t/1]). -spec t(X) -> X when is_subtype(integer()). @@ -3942,6 +3942,50 @@ do_non_latin1_module(Mod) -> ok. +%% OTP-14323: Check the dialyzer attribute. +otp_14323(Config) -> + Ts = [ + {otp_14323_1, + <<"-import(mod, [m/1]). + + -export([f/0, g/0, h/0]). + + -dialyzer({nowarn_function,module_info/0}). % undefined function + -dialyzer({nowarn_function,record_info/2}). % undefined function + -dialyzer({nowarn_function,m/1}). % undefined function + + -dialyzer(nowarn_function). % unknown option + -dialyzer(1). % badly formed + -dialyzer(malformed). % unkonwn option + -dialyzer({malformed,f/0}). % unkonwn option + -dialyzer({nowarn_function,a/1}). % undefined function + -dialyzer({nowarn_function,{a,-1}}). % badly formed + + -dialyzer([no_return, no_match]). + -dialyzer({nowarn_function, f/0}). + -dialyzer(no_improper_lists). + -dialyzer([{nowarn_function, [f/0]}, no_improper_lists]). + -dialyzer({no_improper_lists, g/0}). + -dialyzer({[no_return, no_match], [g/0, h/0]}). + + f() -> a. + g() -> b. + h() -> c.">>, + [], + {errors,[{5,erl_lint,{undefined_function,{module_info,0}}}, + {6,erl_lint,{undefined_function,{record_info,2}}}, + {7,erl_lint,{undefined_function,{m,1}}}, + {9,erl_lint,{bad_dialyzer_option,nowarn_function}}, + {10,erl_lint,{bad_dialyzer_attribute,1}}, + {11,erl_lint,{bad_dialyzer_option,malformed}}, + {12,erl_lint,{bad_dialyzer_option,malformed}}, + {13,erl_lint,{undefined_function,{a,1}}}, + {14,erl_lint,{bad_dialyzer_attribute, + {nowarn_function,{a,-1}}}}], + []}}], + [] = run(Config, Ts), + ok. + run(Config, Tests) -> F = fun({N,P,Ws,E}, BadL) -> case catch run_test(Config, P, Ws) of diff --git a/lib/stdlib/test/io_proto_SUITE.erl b/lib/stdlib/test/io_proto_SUITE.erl index db321d7490..4cc4e3292c 100644 --- a/lib/stdlib/test/io_proto_SUITE.erl +++ b/lib/stdlib/test/io_proto_SUITE.erl @@ -26,15 +26,14 @@ -export([init_per_testcase/2, end_per_testcase/2]). -export([setopts_getopts/1,unicode_options/1,unicode_options_gen/1, - binary_options/1, bc_with_r12/1, - bc_with_r12_gl/1, read_modes_gl/1,bc_with_r12_ogl/1, + binary_options/1, read_modes_gl/1, read_modes_ogl/1, broken_unicode/1,eof_on_pipe/1,unicode_prompt/1]). -export([io_server_proxy/1,start_io_server_proxy/0, proxy_getall/1, proxy_setnext/2, proxy_quit/1]). %% For spawn --export([toerl_server/3,hold_the_line/3,answering_machine1/3, +-export([toerl_server/3,answering_machine1/3, answering_machine2/3]). -export([uprompt/1]). @@ -79,8 +78,7 @@ suite() -> all() -> [setopts_getopts, unicode_options, unicode_options_gen, - binary_options, bc_with_r12, bc_with_r12_gl, - bc_with_r12_ogl, read_modes_gl, read_modes_ogl, + binary_options, read_modes_gl, read_modes_ogl, broken_unicode, eof_on_pipe, unicode_prompt]. groups() -> @@ -742,263 +740,7 @@ binary_options(Config) when is_list(Config) -> ],[],[],"-oldshell"), ok. -%% Test io protocol compatibility with R12 nodes. -bc_with_r12(Config) when is_list(Config) -> - case test_server:is_release_available("r12b") of - true -> bc_with_r12_1(Config); - false -> {skip,"No R12B found"} - end. - -bc_with_r12_1(Config) -> - PA = filename:dirname(code:which(?MODULE)), - Name1 = io_proto_r12_1, - N1 = list_to_atom(atom_to_list(Name1) ++ "@" ++ hostname()), - test_server:start_node(Name1, peer, [{args, "-pz \""++PA++"\""}, - {erl,[{release,"r12b"}]}]), - DataDir = proplists:get_value(data_dir,Config), - FileName1 = filename:join([DataDir,"testdata_latin1.dat"]), - TestDataLine1 = [229,228,246], - TestDataLine2 = [197,196,214], - SPid1 = rpc:call(N1,erlang,spawn,[?MODULE,hold_the_line,[self(),FileName1,[read]]]), - {ok,F1} = receive - {SPid1,Res1} -> - Res1 - after 5000 -> - exit(timeout) - end, - TestDataLine1 = chomp(io:get_line(F1,'')), - SPid1 ! die, - receive after 1000 -> ok end, - SPid2 = rpc:call(N1,erlang,spawn,[?MODULE,hold_the_line,[self(),FileName1,[read,binary]]]), - {ok,F2} = receive - {SPid2,Res2} -> - Res2 - after 5000 -> - exit(timeout) - end, - TestDataLine1BinUtf = unicode:characters_to_binary(TestDataLine1), - TestDataLine1BinLatin = list_to_binary(TestDataLine1), - TestDataLine2BinUtf = unicode:characters_to_binary(TestDataLine2), - TestDataLine2BinLatin = list_to_binary(TestDataLine2), - TestDataLine1BinUtf = chomp(io:get_line(F2,'')), - TestDataLine2BinUtf = chomp(io:get_line(F2,'')), - %%io:format(standard_error,"Exec:~s\r\n",[rpc:call(N1,os,find_executable,["erl"])]), - %%io:format(standard_error,"Io:~s\r\n",[rpc:call(N1,code,which,[io])]), - %%io:format(standard_error,"File_io_server:~s\r\n",[rpc:call(N1,code,which,[file_io_server])]), - file:position(F2,0), - TestDataLine1BinLatin = chomp(rpc:call(N1,io,get_line,[F2,''])), - TestDataLine2BinUtf = chomp(io:get_line(F2,'')), - file:position(F2,0), - TestDataLine1BinUtf = chomp(io:get_line(F2,'')), - TestDataLine2BinLatin = chomp(rpc:call(N1,io,get_line,[F2,''])), - eof = chomp(rpc:call(N1,io,get_line,[F2,''])), - file:position(F2,0), - TestDataLine1BinLatin = rpc:call(N1,io,get_chars,[F2,'',3]), - io:get_chars(F2,'',1), - TestDataLine2BinLatin = chomp(rpc:call(N1,io,get_line,[F2,''])), - file:position(F2,0), - {ok,[TestDataLine1]} = io:fread(F2,'',"~s"), - {ok,[TestDataLine2]} = rpc:call(N1,io,fread,[F2,'',"~s"]), - - DataLen1 = length(TestDataLine1), - DataLen2 = length(TestDataLine2), - file:position(F2,0), - {ok,TestDataLine1BinLatin} = file:read(F2,DataLen1), - {ok,_} = file:read(F2,1), - {ok,TestDataLine2BinLatin} = rpc:call(N1,file,read,[F2,DataLen2]), - {ok,_} = file:read(F2,1), - eof = rpc:call(N1,file,read,[F2,1]), - %% As r12 has a bug when setting options with setopts, we need - %% to reopen the file... - SPid2 ! die, - receive after 1000 -> ok end, - SPid3 = rpc:call(N1,erlang,spawn,[?MODULE,hold_the_line,[self(),FileName1,[read]]]), - {ok,F3} = receive - {SPid3,Res3} -> - Res3 - after 5000 -> - exit(timeout) - end, - - file:position(F3,0), - {ok,[TestDataLine1]} = io:fread(F3,'',"~s"), - {ok,[TestDataLine2]} = rpc:call(N1,io,fread,[F3,'',"~s"]), - - - file:position(F3,0), - {ok,TestDataLine1} = file:read(F3,DataLen1), - {ok,_} = file:read(F3,1), - {ok,TestDataLine2} = rpc:call(N1,file,read,[F3,DataLen2]), - {ok,_} = file:read(F3,1), - eof = rpc:call(N1,file,read,[F3,1]), - - - %% So, lets do it all again, but the other way around - {ok,F4} = file:open(FileName1,[read]), - TestDataLine1 = chomp(io:get_line(F4,'')), - file:position(F4,0), - io:setopts(F4,[binary]), - TestDataLine1BinUtf = chomp(io:get_line(F4,'')), - TestDataLine2BinUtf = chomp(io:get_line(F4,'')), - file:position(F4,0), - TestDataLine1BinUtf = chomp(io:get_line(F4,'')), - TestDataLine2BinUtf = chomp(io:get_line(F4,'')), - file:position(F4,0), - TestDataLine1BinUtf = chomp(io:get_line(F4,'')), - TestDataLine2BinLatin = chomp(rpc:call(N1,io,get_line,[F4,''])), - file:position(F4,0), - TestDataLine1BinLatin = chomp(rpc:call(N1,io,get_line,[F4,''])), - TestDataLine2BinUtf = chomp(io:get_line(F4,'')), - eof = chomp(rpc:call(N1,io,get_line,[F4,''])), - file:position(F4,0), - TestDataLine1BinLatin = rpc:call(N1,io,get_chars,[F4,'',3]), - io:get_chars(F4,'',1), - TestDataLine2BinLatin = chomp(rpc:call(N1,io,get_line,[F4,''])), - file:position(F4,0), - {ok,[TestDataLine1]} = io:fread(F4,'',"~s"), - {ok,[TestDataLine2]} = rpc:call(N1,io,fread,[F4,'',"~s"]), - file:position(F4,0), - {ok,TestDataLine1BinLatin} = file:read(F4,DataLen1), - {ok,_} = file:read(F4,1), - {ok,TestDataLine2BinLatin} = rpc:call(N1,file,read,[F4,DataLen2]), - {ok,_} = file:read(F4,1), - eof = rpc:call(N1,file,read,[F4,1]), - io:setopts(F4,[list]), - - file:position(F4,0), - {ok,[TestDataLine1]} = io:fread(F4,'',"~s"), - {ok,[TestDataLine2]} = rpc:call(N1,io,fread,[F4,'',"~s"]), - - - file:position(F4,0), - {ok,TestDataLine1} = file:read(F4,DataLen1), - {ok,_} = file:read(F4,1), - {ok,TestDataLine2} = rpc:call(N1,file,read,[F4,DataLen2]), - {ok,_} = file:read(F4,1), - eof = rpc:call(N1,file,read,[F4,1]), - - file:close(F4), - test_server:stop_node(N1), - ok. - -hold_the_line(Parent,Filename,Options) -> - Parent ! {self(), file:open(Filename,Options)}, - receive - die -> - ok - end. - - -%% Test io protocol compatibility with R12 nodes (terminals). -bc_with_r12_gl(Config) when is_list(Config) -> - case test_server:is_release_available("r12b") of - true -> - case get_progs() of - {error,Reason} -> - {skip, Reason}; - _ -> - bc_with_r12_gl_1(Config,answering_machine1) - end; - false -> - {skip,"No R12B found"} - end. - -%% Test io protocol compatibility with R12 nodes (oldshell). -bc_with_r12_ogl(Config) when is_list(Config) -> - case test_server:is_release_available("r12b") of - true -> - case get_progs() of - {error,Reason} -> - {skip, Reason}; - _ -> - bc_with_r12_gl_1(Config,answering_machine2) - end; - false -> - {skip,"No R12B found"} - end. - -bc_with_r12_gl_1(_Config,Machine) -> - PA = filename:dirname(code:which(?MODULE)), - Name1 = io_proto_r12_gl_1, - N1 = list_to_atom(atom_to_list(Name1) ++ "@" ++ hostname()), - test_server:start_node(Name1, peer, [{args, "-pz \""++PA++"\""}, - {erl,[{release,"r12b"}]}]), - TestDataLine1 = [229,228,246], - TestDataLine1BinUtf = unicode:characters_to_binary(TestDataLine1), - TestDataLine1BinLatin = list_to_binary(TestDataLine1), - - {ok,N2List} = create_nodename(), - MyNodeList = atom2list(node()), - register(io_proto_suite,self()), - AM1 = spawn(?MODULE,Machine, - [MyNodeList, "io_proto_suite", N2List]), - - GL = receive X when is_pid(X) -> X end, - %% get_line - "Hej\n" = rpc:call(N1,io,get_line,[GL,"Prompt\n"]), - io:setopts(GL,[binary]), - io:format(GL,"Okej~n",[]), - <<"Hej\n">> = rpc:call(N1,io,get_line,[GL,"Prompt\n"]), - io:setopts(GL,[{encoding,latin1}]), - io:format(GL,"Okej~n",[]), - TestDataLine1BinLatin = chomp(rpc:call(N1,io,get_line,[GL,"Prompt\n"])), - io:format(GL,"Okej~n",[]), - TestDataLine1BinUtf = chomp(io:get_line(GL,"Prompt\n")), - io:setopts(GL,[{encoding,unicode}]), - - io:format(GL,"Okej~n",[]), - TestDataLine1BinLatin = chomp(rpc:call(N1,io,get_line,[GL,"Prompt\n"])), - io:format(GL,"Okej~n",[]), - TestDataLine1BinUtf = chomp(io:get_line(GL,"Prompt\n")), - io:setopts(GL,[list]), - io:format(GL,"Okej~n",[]), - - %%get_chars - "Hej" = rpc:call(N1,io,get_chars,[GL,"Prompt\n",3]), - io:setopts(GL,[binary]), - io:format(GL,"Okej~n",[]), - <<"Hej">> = rpc:call(N1,io,get_chars,[GL,"Prompt\n",3]), - io:setopts(GL,[{encoding,latin1}]), - io:format(GL,"Okej~n",[]), - TestDataLine1BinLatin = rpc:call(N1,io,get_chars,[GL,"Prompt\n",3]), - io:format(GL,"Okej~n",[]), - TestDataLine1BinUtf = io:get_chars(GL,"Prompt\n",3), - io:setopts(GL,[{encoding,unicode}]), - - io:format(GL,"Okej~n",[]), - TestDataLine1BinLatin = rpc:call(N1,io,get_chars,[GL,"Prompt\n",3]), - io:format(GL,"Okej~n",[]), - TestDataLine1BinUtf = io:get_chars(GL,"Prompt\n",3), - io:setopts(GL,[list]), - io:format(GL,"Okej~n",[]), - %%fread - {ok,["Hej"]} = rpc:call(N1,io,fread,[GL,"Prompt\n","~s"]), - io:setopts(GL,[binary]), - io:format(GL,"Okej~n",[]), - {ok,["Hej"]} = rpc:call(N1,io,fread,[GL,"Prompt\n","~s"]), - io:setopts(GL,[{encoding,latin1}]), - io:format(GL,"Okej~n",[]), - {ok,[TestDataLine1]} = rpc:call(N1,io,fread,[GL,"Prompt\n","~s"]), - io:format(GL,"Okej~n",[]), - {ok,[TestDataLine1]} = io:fread(GL,"Prompt\n","~s"), - io:setopts(GL,[{encoding,unicode}]), - io:format(GL,"Okej~n",[]), - {ok,[TestDataLine1]} = rpc:call(N1,io,fread,[GL,"Prompt\n","~s"]), - io:format(GL,"Okej~n",[]), - {ok,[TestDataLine1]} = io:fread(GL,"Prompt\n","~s"), - io:setopts(GL,[list]), - io:format(GL,"Okej~n",[]), - - - receive - {AM1,done} -> - ok - after 5000 -> - exit(timeout) - end, - test_server:stop_node(N1), - ok. answering_machine1(OthNode,OthReg,Me) -> @@ -1900,13 +1642,6 @@ convert(Data, latin1, binary) -> {error, {cannot_convert, unicode, latin1}} end. -hostname() -> - from($@, atom_to_list(node())). - -from(H, [H | T]) -> T; -from(H, [_ | T]) -> from(H, T); -from(_, []) -> []. - atom2list(A) -> lists:flatten(io_lib:format("~w", [A])). diff --git a/lib/stdlib/test/re_SUITE.erl b/lib/stdlib/test/re_SUITE.erl index a76ded5f60..71f86e32e5 100644 --- a/lib/stdlib/test/re_SUITE.erl +++ b/lib/stdlib/test/re_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2016. All Rights Reserved. +%% Copyright Ericsson AB 2008-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ run_options/1,combined_options/1,replace_autogen/1, global_capture/1,replace_input_types/1,replace_return/1, split_autogen/1,split_options/1,split_specials/1, - error_handling/1,pcre_cve_2008_2371/1, + error_handling/1,pcre_cve_2008_2371/1,re_version/1, pcre_compile_workspace_overflow/1,re_infinite_loop/1, re_backwards_accented/1,opt_dupnames/1,opt_all_names/1,inspect/1, opt_no_start_optimize/1,opt_never_utf/1,opt_ucp/1, @@ -45,7 +45,7 @@ all() -> pcre_compile_workspace_overflow, re_infinite_loop, re_backwards_accented, opt_dupnames, opt_all_names, inspect, opt_no_start_optimize,opt_never_utf,opt_ucp, - match_limit, sub_binaries]. + match_limit, sub_binaries, re_version]. groups() -> []. @@ -194,6 +194,14 @@ run_options(Config) when is_list(Config) -> +%% Test the version is retorned correctly +re_version(_Config) -> + Version = re:version(), + {match,[Version]} = re:run(Version,"^[0-9]\\.[0-9]{2} 20[0-9]{2}-[0-9]{2}-[0-9]{2}",[{capture,all,binary}]), + ok. + + + %% Test compile options given directly to run. combined_options(Config) when is_list(Config) -> ok = crtest("ABDabcdABCD","abcd",[],true,{match,[{3,4}]}), diff --git a/lib/stdlib/test/tar_SUITE.erl b/lib/stdlib/test/tar_SUITE.erl index 2e1ae7bcff..76fa760fa8 100644 --- a/lib/stdlib/test/tar_SUITE.erl +++ b/lib/stdlib/test/tar_SUITE.erl @@ -27,7 +27,7 @@ extract_from_binary_compressed/1, extract_filtered/1, extract_from_open_file/1, symlinks/1, open_add_close/1, cooked_compressed/1, memory/1,unicode/1,read_other_implementations/1, - sparse/1, init/1]). + sparse/1, init/1, leading_slash/1, dotdot/1]). -include_lib("common_test/include/ct.hrl"). -include_lib("kernel/include/file.hrl"). @@ -41,7 +41,7 @@ all() -> extract_filtered, symlinks, open_add_close, cooked_compressed, memory, unicode, read_other_implementations, - sparse,init]. + sparse,init,leading_slash,dotdot]. groups() -> []. @@ -920,6 +920,39 @@ unicode_create_files() -> [] end]. +leading_slash(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + Dir = filename:join(PrivDir, ?FUNCTION_NAME), + TarFile = filename:join(Dir, "leading_slash.tar"), + ok = filelib:ensure_dir(TarFile), + {ok,Fd} = erl_tar:open(TarFile, [write]), + TarMemberName = "e/d/c/b/a_member", + TarMemberNameAbs = "/" ++ TarMemberName, + Contents = <<"contents\n">>, + ok = erl_tar:add(Fd, Contents, TarMemberNameAbs, [verbose]), + ok = erl_tar:close(Fd), + + ok = erl_tar:extract(TarFile, [{cwd,Dir}]), + + {ok,Contents} = file:read_file(filename:join(Dir, TarMemberName)), + ok. + +dotdot(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + Dir = filename:join(PrivDir, ?FUNCTION_NAME), + ok = file:make_dir(Dir), + Tar = filename:join(Dir, "dotdot.tar"), + {ok,Fd} = erl_tar:open(Tar, [write]), + BeamFile = code:which(?MODULE), + ok = erl_tar:add(Fd, BeamFile, "a/./../../some_file", []), + ok = erl_tar:close(Fd), + + {error,{_,unsafe_path=Error}} = erl_tar:extract(Tar, [{cwd,Dir}]), + false = filelib:is_regular(filename:join(PrivDir, "some_file")), + io:format("~s\n", [erl_tar:format_error(Error)]), + + ok. + %% Delete the given list of files. delete_files([]) -> ok; delete_files([Item|Rest]) -> diff --git a/lib/tools/src/lcnt.erl b/lib/tools/src/lcnt.erl index 23d66b084e..864d62f6c5 100644 --- a/lib/tools/src/lcnt.erl +++ b/lib/tools/src/lcnt.erl @@ -698,19 +698,47 @@ stats2record([{{File,Line},{Tries,Colls,{S,Ns,N}}}|Stats]) -> nt = N} | stats2record(Stats)]; stats2record([]) -> []. + clean_id_creation(Id) when is_pid(Id) -> Bin = term_to_binary(Id), - <<H:3/binary, L:16, Node:L/binary, Ids:8/binary, _Creation/binary>> = Bin, - Bin2 = list_to_binary([H, bytes16(L), Node, Ids, 0]), + <<H:3/binary, Rest/binary>> = Bin, + <<131, PidTag, AtomTag>> = H, + LL = atomlen_bits(AtomTag), + CL = creation_bits(PidTag), + <<L:LL, Node:L/binary, Ids:8/binary, _Creation/binary>> = Rest, + Bin2 = list_to_binary([H, <<L:LL>>, Node, Ids, <<0:CL>>]), binary_to_term(Bin2); clean_id_creation(Id) when is_port(Id) -> Bin = term_to_binary(Id), - <<H:3/binary, L:16, Node:L/binary, Ids:4/binary, _Creation/binary>> = Bin, - Bin2 = list_to_binary([H, bytes16(L), Node, Ids, 0]), + <<H:3/binary, Rest/binary>> = Bin, + <<131, PortTag, AtomTag>> = H, + LL = atomlen_bits(AtomTag), + CL = creation_bits(PortTag), + <<L:LL, Node:L/binary, Ids:4/binary, _Creation/binary>> = Rest, + Bin2 = list_to_binary([H, <<L:LL>>, Node, Ids, <<0:CL>>]), binary_to_term(Bin2); clean_id_creation(Id) -> Id. +-define(PID_EXT, $g). +-define(NEW_PID_EXT, $X). +-define(PORT_EXT, $f). +-define(NEW_PORT_EXT, $Y). +-define(ATOM_EXT, $d). +-define(SMALL_ATOM_EXT, $s). +-define(ATOM_UTF8_EXT, $v). +-define(SMALL_ATOM_UTF8_EXT, $w). + +atomlen_bits(?ATOM_EXT) -> 16; +atomlen_bits(?SMALL_ATOM_EXT) -> 8; +atomlen_bits(?ATOM_UTF8_EXT) -> 16; +atomlen_bits(?SMALL_ATOM_UTF8_EXT) -> 8. + +creation_bits(?PID_EXT) -> 8; +creation_bits(?NEW_PID_EXT) -> 32; +creation_bits(?PORT_EXT) -> 8; +creation_bits(?NEW_PORT_EXT) -> 32. + %% serializer state_default(Field) -> proplists:get_value(Field, state2list(#state{})). @@ -936,12 +964,24 @@ strings([S|Ss], Out) -> strings(Ss, Out ++ term2string("~ts", [S])). term2string({M,F,A}) when is_atom(M), is_atom(F), is_integer(A) -> term2string("~p:~p/~p", [M,F,A]); term2string(Term) when is_port(Term) -> % ex #Port<6442.816> - <<_:3/binary, L:16, Node:L/binary, Ids:32, _/binary>> = term_to_binary(Term), - term2string("#Port<~s.~w>", [Node, Ids]); + case term_to_binary(Term) of + <<_:2/binary, ?SMALL_ATOM_UTF8_EXT, L:8, Node:L/binary, Ids:32, _/binary>> -> + term2string("#Port<~ts.~w>", [Node, Ids]); + <<_:2/binary, ?ATOM_UTF8_EXT, L:16, Node:L/binary, Ids:32, _/binary>> -> + term2string("#Port<~ts.~w>", [Node, Ids]); + <<_:2/binary, ?ATOM_EXT, L:16, Node:L/binary, Ids:32, _/binary>> -> + term2string("#Port<~s.~w>", [Node, Ids]) + end; term2string(Term) when is_pid(Term) -> % ex <0.80.0> - <<_:3/binary, L:16, Node:L/binary, Ids:32, Serial:32, _/binary>> = term_to_binary(Term), - term2string("<~s.~w.~w>", [Node, Ids, Serial]); + case term_to_binary(Term) of + <<_:2/binary, ?SMALL_ATOM_UTF8_EXT, L:8, Node:L/binary, Ids:32, Serial:32, _/binary>> -> + term2string("<~ts.~w.~w>", [Node, Ids, Serial]); + <<_:2/binary, ?ATOM_UTF8_EXT, L:16, Node:L/binary, Ids:32, Serial:32, _/binary>> -> + term2string("<~ts.~w.~w>", [Node, Ids, Serial]); + <<_:2/binary, ?ATOM_EXT, L:16, Node:L/binary, Ids:32, Serial:32, _/binary>> -> + term2string("<~s.~w.~w>", [Node, Ids, Serial]) + end; term2string(Term) -> term2string("~w", [Term]). term2string(Format, Terms) -> lists:flatten(io_lib:format(Format, Terms)). diff --git a/lib/tools/src/make.erl b/lib/tools/src/make.erl index 60695febb4..0363dee02d 100644 --- a/lib/tools/src/make.erl +++ b/lib/tools/src/make.erl @@ -290,11 +290,12 @@ coerce_2_list(X) when is_atom(X) -> coerce_2_list(X) -> X. -%%% If you an include file is found with a modification +%%% If an include file is found with a modification %%% time larger than the modification time of the object %%% file, return true. Otherwise return false. check_includes(File, IncludePath, ObjMTime) -> - Path = [filename:dirname(File)|IncludePath], + {ok,Cwd} = file:get_cwd(), + Path = [Cwd,filename:dirname(File)|IncludePath], case epp:open(File, Path, []) of {ok, Epp} -> check_includes2(Epp, File, ObjMTime); diff --git a/lib/tools/test/make_SUITE.erl b/lib/tools/test/make_SUITE.erl index 2a94ead329..2db5c6844a 100644 --- a/lib/tools/test/make_SUITE.erl +++ b/lib/tools/test/make_SUITE.erl @@ -19,11 +19,7 @@ %% -module(make_SUITE). --export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, - init_per_group/2,end_per_group/2, make_all/1, make_files/1, emake_opts/1]). --export([otp_6057_init/1, - otp_6057_a/1, otp_6057_b/1, otp_6057_c/1, - otp_6057_end/1]). +-compile(export_all). -include_lib("common_test/include/ct.hrl"). @@ -40,7 +36,8 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [make_all, make_files, emake_opts, {group, otp_6057}]. + [make_all, make_files, recompile_on_changed_include, + emake_opts, {group, otp_6057}]. groups() -> [{otp_6057,[],[otp_6057_a, otp_6057_b, @@ -86,6 +83,41 @@ make_files(Config) when is_list(Config) -> ensure_no_messages(), ok. +recompile_on_changed_include(Config) -> + Current = prepare_data_dir(Config), + + Files = [test_incl1,"incl_src/test_incl2"], + up_to_date = make:files(Files), + ok = ensure_exists([test_incl1,test_incl2]), + + {ok, FileInfo11} = file:read_file_info("test_incl1.beam"), + Date11 = FileInfo11#file_info.mtime, + {ok, FileInfo21} = file:read_file_info("test_incl2.beam"), + Date21 = FileInfo21#file_info.mtime, + timer:sleep(2000), + + %% Touch the include file + {ok,Bin} = file:read_file("test_incl.hrl"), + ok = file:delete("test_incl.hrl"), + ok = file:write_file("test_incl.hrl",Bin), + + up_to_date = make:files(Files), + + {ok, FileInfo12} = file:read_file_info("test_incl1.beam"), + case FileInfo12#file_info.mtime of + Date11 -> ct:fail({"file not recompiled", "test_incl1.beam"}); + _Date12 -> ok + end, + {ok, FileInfo22} = file:read_file_info("test_incl2.beam"), + case FileInfo22#file_info.mtime of + Date21 -> ct:fail({"file not recompiled", "test_incl2.beam"}); + _Date22 -> ok + end, + + file:set_cwd(Current), + ensure_no_messages(), + ok. + emake_opts(Config) when is_list(Config) -> Current = prepare_data_dir(Config), diff --git a/lib/tools/test/make_SUITE_data/incl_src/test_incl2.erl b/lib/tools/test/make_SUITE_data/incl_src/test_incl2.erl new file mode 100644 index 0000000000..d0db98c19a --- /dev/null +++ b/lib/tools/test/make_SUITE_data/incl_src/test_incl2.erl @@ -0,0 +1,9 @@ +-module(test_incl2). +-compile(export_all). +-include("test_incl.hrl"). + +f1() -> + ?d. + +f2() -> + true. diff --git a/lib/tools/test/make_SUITE_data/test_incl.hrl b/lib/tools/test/make_SUITE_data/test_incl.hrl new file mode 100644 index 0000000000..3a1bcfadd0 --- /dev/null +++ b/lib/tools/test/make_SUITE_data/test_incl.hrl @@ -0,0 +1 @@ +-define(d,"defined"). diff --git a/lib/tools/test/make_SUITE_data/test_incl1.erl b/lib/tools/test/make_SUITE_data/test_incl1.erl new file mode 100644 index 0000000000..4a1dd0e73d --- /dev/null +++ b/lib/tools/test/make_SUITE_data/test_incl1.erl @@ -0,0 +1,9 @@ +-module(test_incl1). +-compile(export_all). +-include("test_incl.hrl"). + +f1() -> + ?d. + +f2() -> + true. diff --git a/lib/tools/test/xref_SUITE.erl b/lib/tools/test/xref_SUITE.erl index f308ea1204..b8e7113595 100644 --- a/lib/tools/test/xref_SUITE.erl +++ b/lib/tools/test/xref_SUITE.erl @@ -89,11 +89,11 @@ init_per_suite(Conf) when is_list(Conf) -> DataDir = ?datadir, PrivDir = ?privdir, CopyDir = fname(PrivDir, "datacopy"), + ok = file:make_dir(CopyDir), TarFile = fname(PrivDir, "datacopy.tgz"), - {ok, Tar} = erl_tar:open(TarFile, [write, compressed]), - ok = erl_tar:add(Tar, DataDir, CopyDir, [compressed]), - ok = erl_tar:close(Tar), - ok = erl_tar:extract(TarFile, [compressed]), + ok = file:set_cwd(DataDir), + ok = erl_tar:create(TarFile, ["."], [compressed]), + ok = erl_tar:extract(TarFile, [compressed, {cwd,CopyDir}]), ok = file:delete(TarFile), [{copy_dir, CopyDir}|Conf]. |