diff options
31 files changed, 778 insertions, 104 deletions
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index ff03151619..1121a450cd 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -1084,7 +1084,7 @@ static byte* convert_environment(Process* p, Eterm env) goto done; } - if ((size = erts_native_filename_need(all,encoding)) < 0) { + if ((size = erts_native_filename_need(all, encoding, 1)) < 0) { goto done; } diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index 2d1d1443a7..efd2ca3db2 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -1988,7 +1988,7 @@ char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, size_t statbu is_list(name) || (allow_empty && is_nil(name))) { Sint need; - if ((need = erts_native_filename_need(name,encoding)) < 0) { + if ((need = erts_native_filename_need(name, encoding, 0)) < 0) { return NULL; } if (encoding == ERL_FILENAME_WIN_WCHAR) { @@ -2152,12 +2152,13 @@ Eterm erts_convert_native_to_filename(Process *p, byte *bytes) } -Sint erts_native_filename_need(Eterm ioterm, int encoding) +Sint erts_native_filename_need(Eterm ioterm, int encoding, int allow_null) { Eterm *objp; Eterm obj; DECLARE_ESTACK(stack); Sint need = 0; + int seen_null = 0; if (is_atom(ioterm)) { Atom* ap; @@ -2194,6 +2195,24 @@ Sint erts_native_filename_need(Eterm ioterm, int encoding) default: need = -1; } + if (!allow_null) { + /* + * Do not allow null in + * the middle of filenames + */ + if (need > 0) { + byte *name = ap->name; + int len = ap->len; + for (i = 0; i < len; i++) { + if (name[i] == 0) + seen_null = 1; + else if (seen_null) { + need = -1; + break; + } + } + } + } DESTROY_ESTACK(stack); return need; } @@ -2224,6 +2243,18 @@ L_Again: /* Restart with sublist, old listend was pushed on stack */ if (is_small(obj)) { /* Always small */ for(;;) { Uint x = unsigned_val(obj); + if (!allow_null) { + /* + * Do not allow null in + * the middle of filenames + */ + if (x == 0) + seen_null = 1; + else if (seen_null) { + DESTROY_ESTACK(stack); + return ((Sint) -1); + } + } switch (encoding) { case ERL_FILENAME_LATIN1: if (x > 255) { @@ -2515,6 +2546,7 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1) BIF_ERROR(BIF_P,BADARG); } if (is_binary(BIF_ARG_1)) { + int seen_null = 0; byte *temp_alloc = NULL; byte *bytes; byte *err_pos; @@ -2524,10 +2556,18 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1) size = binary_size(BIF_ARG_1); bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc); if (encoding != ERL_FILENAME_WIN_WCHAR) { + Uint i; /*Add 0 termination only*/ bin_term = new_binary(BIF_P, NULL, size+1); bin_p = binary_bytes(bin_term); - memcpy(bin_p,bytes,size); + for (i = 0; i < size; i++) { + /* Don't allow null in the middle of filenames... */ + if (bytes[i] == 0) + seen_null = 1; + else if (seen_null) + goto bin_name_error; + bin_p[i] = bytes[i]; + } bin_p[size]=0; erts_free_aligned_binary_bytes(temp_alloc); BIF_RET(bin_term); @@ -2541,6 +2581,11 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1) bin_term = new_binary(BIF_P, 0, (size+1)*2); bin_p = binary_bytes(bin_term); while (size--) { + /* Don't allow null in the middle of filenames... */ + if (*bytes == 0) + seen_null = 1; + else if (seen_null) + goto bin_name_error; *bin_p++ = *bytes++; *bin_p++ = 0; } @@ -2558,11 +2603,14 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1) bin_p[num_chars*2+1] = 0; erts_free_aligned_binary_bytes(temp_alloc); BIF_RET(bin_term); + bin_name_error: + erts_free_aligned_binary_bytes(temp_alloc); + BIF_ERROR(BIF_P,BADARG); } /* binary */ - if ((need = erts_native_filename_need(BIF_ARG_1,encoding)) < 0) { - BIF_ERROR(BIF_P,BADARG); + if ((need = erts_native_filename_need(BIF_ARG_1, encoding, 0)) < 0) { + BIF_ERROR(BIF_P,BADARG); } if (encoding == ERL_FILENAME_WIN_WCHAR) { need += 2; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 182d3aa44e..ddf7f03265 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1253,7 +1253,7 @@ void erts_init_unicode(void); Sint erts_unicode_set_loop_limit(Sint limit); void erts_native_filename_put(Eterm ioterm, int encoding, byte *p) ; -Sint erts_native_filename_need(Eterm ioterm, int encoding); +Sint erts_native_filename_need(Eterm ioterm, int encoding, int allow_null); void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars); int erts_analyze_utf8(byte *source, Uint size, byte **err_pos, Uint *num_chars, int *left); diff --git a/erts/preloaded/src/erts.app.src b/erts/preloaded/src/erts.app.src index 7ab06164b4..beb29a7c89 100644 --- a/erts/preloaded/src/erts.app.src +++ b/erts/preloaded/src/erts.app.src @@ -37,7 +37,7 @@ {registered, []}, {applications, []}, {env, []}, - {runtime_dependencies, ["stdlib-3.0", "kernel-5.0", "sasl-3.0.1"]} + {runtime_dependencies, ["stdlib-3.4.3", "kernel-5.4.1", "sasl-3.0.1"]} ]}. %% vim: ft=erlang diff --git a/lib/compiler/src/beam_validator.erl b/lib/compiler/src/beam_validator.erl index 00901077d3..be8908dd6b 100644 --- a/lib/compiler/src/beam_validator.erl +++ b/lib/compiler/src/beam_validator.erl @@ -1430,13 +1430,13 @@ merge_types(bool, {atom,A}) -> merge_bool(A); merge_types({atom,A}, bool) -> merge_bool(A); -merge_types(#ms{id=Id1,valid=B0,slots=Slots}, - #ms{id=Id2,valid=B1,slots=Slots}) -> +merge_types(#ms{id=Id1,valid=B1,slots=Slots1}, + #ms{id=Id2,valid=B2,slots=Slots2}) -> Id = if Id1 =:= Id2 -> Id1; true -> make_ref() end, - #ms{id=Id,valid=B0 band B1,slots=Slots}; + #ms{id=Id,valid=B1 band B2,slots=min(Slots1, Slots2)}; merge_types(T1, T2) when T1 =/= T2 -> %% Too different. All we know is that the type is a 'term'. term. diff --git a/lib/compiler/test/bs_match_SUITE.erl b/lib/compiler/test/bs_match_SUITE.erl index 0ec05456ec..2fe8cd0cff 100644 --- a/lib/compiler/test/bs_match_SUITE.erl +++ b/lib/compiler/test/bs_match_SUITE.erl @@ -39,7 +39,7 @@ match_string_opt/1,select_on_integer/1, map_and_binary/1,unsafe_branch_caching/1, bad_literals/1,good_literals/1,constant_propagation/1, - parse_xml/1,get_payload/1]). + parse_xml/1,get_payload/1,num_slots_different/1]). -export([coverage_id/1,coverage_external_ignore/2]). @@ -71,7 +71,7 @@ groups() -> match_string_opt,select_on_integer, map_and_binary,unsafe_branch_caching, bad_literals,good_literals,constant_propagation,parse_xml, - get_payload]}]. + get_payload,num_slots_different]}]. init_per_suite(Config) -> @@ -1524,6 +1524,40 @@ do_get_payload(ExtHdr) -> <<_:13,_:35>> = ExtHdr#ext_header.ext_hdr_opts, ExtHdrOptions. +%% ERL-490 +num_slots_different(_Config) -> + Ts = [{<<"de">>, <<"default">>, <<"Remove">>, <<"a">>}, + {<<"de">>, <<"default">>, <<"Remove from list">>, <<"a">>}, + {<<"de">>, <<"default">>, <<"Remove from the list">>, <<"a">>}, + {<<"de">>, <<"default">>, <<"Results">>, <<"Ergebnisse">>}, + {<<"de">>, <<"default">>, <<"Reservatio">>, <<"a">>}, + {<<"de">>, <<"navigation">>, <<"Results">>, <<"Ergebnisse">>}, + {<<"de">>, <<"navigation">>, <<"Resources">>, <<"Ressourcen">>}], + _ = [{ok,Res} = lgettext(A, B, C) || {A,B,C,Res} <- Ts], + + {'EXIT',_} = (catch lgettext(<<"d">>, <<"default">>, <<"Remove">>)), + {'EXIT',_} = (catch lgettext("", <<"default">>, <<"Remove">>)), + {'EXIT',_} = (catch lgettext(<<"de">>, <<"def">>, <<"Remove">>)), + {'EXIT',_} = (catch lgettext(<<"de">>, <<"default">>, <<"Res">>)), + ok. + + +lgettext(<<"de">>, <<"default">>, <<"Remove">>) -> + {ok, <<"a">>}; +lgettext(<<"de">>, <<"default">>, <<"Remove from list">>) -> + {ok, <<"a">>}; +lgettext(<<"de">>, <<"default">>, <<"Remove from the list">>) -> + {ok, <<"a">>}; +lgettext(<<"de">>, <<"default">>, <<"Results">>) -> + {ok, <<"Ergebnisse">>}; +lgettext(<<"de">>, <<"default">>, <<"Reservatio">>) -> + {ok, <<"a">>}; +lgettext(<<"de">>, <<"navigation">>, <<"Results">>) -> + {ok, <<"Ergebnisse">>}; +lgettext(<<"de">>, <<"navigation">>, <<"Resources">>) -> + {ok, <<"Ressourcen">>}. + + check(F, R) -> R = F(). diff --git a/lib/inets/doc/src/mod_esi.xml b/lib/inets/doc/src/mod_esi.xml index a8393c9248..d024c8afa8 100644 --- a/lib/inets/doc/src/mod_esi.xml +++ b/lib/inets/doc/src/mod_esi.xml @@ -142,7 +142,7 @@ to the client and <c>SessionID</c> is an identifier that shall by used when calling this function, do not assume anything about the datatype. This function may be called - several times to chunk the the respons data. Notice that the + several times to chunk the response data. Notice that the first chunk of data sent to the client must at least contain all HTTP header fields that the response will generate. If the first chunk does not contain the <em>end of HTTP header</em>, diff --git a/lib/kernel/doc/src/file.xml b/lib/kernel/doc/src/file.xml index b674b3ca93..593bee74fe 100644 --- a/lib/kernel/doc/src/file.xml +++ b/lib/kernel/doc/src/file.xml @@ -59,7 +59,7 @@ terminal supports UTF-8, otherwise <c>latin1</c>. The default can be overridden using <c>+fnl</c> (to force <c>latin1</c> mode) or <c>+fnu</c> (to force <c>utf8</c> mode) when starting - <seealso marker="erts:erl"><c>erts:erl</c></seealso>.</p> + <seealso marker="erts:erl"><c>erl</c></seealso>.</p> <p>On operating systems with transparent naming, files can be inconsistently named, for example, some files are encoded in UTF-8 while @@ -81,6 +81,22 @@ <p>See also section <seealso marker="stdlib:unicode_usage#notes-about-raw-filenames">Notes About Raw Filenames</seealso> in the STDLIB User's Guide.</p> + <note><p> + File operations used to accept filenames containing + null characters (integer value zero). This caused + the name to be truncated at the first null character. + Filenames containing null characters inside the filename + are now <em>rejected</em> and will cause primitive + file operations fail. + </p></note> + <warning><p> + Currently null characters at the end of the filename + will be accepted by primitive file operations. Such + filenames are however still documented as invalid. The + implementation will also change in the future and + reject such filenames. + </p></warning> + </description> <datatypes> @@ -96,9 +112,21 @@ </datatype> <datatype> <name name="filename"/> + <desc> + <p> + See also the documentation of the + <seealso marker="#type-name_all"><c>name_all()</c></seealso> type. + </p> + </desc> </datatype> <datatype> <name name="filename_all"/> + <desc> + <p> + See also the documentation of the + <seealso marker="#type-name_all"><c>name_all()</c></seealso> type. + </p> + </desc> </datatype> <datatype> <name name="io_device"/> @@ -112,21 +140,23 @@ <name name="name"/> <desc> <p>If VM is in Unicode filename mode, <c>string()</c> and <c>char()</c> - are allowed to be > 255. + are allowed to be > 255. See also the documentation of the + <seealso marker="#type-name_all"><c>name_all()</c></seealso> type. </p> </desc> </datatype> <datatype> <name name="name_all"/> <desc> - <p>If VM is in Unicode filename mode, <c>string()</c> and <c>char()</c> + <p>If VM is in Unicode filename mode, characters are allowed to be > 255. <c><anno>RawFilename</anno></c> is a filename not subject to Unicode translation, meaning that it can contain characters not conforming to the Unicode encoding expected from the file system (that is, non-UTF-8 characters although the VM is started - in Unicode filename mode). + in Unicode filename mode). Null characters (integer value zero) + are <em>not</em> allowed in filenames (not even at the end). </p> </desc> </datatype> diff --git a/lib/kernel/src/kernel.app.src b/lib/kernel/src/kernel.app.src index 2a88cc7e26..b5e5f8eb73 100644 --- a/lib/kernel/src/kernel.app.src +++ b/lib/kernel/src/kernel.app.src @@ -120,6 +120,6 @@ {applications, []}, {env, [{error_logger, tty}]}, {mod, {kernel, []}}, - {runtime_dependencies, ["erts-9.1", "stdlib-3.4", "sasl-3.0"]} + {runtime_dependencies, ["erts-9.1.1", "stdlib-3.4.3", "sasl-3.0"]} ] }. diff --git a/lib/kernel/test/file_name_SUITE.erl b/lib/kernel/test/file_name_SUITE.erl index 899102c908..f23529fec9 100644 --- a/lib/kernel/test/file_name_SUITE.erl +++ b/lib/kernel/test/file_name_SUITE.erl @@ -302,7 +302,9 @@ check_normal(Mod) -> {ok, BC} = Mod:read(FD,1024), ok = file:close(FD) end || {regular,Name,Content} <- NormalDir ], + {error, badarg} = Mod:rename("fil1\0tmp_fil2","tmp_fil1"), Mod:rename("fil1","tmp_fil1"), + {error, badarg} = Mod:read_file("tmp_fil1\0.txt"), {ok, <<"fil1">>} = Mod:read_file("tmp_fil1"), {error,enoent} = Mod:read_file("fil1"), Mod:rename("tmp_fil1","fil1"), diff --git a/lib/public_key/doc/src/public_key.xml b/lib/public_key/doc/src/public_key.xml index fcf37a7a4d..3040f2db0d 100644 --- a/lib/public_key/doc/src/public_key.xml +++ b/lib/public_key/doc/src/public_key.xml @@ -871,12 +871,13 @@ fun(#'DistributionPoint'{}, #'CertificateList'{}, <type> <v>Cert = der_encoded() | #'OTPCertificate'{} </v> <v>ReferenceIDs = [ RefID ]</v> - <v>RefID = {IdType,string()}</v> - <v>IdType = dns_id | srv_id | uri_id</v> + <v>RefID = {dns_id,string()} | {srv_id,string()} | {uri_id,string()} | {ip,inet:ip_address()|string()} | {OtherRefID,term()}}</v> + <v>OtherRefID = atom()</v> <v>Opts = [ PvhOpt() ]</v> <v>PvhOpt = [MatchOpt | FailCallBackOpt | FqdnExtractOpt]</v> - <v>MatchOpt = {fun(RefId | FQDN::string(), PresentedID) -> boolean() | default}</v> - <v>PresentedID = {dNSName,string()} | {uniformResourceIdentifier,string()}</v> + <v>MatchOpt = {match_fun, fun(RefId | FQDN::string(), PresentedID) -> boolean() | default}</v> + <v>PresentedID = {dNSName,string()} | {uniformResourceIdentifier,string() | {iPAddress,list(byte())} | {OtherPresId,term()}}</v> + <v>OtherPresID = atom()</v> <v>FailCallBackOpt = {fail_callback, fun(#'OTPCertificate'{}) -> boolean()}</v> <v>FqdnExtractOpt = {fqdn_fun, fun(RefID) -> FQDN::string() | default | undefined}</v> </type> @@ -893,6 +894,11 @@ fun(#'DistributionPoint'{}, #'CertificateList'{}, <seealso marker="using_public_key#verify_hostname_examples">code examples</seealso> describes this function more detailed. </p> + <p>The <c>{OtherRefId,term()}</c> is defined by the user and is passed to the <c>match_fun</c>, if defined. + If that term is a binary, it will be converted to a string. + </p> + <p>The <c>ip</c> takes a 4-tuple or a + </p> </desc> </func> diff --git a/lib/public_key/src/public_key.erl b/lib/public_key/src/public_key.erl index cc01b61433..c3f2d791a3 100644 --- a/lib/public_key/src/public_key.erl +++ b/lib/public_key/src/public_key.erl @@ -1454,13 +1454,43 @@ verify_hostname_match_default0({dns_id,R}, {dNSName,P}) -> R==P; verify_hostname_match_default0({uri_id,R}, {uniformResourceIdentifier,P}) -> R==P; -verify_hostname_match_default0({srv_id,R}, {T,P}) when T == srvName ; - T == ?srvName_OID -> +verify_hostname_match_default0({ip,R}, {iPAddress,P}) when length(P) == 4 -> + %% IPv4 + try + list_to_tuple(P) + == if is_tuple(R), size(R)==4 -> R; + is_list(R) -> ok(inet:parse_ipv4strict_address(R)) + end + catch + _:_ -> + false + end; + +verify_hostname_match_default0({ip,R}, {iPAddress,P}) when length(P) == 16 -> + %% IPv6. The length 16 is due to the certificate specification. + try + l16_to_tup(P) + == if is_tuple(R), size(R)==8 -> R; + is_list(R) -> ok(inet:parse_ipv6strict_address(R)) + end + catch + _:_ -> + false + end; +verify_hostname_match_default0({srv_id,R}, {srvName,P}) -> + R==P; +verify_hostname_match_default0({srv_id,R}, {?srvName_OID,P}) -> R==P; verify_hostname_match_default0(_, _) -> false. +ok({ok,X}) -> X. +l16_to_tup(L) -> list_to_tuple(l16_to_tup(L, [])). +%% +l16_to_tup([A,B|T], Acc) -> l16_to_tup(T, [(A bsl 8) bor B | Acc]); +l16_to_tup([], Acc) -> lists:reverse(Acc). + match_wild(A, [$*|B]) -> match_wild_suffixes(A, B); match_wild([C|A], [ C|B]) -> match_wild(A, B); match_wild([], []) -> true; @@ -1505,7 +1535,8 @@ to_lower_ascii(C) when $A =< C,C =< $Z -> C + ($a-$A); to_lower_ascii(C) -> C. to_string(S) when is_list(S) -> S; -to_string(B) when is_binary(B) -> binary_to_list(B). +to_string(B) when is_binary(B) -> binary_to_list(B); +to_string(X) -> X. format_details([]) -> no_relevant_crls; diff --git a/lib/public_key/test/public_key_SUITE.erl b/lib/public_key/test/public_key_SUITE.erl index 374fb20375..0077c7908c 100644 --- a/lib/public_key/test/public_key_SUITE.erl +++ b/lib/public_key/test/public_key_SUITE.erl @@ -47,6 +47,7 @@ all() -> pkix_iso_rsa_oid, pkix_iso_dsa_oid, pkix_crl, general_name, pkix_verify_hostname_cn, pkix_verify_hostname_subjAltName, + pkix_verify_hostname_subjAltName_IP, pkix_verify_hostname_options, pkix_test_data_all_default, pkix_test_data, @@ -985,6 +986,41 @@ pkix_verify_hostname_options(Config) -> false = public_key:pkix_verify_hostname(Cert, [{uri_id,"some://very.wrong.domain"}]). %%-------------------------------------------------------------------- +%% To generate the PEM file contents: +%% +%% openssl req -x509 -nodes -newkey rsa:1024 -keyout /dev/null -extensions SAN -config public_key_SUITE_data/verify_hostname_ip.conf 2>/dev/null > public_key_SUITE_data/pkix_verify_hostname_subjAltName_IP.pem +%% +%% Subject: C=SE, CN=example.com +%% Subject Alternative Name: DNS:1.2.3.4, DNS: abcd:ef::1, IP:5.6.7.8, URI:https://10.11.12.13 + +pkix_verify_hostname_subjAltName_IP(Config) -> + DataDir = proplists:get_value(data_dir, Config), + {ok,Bin} = file:read_file(filename:join(DataDir,"pkix_verify_hostname_subjAltName_IP.pem")), + Cert = public_key:pkix_decode_cert(element(2,hd(public_key:pem_decode(Bin))), otp), + + %% Print the tests that a matchfun has to handle + catch public_key:pkix_verify_hostname(Cert, [{some_tag,"some.domain"}, + {ip, {5,6,7,8}} + ], + [{match_fun, + fun(Ref,Pres) -> + ct:pal("~p:~p:~nRef : ~p~nPres: ~p",[?MODULE,?LINE,Ref,Pres]), + false + end}]), + + false = public_key:pkix_verify_hostname(Cert, [{uri_id,"https://1.2.3.4"}]), + true = public_key:pkix_verify_hostname(Cert, [{uri_id,"https://10.11.12.13"}]), + true = public_key:pkix_verify_hostname(Cert, [{dns_id,"1.2.3.4"}]), + true = public_key:pkix_verify_hostname(Cert, [{dns_id,<<"1.2.3.4">>}]), + false = public_key:pkix_verify_hostname(Cert, [{dns_id,"5.6.7.8"}]), + true = public_key:pkix_verify_hostname(Cert, [{ip, "aBcD:ef:0::0:1"}]), + true = public_key:pkix_verify_hostname(Cert, [{ip, {16#abcd,16#ef,0,0,0,0,0,1}}]), + true = public_key:pkix_verify_hostname(Cert, [{ip, "5.6.7.8"}]), + true = public_key:pkix_verify_hostname(Cert, [{ip, <<"5.6.7.8">>}]), + true = public_key:pkix_verify_hostname(Cert, [{ip, {5,6,7,8}}]). + + +%%-------------------------------------------------------------------- pkix_iso_rsa_oid() -> [{doc, "Test workaround for supporting certs that use ISO oids" " 1.3.14.3.2.29 instead of PKIX/PKCS oid"}]. diff --git a/lib/public_key/test/public_key_SUITE_data/pkix_verify_hostname_subjAltName_IP.pem b/lib/public_key/test/public_key_SUITE_data/pkix_verify_hostname_subjAltName_IP.pem new file mode 100644 index 0000000000..f9ffb257b5 --- /dev/null +++ b/lib/public_key/test/public_key_SUITE_data/pkix_verify_hostname_subjAltName_IP.pem @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB/zCCAWigAwIBAgIJAMoSejmTjwAGMA0GCSqGSIb3DQEBCwUAMB8xCzAJBgNV +BAYTAlNFMRAwDgYDVQQDEwc1LjYuNy44MB4XDTE3MDkyODE0MDAxNVoXDTE3MTAy +ODE0MDAxNVowHzELMAkGA1UEBhMCU0UxEDAOBgNVBAMTBzUuNi43LjgwgZ8wDQYJ +KoZIhvcNAQEBBQADgY0AMIGJAoGBAMUPU89KwVbTCDkyxQSz3wprMbZTLe35K6jm +Q7oY1rJyVXjsFHwZrFqqNMScEyX40rJhczQ2Z9etEX6qYLbdb/DZeFcKo14fR583 +QMFZC+qqpLWHdvjaQN0KwD99VFeZIGpRgywG8SR+BXZjDHUkGsMrikAEJtf0Tgih +IPyiFtiJAgMBAAGjQzBBMD8GA1UdEQQ4MDaCBzEuMi4zLjSHBAUGBwiHEKvNAO8A +AAAAAAAAAAAAAAGGE2h0dHBzOi8vMTAuMTEuMTIuMTMwDQYJKoZIhvcNAQELBQAD +gYEAtWVeQaRFZ0kH/pzSWMSsOCUrjbwlWRwDNbagNKoM6nCRv0QQ59fG6XrVZwR3 +c0s5arlMh3U2+bjKE+Iq9+b/lN1lGzf8iaAqBNa7KptwTSUEY3TiNG5X0zlSXKTI +3z7AaUEtghL9ImCPj5V3tVksqWd7U0zLmeeLZnM+wGAL9Hc= +-----END CERTIFICATE----- diff --git a/lib/public_key/test/public_key_SUITE_data/verify_hostname_ip.conf b/lib/public_key/test/public_key_SUITE_data/verify_hostname_ip.conf new file mode 100644 index 0000000000..0a738f2586 --- /dev/null +++ b/lib/public_key/test/public_key_SUITE_data/verify_hostname_ip.conf @@ -0,0 +1,18 @@ +[req] +prompt = no +distinguished_name = DN + +[DN] +C=SE +CN=example.com +CN=5.6.7.8 + +[SAN] +subjectAltName = @alt_names + +[alt_names] +DNS = 1.2.3.4 +IP.1 = 5.6.7.8 +IP.2 = abcd:ef::1 +URI = https://10.11.12.13 + diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl index ff3e69bae5..a352b7e025 100644 --- a/lib/ssl/src/dtls_connection.erl +++ b/lib/ssl/src/dtls_connection.erl @@ -45,7 +45,7 @@ -export([renegotiate/2, reinit_handshake_data/1, send_handshake/2, queue_handshake/2, queue_change_cipher/2, - select_sni_extension/1]). + select_sni_extension/1, empty_connection_state/2]). %% Alert and close handling -export([encode_alert/3,send_alert/2, close/5, protocol_name/0]). @@ -79,9 +79,9 @@ start_fsm(Role, Host, Port, Socket, {#ssl_options{erl_dist = false},_, Tracker} Error end. -send_handshake(Handshake, #state{connection_states = ConnectionStates} = States) -> +send_handshake(Handshake, #state{connection_states = ConnectionStates} = State) -> #{epoch := Epoch} = ssl_record:current_connection_state(ConnectionStates, write), - send_handshake_flight(queue_handshake(Handshake, States), Epoch). + send_handshake_flight(queue_handshake(Handshake, State), Epoch). queue_handshake(Handshake0, #state{tls_handshake_history = Hist0, negotiated_version = Version, @@ -114,8 +114,8 @@ send_handshake_flight(#state{socket = Socket, %% TODO remove hardcoded Max size {Encoded, ConnectionStates} = encode_handshake_flight(lists:reverse(Flight), Version, 1400, Epoch, ConnectionStates0), - send(Transport, Socket, Encoded), - {State0#state{connection_states = ConnectionStates}, []}; + send(Transport, Socket, Encoded), + {State0#state{connection_states = ConnectionStates}, []}; send_handshake_flight(#state{socket = Socket, transport_cb = Transport, @@ -188,9 +188,10 @@ reinit_handshake_data(#state{protocol_buffers = Buffers} = State) -> public_key_info = undefined, tls_handshake_history = ssl_handshake:init_handshake_history(), flight_state = {retransmit, ?INITIAL_RETRANSMIT_TIMEOUT}, - protocol_buffers = + flight_buffer = new_flight(), + protocol_buffers = Buffers#protocol_buffers{ - dtls_handshake_next_seq = 0, + dtls_handshake_next_seq = 0, dtls_handshake_next_fragments = [], dtls_handshake_later_fragments = [] }}. @@ -199,6 +200,9 @@ select_sni_extension(#client_hello{extensions = HelloExtensions}) -> HelloExtensions#hello_extensions.sni; select_sni_extension(_) -> undefined. +empty_connection_state(ConnectionEnd, BeastMitigation) -> + Empty = ssl_record:empty_connection_state(ConnectionEnd, BeastMitigation), + dtls_record:empty_connection_state(Empty). socket(Pid, Transport, Socket, Connection, _) -> dtls_socket:socket(Pid, Transport, Socket, Connection). @@ -355,13 +359,14 @@ hello(internal, #hello_verify_request{cookie = Cookie}, #state{role = client, session_cache = Cache, session_cache_cb = CacheCb } = State0) -> - State1 = prepare_flight(State0#state{tls_handshake_history = ssl_handshake:init_handshake_history()}), + Hello = dtls_handshake:client_hello(Host, Port, Cookie, ConnectionStates0, SslOpts, Cache, CacheCb, Renegotiation, OwnCert), Version = Hello#client_hello.client_version, - HelloVersion = dtls_record:lowest_protocol_version(SslOpts#ssl_options.versions), - {State2, Actions} = send_handshake(Hello, State1#state{negotiated_version = HelloVersion}), + State1 = prepare_flight(State0#state{tls_handshake_history = ssl_handshake:init_handshake_history()}), + + {State2, Actions} = send_handshake(Hello, State1), State3 = State2#state{negotiated_version = Version, %% Requested version session = Session0#session{session_id = @@ -451,17 +456,22 @@ connection(enter, _, State) -> connection(info, Event, State) -> handle_info(Event, connection, State); connection(internal, #hello_request{}, #state{host = Host, port = Port, - session = #session{own_certificate = Cert} = Session0, - session_cache = Cache, session_cache_cb = CacheCb, - ssl_options = SslOpts, - connection_states = ConnectionStates0, - renegotiation = {Renegotiation, _}} = State0) -> + session = #session{own_certificate = Cert} = Session0, + session_cache = Cache, session_cache_cb = CacheCb, + ssl_options = SslOpts, + connection_states = ConnectionStates0, + renegotiation = {Renegotiation, _}} = State0) -> + Hello = dtls_handshake:client_hello(Host, Port, ConnectionStates0, SslOpts, Cache, CacheCb, Renegotiation, Cert), - {State1, Actions} = send_handshake(Hello, State0), + Version = Hello#client_hello.client_version, + HelloVersion = dtls_record:hello_version(Version, SslOpts#ssl_options.versions), + State1 = prepare_flight(State0), + {State2, Actions} = send_handshake(Hello, State1#state{negotiated_version = HelloVersion}), {Record, State} = next_record( - State1#state{session = Session0#session{session_id + State2#state{flight_state = {retransmit, ?INITIAL_RETRANSMIT_TIMEOUT}, + session = Session0#session{session_id = Hello#client_hello.session_id}}), next_event(hello, Record, State, Actions); connection(internal, #client_hello{} = Hello, #state{role = server, allow_renegotiate = true} = State) -> @@ -471,7 +481,8 @@ connection(internal, #client_hello{} = Hello, #state{role = server, allow_renego %% initiated renegotiation we will disallow many client initiated %% renegotiations immediately after each other. erlang:send_after(?WAIT_TO_ALLOW_RENEGOTIATION, self(), allow_renegotiate), - {next_state, hello, State#state{allow_renegotiate = false}, [{next_event, internal, Hello}]}; + {next_state, hello, State#state{allow_renegotiate = false, renegotiation = {true, peer}}, + [{next_event, internal, Hello}]}; connection(internal, #client_hello{}, #state{role = server, allow_renegotiate = false} = State0) -> Alert = ?ALERT_REC(?WARNING, ?NO_RENEGOTIATION), State1 = send_alert(Alert, State0), @@ -542,7 +553,6 @@ handle_info(new_cookie_secret, StateName, handle_info(Msg, StateName, State) -> ssl_connection:handle_info(Msg, StateName, State). - handle_call(Event, From, StateName, State) -> ssl_connection:handle_call(Event, From, StateName, State, ?MODULE). @@ -796,7 +806,13 @@ next_event(connection = StateName, no_record, case next_record_if_active(State0) of {no_record, State} -> ssl_connection:hibernate_after(StateName, State, Actions); - {#ssl_tls{epoch = CurrentEpoch} = Record, State} -> + {#ssl_tls{epoch = CurrentEpoch, + type = ?HANDSHAKE, + version = Version} = Record, State1} -> + State = dtls_version(StateName, Version, State1), + {next_state, StateName, State, + [{next_event, internal, {protocol_record, Record}} | Actions]}; + {#ssl_tls{epoch = CurrentEpoch} = Record, State} -> {next_state, StateName, State, [{next_event, internal, {protocol_record, Record}} | Actions]}; {#ssl_tls{epoch = Epoch, type = ?HANDSHAKE, @@ -822,6 +838,12 @@ next_event(connection = StateName, no_record, next_event(connection = StateName, Record, #state{connection_states = #{current_read := #{epoch := CurrentEpoch}}} = State0, Actions) -> case Record of + #ssl_tls{epoch = CurrentEpoch, + type = ?HANDSHAKE, + version = Version} = Record -> + State = dtls_version(StateName, Version, State0), + {next_state, StateName, State, + [{next_event, internal, {protocol_record, Record}} | Actions]}; #ssl_tls{epoch = CurrentEpoch} -> {next_state, StateName, State0, [{next_event, internal, {protocol_record, Record}} | Actions]}; #ssl_tls{epoch = Epoch, @@ -845,11 +867,11 @@ next_event(StateName, Record, case Record of no_record -> {next_state, StateName, State0, Actions}; - #ssl_tls{epoch = CurrentEpoch, - version = Version} = Record -> - {next_state, StateName, - dtls_version(StateName, Version, State0), - [{next_event, internal, {protocol_record, Record}} | Actions]}; + #ssl_tls{epoch = CurrentEpoch, + version = Version} = Record -> + State = dtls_version(StateName, Version, State0), + {next_state, StateName, State, + [{next_event, internal, {protocol_record, Record}} | Actions]}; #ssl_tls{epoch = _Epoch, version = _Version} = _Record -> %% TODO maybe buffer later epoch @@ -895,7 +917,7 @@ next_flight(Flight) -> Flight#{handshakes => [], change_cipher_spec => undefined, handshakes_after_change_cipher_spec => []}. - + handle_flight_timer(#state{transport_cb = gen_udp, flight_state = {retransmit, Timeout}} = State) -> start_retransmision_timer(Timeout, State); @@ -923,21 +945,15 @@ dtls_handshake_events(Packets) -> renegotiate(#state{role = client} = State, Actions) -> %% Handle same way as if server requested %% the renegotiation - Hs0 = ssl_handshake:init_handshake_history(), - {next_state, connection, State#state{tls_handshake_history = Hs0, - protocol_buffers = #protocol_buffers{}}, + %% Hs0 = ssl_handshake:init_handshake_history(), + {next_state, connection, State, [{next_event, internal, #hello_request{}} | Actions]}; -renegotiate(#state{role = server, - connection_states = CS0} = State0, Actions) -> +renegotiate(#state{role = server} = State0, Actions) -> HelloRequest = ssl_handshake:hello_request(), - CS = CS0#{write_msg_seq => 0}, - {State1, MoreActions} = send_handshake(HelloRequest, - State0#state{connection_states = - CS}), - Hs0 = ssl_handshake:init_handshake_history(), - {Record, State} = next_record(State1#state{tls_handshake_history = Hs0, - protocol_buffers = #protocol_buffers{}}), + State1 = prepare_flight(State0), + {State2, MoreActions} = send_handshake(HelloRequest, State1), + {Record, State} = next_record(State2), next_event(hello, Record, State, Actions ++ MoreActions). handle_alerts([], Result) -> @@ -953,7 +969,6 @@ retransmit_epoch(_StateName, #state{connection_states = ConnectionStates}) -> #{epoch := Epoch} = ssl_record:current_connection_state(ConnectionStates, write), Epoch. - update_handshake_history(#hello_verify_request{}, _, Hist) -> Hist; diff --git a/lib/ssl/src/dtls_record.erl b/lib/ssl/src/dtls_record.erl index 8a7f8c1d0a..a8520717e5 100644 --- a/lib/ssl/src/dtls_record.erl +++ b/lib/ssl/src/dtls_record.erl @@ -30,7 +30,7 @@ -include("ssl_cipher.hrl"). %% Handling of incoming data --export([get_dtls_records/2, init_connection_states/2]). +-export([get_dtls_records/2, init_connection_states/2, empty_connection_state/1]). %% Decoding -export([decode_cipher_text/2]). @@ -75,7 +75,7 @@ init_connection_states(Role, BeastMitigation) -> Initial = initial_connection_state(ConnectionEnd, BeastMitigation), Current = Initial#{epoch := 0}, InitialPending = ssl_record:empty_connection_state(ConnectionEnd, BeastMitigation), - Pending = InitialPending#{epoch => undefined, replay_window => init_replay_window(?REPLAY_WINDOW_SIZE)}, + Pending = empty_connection_state(InitialPending), #{saved_read => Current, current_read => Current, pending_read => Pending, @@ -83,6 +83,10 @@ init_connection_states(Role, BeastMitigation) -> current_write => Current, pending_write => Pending}. +empty_connection_state(Empty) -> + Empty#{epoch => undefined, replay_window => init_replay_window(?REPLAY_WINDOW_SIZE)}. + + %%-------------------------------------------------------------------- -spec save_current_connection_state(ssl_record:connection_states(), read | write) -> ssl_record:connection_states(). diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index b031d3d47b..2dbe08e0a7 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -448,7 +448,7 @@ abbreviated(internal, #change_cipher_spec{type = <<1>>}, #state{connection_states = ConnectionStates0} = State0, Connection) -> ConnectionStates1 = - ssl_record:activate_pending_connection_state(ConnectionStates0, read), + ssl_record:activate_pending_connection_state(ConnectionStates0, read, Connection), {Record, State} = Connection:next_record(State0#state{connection_states = ConnectionStates1}), Connection:next_event(abbreviated, Record, State#state{expecting_finished = true}); @@ -727,7 +727,7 @@ cipher(internal, #next_protocol{selected_protocol = SelectedProtocol}, cipher(internal, #change_cipher_spec{type = <<1>>}, #state{connection_states = ConnectionStates0} = State0, Connection) -> ConnectionStates1 = - ssl_record:activate_pending_connection_state(ConnectionStates0, read), + ssl_record:activate_pending_connection_state(ConnectionStates0, read, Connection), {Record, State} = Connection:next_record(State0#state{connection_states = ConnectionStates1}), Connection:next_event(cipher, Record, State#state{expecting_finished = true}); @@ -1168,8 +1168,9 @@ handle_alert(#alert{level = ?WARNING, description = ?NO_RENEGOTIATION} = Alert, log_alert(SslOpts#ssl_options.log_alert, Role, Connection:protocol_name(), StateName, Alert#alert{role = opposite_role(Role)}), gen_statem:reply(From, {error, renegotiation_rejected}), - {Record, State} = Connection:next_record(State0), + {Record, State1} = Connection:next_record(State0), %% Go back to connection! + State = Connection:reinit_handshake_data(State1#state{renegotiation = undefined}), Connection:next_event(connection, Record, State); %% Gracefully log and ignore all other warning alerts @@ -1721,7 +1722,7 @@ finalize_handshake(State0, StateName, Connection) -> ConnectionStates = ssl_record:activate_pending_connection_state(ConnectionStates0, - write), + write, Connection), State2 = State1#state{connection_states = ConnectionStates}, State = next_protocol(State2, Connection), diff --git a/lib/ssl/src/ssl_record.erl b/lib/ssl/src/ssl_record.erl index 62c2ffce8b..003ad4994b 100644 --- a/lib/ssl/src/ssl_record.erl +++ b/lib/ssl/src/ssl_record.erl @@ -31,7 +31,7 @@ %% Connection state handling -export([initial_security_params/1, current_connection_state/2, pending_connection_state/2, - activate_pending_connection_state/2, + activate_pending_connection_state/3, set_security_params/3, set_mac_secret/4, set_master_secret/2, @@ -83,7 +83,7 @@ pending_connection_state(ConnectionStates, write) -> maps:get(pending_write, ConnectionStates). %%-------------------------------------------------------------------- --spec activate_pending_connection_state(connection_states(), read | write) -> +-spec activate_pending_connection_state(connection_states(), read | write, tls_connection | dtls_connection) -> connection_states(). %% %% Description: Creates a new instance of the connection_states record @@ -91,13 +91,13 @@ pending_connection_state(ConnectionStates, write) -> %%-------------------------------------------------------------------- activate_pending_connection_state(#{current_read := Current, pending_read := Pending} = States, - read) -> + read, Connection) -> #{secure_renegotiation := SecureRenegotation} = Current, #{beast_mitigation := BeastMitigation, security_parameters := SecParams} = Pending, NewCurrent = Pending#{sequence_number => 0}, ConnectionEnd = SecParams#security_parameters.connection_end, - EmptyPending = empty_connection_state(ConnectionEnd, BeastMitigation), + EmptyPending = Connection:empty_connection_state(ConnectionEnd, BeastMitigation), NewPending = EmptyPending#{secure_renegotiation => SecureRenegotation}, States#{current_read => NewCurrent, pending_read => NewPending @@ -105,13 +105,13 @@ activate_pending_connection_state(#{current_read := Current, activate_pending_connection_state(#{current_write := Current, pending_write := Pending} = States, - write) -> + write, Connection) -> NewCurrent = Pending#{sequence_number => 0}, #{secure_renegotiation := SecureRenegotation} = Current, #{beast_mitigation := BeastMitigation, security_parameters := SecParams} = Pending, ConnectionEnd = SecParams#security_parameters.connection_end, - EmptyPending = empty_connection_state(ConnectionEnd, BeastMitigation), + EmptyPending = Connection:empty_connection_state(ConnectionEnd, BeastMitigation), NewPending = EmptyPending#{secure_renegotiation => SecureRenegotation}, States#{current_write => NewCurrent, pending_write => NewPending diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index e3ffbea3d3..010e904839 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -53,7 +53,7 @@ %% Handshake handling -export([renegotiate/2, send_handshake/2, queue_handshake/2, queue_change_cipher/2, - reinit_handshake_data/1, select_sni_extension/1]). + reinit_handshake_data/1, select_sni_extension/1, empty_connection_state/2]). %% Alert and close handling -export([send_alert/2, close/5, protocol_name/0]). @@ -152,6 +152,9 @@ select_sni_extension(#client_hello{extensions = HelloExtensions}) -> select_sni_extension(_) -> undefined. +empty_connection_state(ConnectionEnd, BeastMitigation) -> + ssl_record:empty_connection_state(ConnectionEnd, BeastMitigation). + encode_data(Data, Version, ConnectionStates0)-> tls_record:encode_data(Data, Version, ConnectionStates0). diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index 9efde4752f..3b4ca40058 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -83,13 +83,14 @@ groups() -> ]. tls_versions_groups ()-> - [{group, renegotiate}, %% Should be in all_versions_groups not fixed for DTLS yet + [ {group, api_tls}, {group, tls_ciphers}, {group, error_handling_tests_tls}]. all_versions_groups ()-> [{group, api}, + {group, renegotiate}, {group, ciphers}, {group, ciphers_ec}, {group, error_handling_tests}]. diff --git a/lib/ssl/test/ssl_to_openssl_SUITE.erl b/lib/ssl/test/ssl_to_openssl_SUITE.erl index 2e1a0b94ea..9118e4b7e3 100644 --- a/lib/ssl/test/ssl_to_openssl_SUITE.erl +++ b/lib/ssl/test/ssl_to_openssl_SUITE.erl @@ -90,9 +90,9 @@ dtls_all_versions_tests() -> erlang_client_openssl_server_dsa_cert, erlang_server_openssl_client_dsa_cert, erlang_server_openssl_client_reuse_session, - %%erlang_client_openssl_server_renegotiate, - %%erlang_client_openssl_server_nowrap_seqnum, - %%erlang_server_openssl_client_nowrap_seqnum, + erlang_client_openssl_server_renegotiate, + erlang_client_openssl_server_nowrap_seqnum, + erlang_server_openssl_client_nowrap_seqnum, erlang_client_openssl_server_no_server_ca_cert, erlang_client_openssl_server_client_cert, erlang_server_openssl_client_client_cert, diff --git a/lib/stdlib/doc/src/ets.xml b/lib/stdlib/doc/src/ets.xml index 95af2b77a5..576959b1c8 100644 --- a/lib/stdlib/doc/src/ets.xml +++ b/lib/stdlib/doc/src/ets.xml @@ -325,7 +325,7 @@ <p><c><anno>Acc0</anno></c> is returned if the table is empty. This function is similar to <seealso marker="lists#foldl/3"><c>lists:foldl/3</c></seealso>. - The table elements are traversed is unspecified order, except for + The table elements are traversed in an unspecified order, except for <c>ordered_set</c> tables, where they are traversed first to last.</p> <p>If <c><anno>Function</anno></c> inserts objects into the table, or another @@ -341,7 +341,7 @@ <p><c><anno>Acc0</anno></c> is returned if the table is empty. This function is similar to <seealso marker="lists#foldr/3"><c>lists:foldr/3</c></seealso>. - The table elements are traversed is unspecified order, except for + The table elements are traversed in an unspecified order, except for <c>ordered_set</c> tables, where they are traversed last to first.</p> <p>If <c><anno>Function</anno></c> inserts objects into the table, or another diff --git a/lib/stdlib/doc/src/filelib.xml b/lib/stdlib/doc/src/filelib.xml index 80c4acffdb..57c4348745 100644 --- a/lib/stdlib/doc/src/filelib.xml +++ b/lib/stdlib/doc/src/filelib.xml @@ -45,6 +45,30 @@ <p>For more information about raw filenames, see the <seealso marker="kernel:file"><c>file</c></seealso> module.</p> + + <note> + <p> + Functionality in this module generally assumes valid input and + does not necessarily fail on input that does not use a valid + encoding. You can validate the encoding of a filename using + <seealso marker="stdlib:filename#validate/1">filename:validate/1</seealso>. + </p> + <p> + File operations used to accept filenames containing + null characters (integer value zero). This caused + the name to be truncated at the first null character. + Filenames containing null characters inside the filename + are now <em>rejected</em> and will cause primitive + file operations fail. + </p> + </note> + <warning><p> + Currently null characters at the end of the filename + will be accepted by primitive file operations. Such + filenames are however still documented as invalid. The + implementation will also change in the future and + reject such filenames. + </p></warning> </description> <datatypes> diff --git a/lib/stdlib/doc/src/filename.xml b/lib/stdlib/doc/src/filename.xml index 14fd5ef787..b6028fc066 100644 --- a/lib/stdlib/doc/src/filename.xml +++ b/lib/stdlib/doc/src/filename.xml @@ -46,7 +46,10 @@ filename by removing redundant directory separators, use <seealso marker="#join/1"><c>join/1</c></seealso>.</p> - <p>The module supports raw filenames in the way that if a binary is + <p> + The module supports + <seealso marker="unicode_usage#notes-about-raw-filenames">raw + filenames</seealso> in the way that if a binary is present, or the filename cannot be interpreted according to the return value of <seealso marker="kernel:file#native_name_encoding/0"> <c>file:native_name_encoding/0</c></seealso>, a raw filename is also @@ -56,6 +59,30 @@ (the join operation is performed of course). For more information about raw filenames, see the <seealso marker="kernel:file"><c>file</c></seealso> module.</p> + + <note> + <p> + Functionality in this module generally assumes valid input and + does not necessarily fail on input that does not use a valid + encoding. You can validate the encoding of a filename using + <seealso marker="#validate/1">filename:validate/1</seealso>. + </p> + <p> + File operations used to accept filenames containing + null characters (integer value zero). This caused + the name to be truncated at the first null character. + Filenames containing null characters inside the filename + are now <em>rejected</em> and will cause primitive + file operations fail. + </p> + </note> + <warning><p> + Currently null characters at the end of the filename + will be accepted by primitive file operations. Such + filenames are however still documented as invalid. The + implementation will also change in the future and + reject such filenames. + </p></warning> </description> <datatypes> <datatype> @@ -555,6 +582,55 @@ unsafe</pre> ["a:/","msdev","include"]</pre> </desc> </func> + + <func> + <name name="validate" arity="1"/> + <fsummary>Validate encoding of filename</fsummary> + <desc> + <p> + Validates filename encoding. Returns <c>true</c> if + <c><anno>FileName</anno></c> has a valid encoding; + otherwise, returns <c>false</c>. + </p> + <taglist> + <tag>Ordinary Filename</tag> + <item> + <p> + Type: <c><anno>FileName</anno> = </c><seealso marker="kernel:file#type-name"><c>file:name()</c></seealso> + </p> + <p> + Validates encoding against the + <seealso marker="kernel:file#native_name_encoding/0">native file + name encoding</seealso>, and the + capabilities of the operating system used. + Regardless of configuration and OS, null + characters (integer value zero) will be + rejected by the validation (even when only + present at the end of the filename). + </p> + </item> + <tag><seealso marker="unicode_usage#notes-about-raw-filenames">Raw + Filename</seealso></tag> + <item> + <p> + Type: <c><anno>FileName</anno> = binary()</c> + </p> + <p> + The encoding will not be interpreted, but + null bytes (integer value zero) will be + rejected by the validation (even when only + present at the end of the filename). + </p> + </item> + </taglist> + <p> + For information on filename encoding see the documentation + of unicode filenames in + <seealso marker="stdlib:unicode_usage#unicode_file_names">STDLIB + Users Guide ➜ Using Unicode in Erlang ➜ Unicode Filenames</seealso>. + </p> + </desc> + </func> </funcs> </erlref> diff --git a/lib/stdlib/doc/src/unicode_usage.xml b/lib/stdlib/doc/src/unicode_usage.xml index 26dc46719e..ff1f864e22 100644 --- a/lib/stdlib/doc/src/unicode_usage.xml +++ b/lib/stdlib/doc/src/unicode_usage.xml @@ -719,8 +719,8 @@ Eshell V5.10.1 (abort with ^G) </section> <section> - <title>Unicode Filenames</title> <marker id="unicode_file_names"/> + <title>Unicode Filenames</title> <p>Most modern operating systems support Unicode filenames in some way. There are many different ways to do this and Erlang by default treats the different approaches differently:</p> @@ -855,8 +855,8 @@ Eshell V5.10.1 (abort with ^G) </note> <section> - <title>Notes About Raw Filenames</title> <marker id="notes-about-raw-filenames"/> + <title>Notes About Raw Filenames</title> <p>Raw filenames were introduced together with Unicode filename support in ERTS 5.8.2 (Erlang/OTP R14B01). The reason "raw filenames" were introduced in the system was diff --git a/lib/stdlib/src/filename.erl b/lib/stdlib/src/filename.erl index 9bf4290916..1c3ab6d274 100644 --- a/lib/stdlib/src/filename.erl +++ b/lib/stdlib/src/filename.erl @@ -41,6 +41,7 @@ safe_relative_path/1]). -export([find_src/1, find_src/2]). % deprecated -export([basedir/2, basedir/3]). +-export([validate/1]). %% Undocumented and unsupported exports. -export([append/2]). @@ -1135,3 +1136,72 @@ basedir_os_type() -> {win32,_} -> windows; _ -> linux end. + +%% +%% validate/1 +%% + +-spec validate(FileName) -> boolean() when + FileName :: file:name_all(). + +validate(FileName) when is_binary(FileName) -> + %% Raw filename... + validate_bin(FileName); +validate(FileName) when is_list(FileName); + is_atom(FileName) -> + validate_list(FileName, + file:native_name_encoding(), + os:type()). + +validate_list(FileName, Enc, Os) -> + try + true = validate_list(FileName, Enc, Os, 0) > 0 + catch + _ : _ -> false + end. + +validate_list([], _Enc, _Os, Chars) -> + Chars; +validate_list(C, Enc, Os, Chars) when is_integer(C) -> + validate_char(C, Enc, Os), + Chars+1; +validate_list(A, Enc, Os, Chars) when is_atom(A) -> + validate_list(atom_to_list(A), Enc, Os, Chars); +validate_list([H|T], Enc, Os, Chars) -> + NewChars = validate_list(H, Enc, Os, Chars), + validate_list(T, Enc, Os, NewChars). + +%% C is always an integer... +% validate_char(C, _, _) when not is_integer(C) -> +% throw(invalid); +validate_char(C, _, _) when C < 1 -> + throw(invalid); %% No negative or null characters... +validate_char(C, latin1, _) when C > 255 -> + throw(invalid); +validate_char(C, utf8, _) when C >= 16#110000 -> + throw(invalid); +validate_char(C, utf8, {win32, _}) when C > 16#ffff -> + throw(invalid); %% invalid win wchar... +validate_char(_C, utf8, {win32, _}) -> + ok; %% Range below is accepted on windows... +validate_char(C, utf8, _) when 16#D800 =< C, C =< 16#DFFF -> + throw(invalid); %% invalid unicode range... +validate_char(_, _, _) -> + ok. + +validate_bin(Bin) -> + %% Raw filename. That is, we do not interpret + %% the encoding, but we still do not accept + %% null characters... + try + true = validate_bin(Bin, 0) > 0 + catch + _ : _ -> false + end. + +validate_bin(<<>>, Bs) -> + Bs; +validate_bin(<<0, _Rest/binary>>, _Bs) -> + throw(invalid); %% No null characters allowed... +validate_bin(<<_B, Rest/binary>>, Bs) -> + validate_bin(Rest, Bs+1). diff --git a/lib/stdlib/src/stdlib.app.src b/lib/stdlib/src/stdlib.app.src index 3c449d3cb9..41c89270aa 100644 --- a/lib/stdlib/src/stdlib.app.src +++ b/lib/stdlib/src/stdlib.app.src @@ -107,7 +107,7 @@ dets]}, {applications, [kernel]}, {env, []}, - {runtime_dependencies, ["sasl-3.0","kernel-5.0","erts-9.0","crypto-3.3", + {runtime_dependencies, ["sasl-3.0","kernel-5.4.1","erts-9.1.1","crypto-3.3", "compiler-5.0"]} ]}. diff --git a/lib/stdlib/test/filename_SUITE.erl b/lib/stdlib/test/filename_SUITE.erl index fc77593bb8..4c82ec1c22 100644 --- a/lib/stdlib/test/filename_SUITE.erl +++ b/lib/stdlib/test/filename_SUITE.erl @@ -30,6 +30,7 @@ -export([pathtype_bin/1,rootname_bin/1,split_bin/1]). -export([t_basedir_api/1, t_basedir_xdg/1, t_basedir_windows/1]). -export([safe_relative_path/1]). +-export([validate/1]). -include_lib("common_test/include/ct.hrl"). @@ -43,7 +44,8 @@ all() -> absname_bin, absname_bin_2, {group,p}, t_basedir_xdg, t_basedir_windows, - safe_relative_path]. + safe_relative_path, + validate]. groups() -> [{p, [parallel], @@ -1011,3 +1013,56 @@ basedir_xdg_def(Type,Home,Name) -> Dir <- ["/usr/local/share/","/usr/share/"]]; site_config -> [filename:join(["/etc/xdg",Name])] end. + +validate(Config) when is_list(Config) -> + true = filename:validate(blipp), + false = filename:validate('bli\0pp'), + false = filename:validate('blipp\0'), + true = filename:validate("blipp"), + false = filename:validate("bli"++[0]++"pp"), + false = filename:validate("blipp"++[0]), + true = filename:validate(["one ", blipp, "blopp"]), + false = filename:validate(["one ", 'bli\0pp', "blopp"]), + false = filename:validate(["one ", 'blipp\0', "blopp"]), + false = filename:validate(["one ", 'blipp', "blopp\0"]), + false = filename:validate([0]), + false = filename:validate([]), + false = filename:validate([[[]],[[[[],[[[[[[[[]]], '', [[[[[]]]]]]]]]]]]]]), + false = filename:validate([16#110000]), + false = filename:validate([16#110001]), + false = filename:validate([16#110000*2]), + case file:native_name_encoding() of + latin1 -> + true = filename:validate(lists:seq(1, 255)), + false = filename:validate([256]); + utf8 -> + true = filename:validate(lists:seq(1, 16#D7FF)), + true = filename:validate(lists:seq(16#E000, 16#FFFF)), + true = filename:validate([16#FFFF]), + case os:type() of + {win32, _} -> + false = filename:validate([16#10000]), + true = filename:validate(lists:seq(16#D800,16#DFFF)); + _ -> + true = filename:validate([16#10000]), + true = filename:validate([16#10FFFF]), + lists:foreach(fun (C) -> + false = filename:validate([C]) + end, + lists:seq(16#D800,16#DFFF)) + end + + end, + true = filename:validate(<<1,17,255>>), + false = filename:validate(<<1,0,17,255>>), + false = filename:validate(<<1,17,255,0>>), + false = filename:validate(<<>>), + lists:foreach(fun (N) -> + true = filename:validate(N) + end, + code:get_path()), + ok. + + + + diff --git a/scripts/pre-push b/scripts/pre-push new file mode 100755 index 0000000000..0349378056 --- /dev/null +++ b/scripts/pre-push @@ -0,0 +1,202 @@ +#!/bin/sh + +# This is a git pre-push hook script. +# It limits what you can push toward https://github.com/erlang/otp.git +# +# To activate, make a copy as .git/hooks/pre-push in your repo. + +# Called by "git push" +# after it has checked the remote status, but before anything has been +# pushed. If this script exits with a non-zero status nothing will be pushed. +# +# This hook is called with the following parameters: +# +# $1 -- Name of the remote to which the push is being done +# $2 -- URL to which the push is being done +# +# If pushing without using a named remote those arguments will be equal. +# +# Information about the commits which are being pushed is supplied as lines to +# the standard input in the form: +# +# <local ref> <local sha1> <remote ref> <remote sha1> +# + +RELEASES="20 19 18 17 r16 r15 r14 r13" + +# First commit on master, not allowed in other branches +MASTER_ONLY=f52748254f17ba42e344798e8c787a1e3361fa33 + +# Number of commits and files allowed in one push by this script +NCOMMITS_MAX=100 +NFILES_MAX=100 + +remote="$1" +url="$2" + +null=0000000000000000000000000000000000000000 + +#echo "pre-push hook: remote=$remote" +#echo "pre-push hook: url=$url" + +if [ "$url" = 'https://github.com/erlang/otp.git' -o "$url" = '[email protected]:erlang/otp.git' ] +then + if [ $remote = "$url" ]; then + echo "$0 says:" + echo "***" + echo "*** Push to $url without using a named remote is NOT ALLOWED!!!!" + echo "***" + exit 1 + fi + IFS=' ' + while read local_ref local_sha remote_ref remote_sha + do + #echo "pre-push hook: local_ref=$local_ref" + #echo "pre-push hook: remote_ref=$remote_ref" + #echo "pre-push hook: local_sha=$local_sha" + #echo "pre-push hook: remote_sha=$remote_sha" + + if [ "$local_sha" = $null ] + then + echo "$0 says:" + echo "***" + echo "*** DELETE push to '$remote' NOT ALLOWED!!!!!" + echo "***" + exit 1 + fi + if [ "$local_ref" != "$remote_ref" ] + then + echo "$0 says:" + echo "***" + echo "*** RENAME push: $local_ref pushed as $remote_ref to '$remote' NOT ALLOWED!!!!" + echo "***" + exit 1 + fi + case "$remote_ref" in + refs/heads/master | refs/heads/maint | refs/heads/maint-[0-9][0-9] | refs/heads/maint-r[0-9][0-9]) + branch=${remote_ref#refs/heads/} + if [ "$remote_sha" = $null ] + then + echo "$0 says:" + echo "***" + echo "*** UNKNOWN BRANCH: '$branch' does not exist at '$remote'!!!!" + echo "***" + exit 1 + fi + if ! git log -1 --oneline $remote_sha > /dev/null 2>&1 + then + echo "$0 says:" + echo "***" + echo "*** The top of '$branch' at '$remote' ($remote_sha)" + echo "*** does not exist locally!!!" + echo "*** You probably need to refresh local '$branch' and redo merge." + echo "***" + exit 1 + fi + if ! git merge-base --is-ancestor $remote_sha $local_sha + then + echo "$0 says:" + echo "***" + echo "*** FORCE push branch to '$remote' NOT ALLOWED!!!" + echo "***" + exit 1 + fi + if [ $remote_ref != refs/heads/master -a "$MASTER_ONLY" ] && git merge-base --is-ancestor $MASTER_ONLY $local_sha + then + echo "$0 says:" + echo "***" + echo "*** INVALID MERGE: Commit $MASTER_ONLY should not be reachable from '$branch'!!!!" + echo "*** You have probably merged master into '$branch' by mistake" + echo "***" + exit 1 + fi + if [ ${remote_ref#refs/heads/maint-} != $remote_ref ] && git merge-base --is-ancestor refs/remotes/$remote/maint $local_sha + then + echo "$0 says:" + echo "***" + echo "*** INVALID MERGE: Branch maint should not be reachable from '$branch'!!!!" + echo "*** You have probably merged maint into '$branch' by mistake." + echo "***" + exit 1 + fi + if [ $remote_ref = refs/heads/maint -o $remote_ref = refs/heads/master ]; then + for x in $RELEASES; do + if ! git merge-base --is-ancestor refs/remotes/$remote/maint-$x $local_sha; then + echo "$0 says:" + echo "***" + echo "*** WARNING: Branch '$remote/maint-$x' is not reachable from '$branch'!!!!" + echo "*** Someone needs to merge 'maint-$x' forward and push." + echo "***" + fi + done + fi + if [ $remote_ref = refs/heads/master ] && ! git merge-base --is-ancestor refs/remotes/$remote/maint $local_sha + then + echo "$0 says:" + echo "***" + echo "*** INVALID PUSH: Branch '$remote/maint' is not reachable from master!!!!" + echo "*** Someone needs to merge maint forward to master and push." + echo "***" + exit 1 + fi + NCOMMITS=`git rev-list --count $remote_sha..$local_sha` + if [ $NCOMMITS -gt $NCOMMITS_MAX ] + then + echo "$0 says:" + echo "***" + echo "*** HUGE push: $NCOMMITS commits (> $NCOMMITS_MAX) to '$branch' at '$remote' NOT ALLOWED!!!!" + echo "***" + exit 1 + fi + NFILES=`git diff --name-only $remote_sha $local_sha | wc --lines` + if [ $NFILES -gt $NFILES_MAX ] + then + echo "$0 says:" + echo "***" + echo "*** HUGE push: $NFILES changed files (> $NFILES_MAX) to '$branch' at '$remote' NOT ALLOWED!!!!" + echo "***" + exit 1 + fi + ;; + refs/tags/OTP-20.* | refs/tags/OTP-19.* | refs/tags/OTP-18.* | refs/tags/OTP-17.*) + tag=${remote_ref#refs/tags/} + if [ "$remote_sha" != $null ] + then + echo "$0 says:" + echo "***" + echo "*** FORCE push tag to '$remote' NOT ALLOWED!!!" + echo "*** Tag '$tag' already exists at '$remote'." + echo "***" + exit 1 + fi + ;; + refs/heads/*) + branch=${remote_ref#refs/heads/} + echo "$0 says:" + echo "***" + echo "*** UNKNOWN branch name: '$branch' pushed to '$remote' NOT ALLOWED!!!!" + echo "***" + exit 1 + ;; + refs/tags/*) + tag=${remote_ref#refs/tags/} + echo "$0 says:" + echo "***" + echo "*** UNKNOWN tag name: '$tag' pushed to '$remote' NOT ALLOWED!!!!" + echo "***" + exit 1 + ;; + *) + echo "$0 says:" + echo "***" + echo "*** STRANGE ref: '$remote_ref' pushed to '$remote' NOT ALLOWED!!!!" + echo "***" + exit 1 + ;; + esac + done +else + echo "$0: No checks done for remote '$remote' at $url." +fi + +exit 0 diff --git a/system/doc/efficiency_guide/advanced.xml b/system/doc/efficiency_guide/advanced.xml index e1760d0ded..bb4440a245 100644 --- a/system/doc/efficiency_guide/advanced.xml +++ b/system/doc/efficiency_guide/advanced.xml @@ -264,21 +264,26 @@ </row> <row> <cell><marker id="unique_integers"/>Unique Integers on a Runtime System Instance</cell> - <cell>There are two types of unique integers both created using the - <seealso marker="erts:erlang#unique_integer/1">erlang:unique_integer()</seealso> - BIF. Unique integers created: - <taglist> - <tag>with the <c>monotonic</c> modifier</tag> - <item>consist of a set of <c>2⁶⁴ - 1</c> unique integers.</item> - <tag>without the <c>monotonic</c> modifier</tag> - <item>consist of a set of <c>2⁶⁴ - 1</c> unique integers per scheduler - thread and a set of <c>2⁶⁴ - 1</c> unique integers shared by - other threads. That is the total amount of unique integers without - the <c>monotonic</c> modifier is <c>(NoSchedulers + 1) * (2⁶⁴ - 1)</c></item> - </taglist> - If a unique integer is created each nano second, unique integers - will at earliest be reused after more than 584 years. That is, for - the foreseeable future they are unique enough.</cell> + <cell> + There are two types of unique integers both created using the + <seealso marker="erts:erlang#unique_integer/1">erlang:unique_integer()</seealso> + BIF: + <br/><br/> + <em>1.</em> Unique integers created <em>with</em> the + <c>monotonic</c> modifier consist of a set of <c>2⁶⁴ - 1</c> + unique integers. + <br/><br/> + <em>2.</em> Unique integers created <em>without</em> the + <c>monotonic</c> modifier consist of a set of <c>2⁶⁴ - 1</c> + unique integers per scheduler thread and a set of <c>2⁶⁴ - 1</c> + unique integers shared by other threads. That is, the total + amount of unique integers without the <c>monotonic</c> modifier + is <c>(NoSchedulers + 1) × (2⁶⁴ - 1)</c>. + <br/><br/> + If a unique integer is created each nano second, unique integers + will at earliest be reused after more than 584 years. That is, for + the foreseeable future they are unique enough. + </cell> </row> <tcaption>System Limits</tcaption> </table> |