diff options
| -rw-r--r-- | erts/emulator/test/port_SUITE.erl | 25 | ||||
| -rw-r--r-- | lib/kernel/src/kernel.app.src | 2 | ||||
| -rw-r--r-- | lib/ssl/doc/src/ssl.xml | 31 | ||||
| -rw-r--r-- | lib/ssl/src/ssl.erl | 56 | ||||
| -rw-r--r-- | lib/ssl/src/ssl_connection.erl | 20 | ||||
| -rw-r--r-- | lib/ssl/src/ssl_handshake.erl | 45 | ||||
| -rw-r--r-- | lib/ssl/src/ssl_internal.hrl | 2 | ||||
| -rw-r--r-- | lib/ssl/src/tls_handshake.erl | 6 | ||||
| -rw-r--r-- | lib/ssl/src/tls_v1.erl | 33 | ||||
| -rw-r--r-- | lib/ssl/test/ssl_ECC_SUITE.erl | 232 | ||||
| -rw-r--r-- | lib/ssl/test/ssl_basic_SUITE.erl | 29 | ||||
| -rw-r--r-- | lib/stdlib/test/base64_SUITE.erl | 41 | ||||
| -rw-r--r-- | lib/stdlib/test/ets_SUITE.erl | 282 | ||||
| -rw-r--r-- | lib/stdlib/test/rand_SUITE.erl | 26 | ||||
| -rw-r--r-- | system/doc/design_principles/applications.xml | 160 |
15 files changed, 735 insertions, 255 deletions
diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl index 4323849465..d4e77d634a 100644 --- a/erts/emulator/test/port_SUITE.erl +++ b/erts/emulator/test/port_SUITE.erl @@ -976,21 +976,21 @@ try_bad_env(Env) -> %% Test that we can handle a very very large environment gracefully. huge_env(Config) when is_list(Config) -> ct:timetrap({minutes, 2}), - Vars = case os:type() of - {win32,_} -> 500; - _ -> - %% We create a huge environment, - %% 20000 variables is about 25MB - %% which seems to be the limit on Linux. - 20000 - end, + {Vars, Cmd} = case os:type() of + {win32,_} -> {500, "cmd /q /c ls"}; + _ -> + %% We create a huge environment, + %% 20000 variables is about 25MB + %% which seems to be the limit on Linux. + {20000, "ls"} + end, Env = [{[$a + I div (25*25*25*25) rem 25, $a + I div (25*25*25) rem 25, $a + I div (25*25) rem 25, $a+I div 25 rem 25, $a+I rem 25], lists:duplicate(100,$a+I rem 25)} || I <- lists:seq(1,Vars)], - try erlang:open_port({spawn,"ls"},[exit_status, {env, Env}]) of + try erlang:open_port({spawn,Cmd},[exit_status, {env, Env}]) of P -> receive {P, {exit_status,N}} = M -> @@ -1009,7 +1009,10 @@ huge_env(Config) when is_list(Config) -> %% Test to spawn program with command payload buffer %% just around pipe capacity (9f779819f6bda734c5953468f7798) pipe_limit_env(Config) when is_list(Config) -> - Cmd = "true", + Cmd = case os:type() of + {win32,_} -> "cmd /q /c true"; + _ -> "true" + end, CmdSize = command_payload_size(Cmd), Limits = [4096, 16384, 65536], % Try a couple of common pipe buffer sizes @@ -1026,7 +1029,7 @@ pipe_limit_env_do(Bytes, Cmd, CmdSize) -> try erlang:open_port({spawn,Cmd},[exit_status, {env, Env}]) of P -> receive - {P, {exit_status,N}} = M -> + {P, {exit_status,N}} -> %% Bug caused exit_status 150 (EINVAL+128) 0 = N end diff --git a/lib/kernel/src/kernel.app.src b/lib/kernel/src/kernel.app.src index 56d1699656..d184223524 100644 --- a/lib/kernel/src/kernel.app.src +++ b/lib/kernel/src/kernel.app.src @@ -118,6 +118,6 @@ {applications, []}, {env, [{error_logger, tty}]}, {mod, {kernel, []}}, - {runtime_dependencies, ["erts-8.0", "stdlib-3.0", "sasl-3.0"]} + {runtime_dependencies, ["erts-8.1", "stdlib-3.0", "sasl-3.0"]} ] }. diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index 68f2f97b6e..edc7e0d8b2 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -170,6 +170,14 @@ <tag><c>SNIfun::fun()</c></tag> <item><p><c>= fun(ServerName :: string()) -> [ssl_option()]</c></p></item> + <tag><c>named_curve() =</c></tag> + <item><p><c>sect571r1 | sect571k1 | secp521r1 | brainpoolP512r1 + | sect409k1 | sect409r1 | brainpoolP384r1 | secp384r1 + | sect283k1 | sect283r1 | brainpoolP256r1 | secp256k1 | secp256r1 + | sect239k1 | sect233k1 | sect233r1 | secp224k1 | secp224r1 + | sect193r1 | sect193r2 | secp192k1 | secp192r1 | sect163k1 + | sect163r1 | sect163r2 | secp160k1 | secp160r1 | secp160r2</c></p></item> + </taglist> </section> @@ -217,6 +225,11 @@ Anonymous cipher suites are supported for testing purposes only and are not be used when security matters.</p></item> + <tag><c>{eccs, [named_curve()]}</c></tag> + <item><p> Allows to specify the order of preference for named curves + and to restrict their usage when using a cipher suite supporting them. + </p></item> + <tag><c>{secure_renegotiate, boolean()}</c></tag> <item><p>Specifies if to reject renegotiation attempt that does not live up to @@ -751,6 +764,11 @@ fun(srp, Username :: string(), UserState :: term()) -> (the default), use the client's preference. </item> + <tag><c>{honor_ecc_order, boolean()}</c></tag> + <item>If true, use the server's preference for ECC curve selection. If false + (the default), use the client's preference. + </item> + <tag><c>{signature_algs, [{hash(), ecdsa | rsa | dsa}]}</c></tag> <item><p> The algorithms specified by this option will be the ones accepted by the server in a signature algorithm @@ -804,6 +822,17 @@ fun(srp, Username :: string(), UserState :: term()) -> </func> <func> + <name>eccs() -></name> + <name>eccs(protocol()) -> [named_curve()]</name> + <fsummary>Returns a list of supported ECCs.</fsummary> + + <desc><p>Returns a list of supported ECCs. <c>eccs()</c> + is equivalent to calling <c>eccs(Protocol)</c> with all + supported protocols and then deduplicating the output.</p> + </desc> + </func> + + <func> <name>clear_pem_cache() -> ok </name> <fsummary> Clears the pem cache</fsummary> @@ -898,7 +927,7 @@ fun(srp, Username :: string(), UserState :: term()) -> <fsummary>Returns all the connection information. </fsummary> <type> - <v>Item = protocol | cipher_suite | sni_hostname | atom()</v> + <v>Item = protocol | cipher_suite | sni_hostname | ecc | atom()</v> <d>Meaningful atoms, not specified above, are the ssl option names.</d> <v>Result = [{Item::atom(), Value::term()}]</v> <v>Reason = term()</v> diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index 27b753af2e..aa62ab8865 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -38,7 +38,7 @@ getopts/2, setopts/2, getstat/1, getstat/2 ]). %% SSL/TLS protocol handling --export([cipher_suites/0, cipher_suites/1, +-export([cipher_suites/0, cipher_suites/1, eccs/0, eccs/1, connection_info/1, versions/0, session_info/1, format_error/1, renegotiate/1, prf/5, negotiated_protocol/1, negotiated_next_protocol/1, connection_information/1, connection_information/2]). @@ -420,6 +420,33 @@ cipher_suites(all) -> [ssl_cipher:erl_suite_definition(Suite) || Suite <- available_suites(all)]. %%-------------------------------------------------------------------- +-spec eccs() -> tls_v1:curves(). +%% Description: returns all supported curves across all versions +%%-------------------------------------------------------------------- +eccs() -> + Curves = tls_v1:ecc_curves(all), % only tls_v1 has named curves right now + eccs_filter_supported(Curves). + +%%-------------------------------------------------------------------- +-spec eccs(tls_record:tls_version() | tls_record:tls_atom_version()) -> + tls_v1:curves(). +%% Description: returns the curves supported for a given version of +%% ssl/tls. +%%-------------------------------------------------------------------- +eccs({3,0}) -> + []; +eccs({3,_}) -> + Curves = tls_v1:ecc_curves(all), + eccs_filter_supported(Curves); +eccs(AtomVersion) when is_atom(AtomVersion) -> + eccs(tls_record:protocol_version(AtomVersion)). + +eccs_filter_supported(Curves) -> + CryptoCurves = crypto:ec_curves(), + lists:filter(fun(Curve) -> proplists:get_bool(Curve, CryptoCurves) end, + Curves). + +%%-------------------------------------------------------------------- -spec getopts(#sslsocket{}, [gen_tcp:option_name()]) -> {ok, [gen_tcp:option()]} | {error, reason()}. %% @@ -647,6 +674,8 @@ do_connect(Address, Port, end. %% Handle extra ssl options given to ssl_accept +-spec handle_options([any()], #ssl_options{}) -> #ssl_options{} + ; ([any()], client | server) -> {ok, #config{}}. handle_options(Opts0, #ssl_options{protocol = Protocol, cacerts = CaCerts0, cacertfile = CaCertFile0} = InheritedSslOpts) -> RecordCB = record_cb(Protocol), @@ -725,6 +754,8 @@ handle_options(Opts0, Role) -> srp_identity = handle_option(srp_identity, Opts, undefined), ciphers = handle_cipher_option(proplists:get_value(ciphers, Opts, []), RecordCb:highest_protocol_version(Versions)), + eccs = handle_eccs_option(proplists:get_value(eccs, Opts, eccs()), + RecordCb:highest_protocol_version(Versions)), signature_algs = handle_hashsigns_option(proplists:get_value(signature_algs, Opts, default_option_role(server, tls_v1:default_signature_algs(Versions), Role)), @@ -755,6 +786,9 @@ handle_options(Opts0, Role) -> honor_cipher_order = handle_option(honor_cipher_order, Opts, default_option_role(server, false, Role), server, Role), + honor_ecc_order = handle_option(honor_ecc_order, Opts, + default_option_role(server, false, Role), + server, Role), protocol = proplists:get_value(protocol, Opts, tls), padding_check = proplists:get_value(padding_check, Opts, true), beast_mitigation = handle_option(beast_mitigation, Opts, one_n_minus_one), @@ -780,7 +814,7 @@ handle_options(Opts0, Role) -> alpn_preferred_protocols, next_protocols_advertised, client_preferred_next_protocols, log_alert, server_name_indication, honor_cipher_order, padding_check, crl_check, crl_cache, - fallback, signature_algs, beast_mitigation, v2_hello_compatible], + fallback, signature_algs, eccs, honor_ecc_order, beast_mitigation, v2_hello_compatible], SockOpts = lists:foldl(fun(Key, PropList) -> proplists:delete(Key, PropList) @@ -1010,6 +1044,8 @@ validate_option(sni_fun, Fun) when is_function(Fun) -> Fun; validate_option(honor_cipher_order, Value) when is_boolean(Value) -> Value; +validate_option(honor_ecc_order, Value) when is_boolean(Value) -> + Value; validate_option(padding_check, Value) when is_boolean(Value) -> Value; validate_option(fallback, Value) when is_boolean(Value) -> @@ -1164,6 +1200,14 @@ binary_cipher_suites(Version, Ciphers0) -> Ciphers = [ssl_cipher:openssl_suite(C) || C <- string:tokens(Ciphers0, ":")], binary_cipher_suites(Version, Ciphers). +handle_eccs_option(Value, {_Major, Minor}) when is_list(Value) -> + try tls_v1:ecc_curves(Minor, Value) of + Curves -> #elliptic_curves{elliptic_curve_list = Curves} + catch + exit:_ -> throw({error, {options, {eccs, Value}}}); + error:_ -> throw({error, {options, {eccs, Value}}}) + end. + unexpected_format(Error) -> lists:flatten(io_lib:format("Unexpected error: ~p", [Error])). @@ -1334,6 +1378,14 @@ new_ssl_options([{server_name_indication, Value} | Rest], #ssl_options{} = Opts, new_ssl_options(Rest, Opts#ssl_options{server_name_indication = validate_option(server_name_indication, Value)}, RecordCB); new_ssl_options([{honor_cipher_order, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> new_ssl_options(Rest, Opts#ssl_options{honor_cipher_order = validate_option(honor_cipher_order, Value)}, RecordCB); +new_ssl_options([{honor_ecc_order, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> + new_ssl_options(Rest, Opts#ssl_options{honor_ecc_order = validate_option(honor_ecc_order, Value)}, RecordCB); +new_ssl_options([{eccs, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> + new_ssl_options(Rest, + Opts#ssl_options{eccs = + handle_eccs_option(Value, RecordCB:highest_protocol_version()) + }, + RecordCB); new_ssl_options([{signature_algs, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> new_ssl_options(Rest, Opts#ssl_options{signature_algs = diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index 08fca76123..b6e4d5b433 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -1172,14 +1172,23 @@ handle_alert(#alert{level = ?WARNING} = Alert, StateName, %%% Internal functions %%-------------------------------------------------------------------- connection_info(#state{sni_hostname = SNIHostname, - session = #session{cipher_suite = CipherSuite}, + session = #session{cipher_suite = CipherSuite, ecc = ECCCurve}, protocol_cb = Connection, negotiated_version = {_,_} = Version, ssl_options = Opts}) -> RecordCB = record_cb(Connection), + CipherSuiteDef = ssl_cipher:erl_suite_definition(CipherSuite), + IsNamedCurveSuite = lists:member(element(1,CipherSuiteDef), + [ecdh_ecdsa, ecdhe_ecdsa, ecdh_anon]), + CurveInfo = case ECCCurve of + {namedCurve, Curve} when IsNamedCurveSuite -> + [{ecc, {named_curve, pubkey_cert_records:namedCurves(Curve)}}]; + _ -> + [] + end, [{protocol, RecordCB:protocol_version(Version)}, - {cipher_suite, ssl_cipher:erl_suite_definition(CipherSuite)}, - {sni_hostname, SNIHostname}] ++ ssl_options_list(Opts). + {cipher_suite, CipherSuiteDef}, + {sni_hostname, SNIHostname} | CurveInfo] ++ ssl_options_list(Opts). do_server_hello(Type, #hello_extensions{next_protocol_negotiation = NextProtocols} = ServerHelloExt, @@ -1741,12 +1750,13 @@ calculate_secret(#server_dh_params{dh_p = Prime, dh_g = Base, Connection, certify, certify); calculate_secret(#server_ecdh_params{curve = ECCurve, public = ECServerPubKey}, - State, Connection) -> + State=#state{session=Session}, Connection) -> ECDHKeys = public_key:generate_key(ECCurve), PremasterSecret = ssl_handshake:premaster_secret(#'ECPoint'{point = ECServerPubKey}, ECDHKeys), calculate_master_secret(PremasterSecret, - State#state{diffie_hellman_keys = ECDHKeys}, + State#state{diffie_hellman_keys = ECDHKeys, + session = Session#session{ecc = ECCurve}}, Connection, certify, certify); calculate_secret(#server_psk_params{ diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index 5b51ac0916..4acc745c5f 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -70,7 +70,7 @@ %% Extensions handling -export([client_hello_extensions/6, handle_client_hello_extensions/9, %% Returns server hello extensions - handle_server_hello_extensions/9, select_curve/2 + handle_server_hello_extensions/9, select_curve/2, select_curve/3 ]). %% MISC @@ -120,11 +120,13 @@ server_hello_done() -> #server_hello_done{}. client_hello_extensions(Host, Version, CipherSuites, - #ssl_options{signature_algs = SupportedHashSigns, versions = AllVersions} = SslOpts, ConnectionStates, Renegotiation) -> + #ssl_options{signature_algs = SupportedHashSigns, + eccs = SupportedECCs, + versions = AllVersions} = SslOpts, ConnectionStates, Renegotiation) -> {EcPointFormats, EllipticCurves} = case advertises_ec_ciphers(lists:map(fun ssl_cipher:suite_definition/1, CipherSuites)) of true -> - client_ecc_extensions(tls_v1, Version); + client_ecc_extensions(SupportedECCs); false -> {undefined, undefined} end, @@ -1169,8 +1171,9 @@ select_session(SuggestedSessionId, CipherSuites, HashSigns, Compressions, Port, {resumed, Resumed} end. -supported_ecc({Major, Minor} = Version) when ((Major == 3) and (Minor >= 1)) orelse (Major > 3) -> - Curves = tls_v1:ecc_curves(Version), +%% Deprecated? +supported_ecc({Major, Minor}) when ((Major == 3) and (Minor >= 1)) orelse (Major > 3) -> + Curves = tls_v1:ecc_curves(Minor), #elliptic_curves{elliptic_curve_list = Curves}; supported_ecc(_) -> #elliptic_curves{elliptic_curve_list = []}. @@ -1454,12 +1457,12 @@ srp_user(#ssl_options{srp_identity = {UserName, _}}) -> srp_user(_) -> undefined. -client_ecc_extensions(Module, Version) -> +client_ecc_extensions(SupportedECCs) -> CryptoSupport = proplists:get_value(public_keys, crypto:supports()), case proplists:get_bool(ecdh, CryptoSupport) of true -> EcPointFormats = #ec_point_formats{ec_point_format_list = [?ECPOINT_UNCOMPRESSED]}, - EllipticCurves = #elliptic_curves{elliptic_curve_list = Module:ecc_curves(Version)}, + EllipticCurves = SupportedECCs, {EcPointFormats, EllipticCurves}; _ -> {undefined, undefined} @@ -1493,22 +1496,34 @@ advertises_ec_ciphers([{ecdh_anon, _,_,_} | _]) -> true; advertises_ec_ciphers([_| Rest]) -> advertises_ec_ciphers(Rest). -select_curve(#elliptic_curves{elliptic_curve_list = ClientCurves}, - #elliptic_curves{elliptic_curve_list = ServerCurves}) -> - select_curve(ClientCurves, ServerCurves); -select_curve(undefined, _) -> + +select_curve(Client, Server) -> + select_curve(Client, Server, false). + +select_curve(#elliptic_curves{elliptic_curve_list = ClientCurves}, + #elliptic_curves{elliptic_curve_list = ServerCurves}, + ServerOrder) -> + case ServerOrder of + false -> + select_shared_curve(ClientCurves, ServerCurves); + true -> + select_shared_curve(ServerCurves, ClientCurves) + end; +select_curve(undefined, _, _) -> %% Client did not send ECC extension use default curve if %% ECC cipher is negotiated - {namedCurve, ?secp256r1}; -select_curve(_, []) -> + {namedCurve, ?secp256r1}. + +select_shared_curve([], _) -> no_curve; -select_curve(Curves, [Curve| Rest]) -> +select_shared_curve([Curve | Rest], Curves) -> case lists:member(Curve, Curves) of true -> {namedCurve, Curve}; false -> - select_curve(Curves, Rest) + select_shared_curve(Rest, Curves) end. + %% RFC 6066, Section 3: Currently, the only server names supported are %% DNS hostnames sni(_, disable) -> diff --git a/lib/ssl/src/ssl_internal.hrl b/lib/ssl/src/ssl_internal.hrl index c19c1787ff..487d1fa096 100644 --- a/lib/ssl/src/ssl_internal.hrl +++ b/lib/ssl/src/ssl_internal.hrl @@ -140,6 +140,8 @@ crl_check :: boolean() | peer | best_effort, crl_cache, signature_algs, + eccs, + honor_ecc_order :: boolean(), v2_hello_compatible :: boolean() }). diff --git a/lib/ssl/src/tls_handshake.erl b/lib/ssl/src/tls_handshake.erl index a2486bf752..2bd103c18a 100644 --- a/lib/ssl/src/tls_handshake.erl +++ b/lib/ssl/src/tls_handshake.erl @@ -160,13 +160,15 @@ handle_client_hello(Version, #client_hello{session_id = SugesstedId, extensions = #hello_extensions{elliptic_curves = Curves, signature_algs = ClientHashSigns} = HelloExt}, #ssl_options{versions = Versions, - signature_algs = SupportedHashSigns} = SslOpts, + signature_algs = SupportedHashSigns, + eccs = SupportedECCs, + honor_ecc_order = ECCOrder} = SslOpts, {Port, Session0, Cache, CacheCb, ConnectionStates0, Cert, _}, Renegotiation) -> case tls_record:is_acceptable_version(Version, Versions) of true -> AvailableHashSigns = ssl_handshake:available_signature_algs( ClientHashSigns, SupportedHashSigns, Cert, Version), - ECCCurve = ssl_handshake:select_curve(Curves, ssl_handshake:supported_ecc(Version)), + ECCCurve = ssl_handshake:select_curve(Curves, SupportedECCs, ECCOrder), {Type, #session{cipher_suite = CipherSuite} = Session1} = ssl_handshake:select_session(SugesstedId, CipherSuites, AvailableHashSigns, Compressions, Port, Session0#session{ecc = ECCCurve}, Version, diff --git a/lib/ssl/src/tls_v1.erl b/lib/ssl/src/tls_v1.erl index 711db77708..7f24ce5192 100644 --- a/lib/ssl/src/tls_v1.erl +++ b/lib/ssl/src/tls_v1.erl @@ -31,9 +31,18 @@ -export([master_secret/4, finished/5, certificate_verify/3, mac_hash/7, setup_keys/8, suites/1, prf/5, - ecc_curves/1, oid_to_enum/1, enum_to_oid/1, + ecc_curves/1, ecc_curves/2, oid_to_enum/1, enum_to_oid/1, default_signature_algs/1, signature_algs/2]). +-type named_curve() :: sect571r1 | sect571k1 | secp521r1 | brainpoolP512r1 | + sect409k1 | sect409r1 | brainpoolP384r1 | secp384r1 | + sect283k1 | sect283r1 | brainpoolP256r1 | secp256k1 | secp256r1 | + sect239k1 | sect233k1 | sect233r1 | secp224k1 | secp224r1 | + sect193r1 | sect193r2 | secp192k1 | secp192r1 | sect163k1 | + sect163r1 | sect163r2 | secp160k1 | secp160r1 | secp160r2. +-type curves() :: [named_curve()]. +-export_type([curves/0, named_curve/0]). + %%==================================================================== %% Internal application API %%==================================================================== @@ -399,13 +408,20 @@ is_pair(Hash, rsa, Hashs) -> lists:member(Hash, AtLeastMd5). %% list ECC curves in prefered order -ecc_curves(_Minor) -> - TLSCurves = [sect571r1,sect571k1,secp521r1,brainpoolP512r1, - sect409k1,sect409r1,brainpoolP384r1,secp384r1, - sect283k1,sect283r1,brainpoolP256r1,secp256k1,secp256r1, - sect239k1,sect233k1,sect233r1,secp224k1,secp224r1, - sect193r1,sect193r2,secp192k1,secp192r1,sect163k1, - sect163r1,sect163r2,secp160k1,secp160r1,secp160r2], +-spec ecc_curves(1..3 | all) -> [named_curve()]. +ecc_curves(all) -> + [sect571r1,sect571k1,secp521r1,brainpoolP512r1, + sect409k1,sect409r1,brainpoolP384r1,secp384r1, + sect283k1,sect283r1,brainpoolP256r1,secp256k1,secp256r1, + sect239k1,sect233k1,sect233r1,secp224k1,secp224r1, + sect193r1,sect193r2,secp192k1,secp192r1,sect163k1, + sect163r1,sect163r2,secp160k1,secp160r1,secp160r2]; +ecc_curves(Minor) -> + TLSCurves = ecc_curves(all), + ecc_curves(Minor, TLSCurves). + +-spec ecc_curves(1..3, [named_curve()]) -> [named_curve()]. +ecc_curves(_Minor, TLSCurves) -> CryptoCurves = crypto:ec_curves(), lists:foldr(fun(Curve, Curves) -> case proplists:get_bool(Curve, CryptoCurves) of @@ -414,6 +430,7 @@ ecc_curves(_Minor) -> end end, [], TLSCurves). + %% ECC curves from draft-ietf-tls-ecc-12.txt (Oct. 17, 2005) oid_to_enum(?sect163k1) -> 1; oid_to_enum(?sect163r1) -> 2; diff --git a/lib/ssl/test/ssl_ECC_SUITE.erl b/lib/ssl/test/ssl_ECC_SUITE.erl index 258922d128..bd0c630d41 100644 --- a/lib/ssl/test/ssl_ECC_SUITE.erl +++ b/lib/ssl/test/ssl_ECC_SUITE.erl @@ -46,7 +46,7 @@ groups() -> {'tlsv1', [], all_versions_groups()}, {'erlang_server', [], key_cert_combinations()}, {'erlang_client', [], key_cert_combinations()}, - {'erlang', [], key_cert_combinations() ++ misc()} + {'erlang', [], key_cert_combinations() ++ misc() ++ ecc_negotiation()} ]. all_versions_groups ()-> @@ -68,6 +68,23 @@ key_cert_combinations() -> misc()-> [client_ecdsa_server_ecdsa_with_raw_key]. +ecc_negotiation() -> + [ecc_default_order, + ecc_default_order_custom_curves, + ecc_client_order, + ecc_client_order_custom_curves, + ecc_unknown_curve, + client_ecdh_server_ecdh_ecc_server_custom, + client_rsa_server_ecdh_ecc_server_custom, + client_ecdh_server_rsa_ecc_server_custom, + client_rsa_server_rsa_ecc_server_custom, + client_ecdsa_server_ecdsa_ecc_server_custom, + client_ecdsa_server_rsa_ecc_server_custom, + client_rsa_server_ecdsa_ecc_server_custom, + client_ecdsa_server_ecdsa_ecc_client_custom, + client_rsa_server_ecdsa_ecc_client_custom + ]. + %%-------------------------------------------------------------------- init_per_suite(Config0) -> end_per_suite(Config0), @@ -218,6 +235,132 @@ client_ecdsa_server_ecdsa_with_raw_key(Config) when is_list(Config) -> check_result(Server, SType, Client, CType), close(Server, Client). +ecc_default_order(Config) -> + COpts = proplists:get_value(client_ecdsa_opts, Config), + SOpts = proplists:get_value(server_ecdsa_opts, Config), + ECCOpts = [], + case supported_eccs([{eccs, [sect571r1]}]) of + true -> ecc_test(sect571r1, COpts, SOpts, [], ECCOpts, Config); + false -> {skip, "unsupported named curves"} + end. + +ecc_default_order_custom_curves(Config) -> + COpts = proplists:get_value(client_ecdsa_opts, Config), + SOpts = proplists:get_value(server_ecdsa_opts, Config), + ECCOpts = [{eccs, [secp256r1, sect571r1]}], + case supported_eccs(ECCOpts) of + true -> ecc_test(sect571r1, COpts, SOpts, [], ECCOpts, Config); + false -> {skip, "unsupported named curves"} + end. + +ecc_client_order(Config) -> + COpts = proplists:get_value(client_ecdsa_opts, Config), + SOpts = proplists:get_value(server_ecdsa_opts, Config), + ECCOpts = [{honor_ecc_order, false}], + case supported_eccs([{eccs, [sect571r1]}]) of + true -> ecc_test(sect571r1, COpts, SOpts, [], ECCOpts, Config); + false -> {skip, "unsupported named curves"} + end. + +ecc_client_order_custom_curves(Config) -> + COpts = proplists:get_value(client_ecdsa_opts, Config), + SOpts = proplists:get_value(server_ecdsa_opts, Config), + ECCOpts = [{honor_ecc_order, false}, {eccs, [secp256r1, sect571r1]}], + case supported_eccs(ECCOpts) of + true -> ecc_test(sect571r1, COpts, SOpts, [], ECCOpts, Config); + false -> {skip, "unsupported named curves"} + end. + +ecc_unknown_curve(Config) -> + COpts = proplists:get_value(client_ecdsa_opts, Config), + SOpts = proplists:get_value(server_ecdsa_opts, Config), + ECCOpts = [{eccs, ['123_fake_curve']}], + ecc_test_error(COpts, SOpts, [], ECCOpts, Config). + +%% We can only expect to see a named curve on a conn with +%% a server supporting ecdsa. Otherwise the curve is selected +%% but not used and communicated to the client? +client_ecdh_server_ecdh_ecc_server_custom(Config) -> + COpts = proplists:get_value(client_ecdh_rsa_opts, Config), + SOpts = proplists:get_value(server_ecdh_rsa_opts, Config), + ECCOpts = [{honor_ecc_order, true}, {eccs, [secp256r1, sect571r1]}], + case supported_eccs(ECCOpts) of + true -> ecc_test(undefined, COpts, SOpts, [], ECCOpts, Config); + false -> {skip, "unsupported named curves"} + end. + +client_ecdh_server_rsa_ecc_server_custom(Config) -> + COpts = proplists:get_value(client_ecdh_rsa_opts, Config), + SOpts = proplists:get_value(server_opts, Config), + ECCOpts = [{honor_ecc_order, true}, {eccs, [secp256r1, sect571r1]}], + case supported_eccs(ECCOpts) of + true -> ecc_test(undefined, COpts, SOpts, [], ECCOpts, Config); + false -> {skip, "unsupported named curves"} + end. + +client_rsa_server_ecdh_ecc_server_custom(Config) -> + COpts = proplists:get_value(client_opts, Config), + SOpts = proplists:get_value(server_ecdh_rsa_opts, Config), + ECCOpts = [{honor_ecc_order, true}, {eccs, [secp256r1, sect571r1]}], + case supported_eccs(ECCOpts) of + true -> ecc_test(undefined, COpts, SOpts, [], ECCOpts, Config); + false -> {skip, "unsupported named curves"} + end. + +client_rsa_server_rsa_ecc_server_custom(Config) -> + COpts = proplists:get_value(client_opts, Config), + SOpts = proplists:get_value(server_opts, Config), + ECCOpts = [{honor_ecc_order, true}, {eccs, [secp256r1, sect571r1]}], + case supported_eccs(ECCOpts) of + true -> ecc_test(undefined, COpts, SOpts, [], ECCOpts, Config); + false -> {skip, "unsupported named curves"} + end. + +client_ecdsa_server_ecdsa_ecc_server_custom(Config) -> + COpts = proplists:get_value(client_ecdsa_opts, Config), + SOpts = proplists:get_value(server_ecdsa_opts, Config), + ECCOpts = [{honor_ecc_order, true}, {eccs, [secp256r1, sect571r1]}], + case supported_eccs(ECCOpts) of + true -> ecc_test(secp256r1, COpts, SOpts, [], ECCOpts, Config); + false -> {skip, "unsupported named curves"} + end. + +client_ecdsa_server_rsa_ecc_server_custom(Config) -> + COpts = proplists:get_value(client_ecdsa_opts, Config), + SOpts = proplists:get_value(server_opts, Config), + ECCOpts = [{honor_ecc_order, true}, {eccs, [secp256r1, sect571r1]}], + case supported_eccs(ECCOpts) of + true -> ecc_test(undefined, COpts, SOpts, [], ECCOpts, Config); + false -> {skip, "unsupported named curves"} + end. + +client_rsa_server_ecdsa_ecc_server_custom(Config) -> + COpts = proplists:get_value(client_opts, Config), + SOpts = proplists:get_value(server_ecdsa_opts, Config), + ECCOpts = [{honor_ecc_order, true}, {eccs, [secp256r1, sect571r1]}], + case supported_eccs(ECCOpts) of + true -> ecc_test(secp256r1, COpts, SOpts, [], ECCOpts, Config); + false -> {skip, "unsupported named curves"} + end. + +client_ecdsa_server_ecdsa_ecc_client_custom(Config) -> + COpts = proplists:get_value(client_ecdsa_opts, Config), + SOpts = proplists:get_value(server_ecdsa_opts, Config), + ECCOpts = [{eccs, [secp256r1, sect571r1]}], + case supported_eccs(ECCOpts) of + true -> ecc_test(secp256r1, COpts, SOpts, ECCOpts, [], Config); + false -> {skip, "unsupported named curves"} + end. + +client_rsa_server_ecdsa_ecc_client_custom(Config) -> + COpts = proplists:get_value(client_opts, Config), + SOpts = proplists:get_value(server_ecdsa_opts, Config), + ECCOpts = [{eccs, [secp256r1, sect571r1]}], + case supported_eccs(ECCOpts) of + true -> ecc_test(secp256r1, COpts, SOpts, ECCOpts, [], Config); + false -> {skip, "unsupported named curves"} + end. + %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- @@ -244,6 +387,30 @@ basic_test(ClientCert, ClientKey, ClientCA, ServerCert, ServerKey, ServerCA, Con check_result(Server, SType, Client, CType), close(Server, Client). +ecc_test(Expect, COpts, SOpts, CECCOpts, SECCOpts, Config) -> + CCA = proplists:get_value(cacertfile, COpts), + CCert = proplists:get_value(certfile, COpts), + CKey = proplists:get_value(keyfile, COpts), + SCA = proplists:get_value(cacertfile, SOpts), + SCert = proplists:get_value(certfile, SOpts), + SKey = proplists:get_value(keyfile, SOpts), + {Server, Port} = start_server_ecc(erlang, CCA, SCA, SCert, SKey, Expect, SECCOpts, Config), + Client = start_client_ecc(erlang, Port, SCA, CCA, CCert, CKey, Expect, CECCOpts, Config), + ssl_test_lib:check_result(Server, ok, Client, ok), + close(Server, Client). + +ecc_test_error(COpts, SOpts, CECCOpts, SECCOpts, Config) -> + CCA = proplists:get_value(cacertfile, COpts), + CCert = proplists:get_value(certfile, COpts), + CKey = proplists:get_value(keyfile, COpts), + SCA = proplists:get_value(cacertfile, SOpts), + SCert = proplists:get_value(certfile, SOpts), + SKey = proplists:get_value(keyfile, SOpts), + {Server, Port} = start_server_ecc_error(erlang, CCA, SCA, SCert, SKey, SECCOpts, Config), + Client = start_client_ecc_error(erlang, Port, SCA, CCA, CCert, CKey, CECCOpts, Config), + Error = {error, {tls_alert, "insufficient security"}}, + ssl_test_lib:check_result(Server, Error, Client, Error). + start_client(openssl, Port, PeerCA, OwnCa, Cert, Key, _Config) -> CA = new_openssl_ca("openssl_client_ca", PeerCA, OwnCa), Version = tls_record:protocol_version(tls_record:highest_protocol_version([])), @@ -267,6 +434,31 @@ start_client(erlang, Port, PeerCA, OwnCa, Cert, Key, Config) -> {cacertfile, CA}, {certfile, Cert}, {keyfile, Key}]}]). +start_client_ecc(erlang, Port, PeerCA, OwnCa, Cert, Key, Expect, ECCOpts, Config) -> + CA = new_ca("erlang_client_ca", PeerCA, OwnCa), + {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), + ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, check_ecc, [client, Expect]}}, + {options, + ECCOpts ++ + [{verify, verify_peer}, + {cacertfile, CA}, + {certfile, Cert}, {keyfile, Key}]}]). + +start_client_ecc_error(erlang, Port, PeerCA, OwnCa, Cert, Key, ECCOpts, Config) -> + CA = new_ca("erlang_client_ca", PeerCA, OwnCa), + {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), + ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {options, + ECCOpts ++ + [{verify, verify_peer}, + {cacertfile, CA}, + {certfile, Cert}, {keyfile, Key}]}]). + start_server(openssl, PeerCA, OwnCa, Cert, Key, _Config) -> CA = new_openssl_ca("openssl_server_ca", PeerCA, OwnCa), Port = ssl_test_lib:inet_port(node()), @@ -290,6 +482,7 @@ start_server(erlang, PeerCA, OwnCa, Cert, Key, Config) -> [{verify, verify_peer}, {cacertfile, CA}, {certfile, Cert}, {keyfile, Key}]}]), {Server, ssl_test_lib:inet_port(Server)}. + start_server_with_raw_key(erlang, PeerCA, OwnCa, Cert, Key, Config) -> CA = new_ca("erlang_server_ca", PeerCA, OwnCa), {_, ServerNode, _} = ssl_test_lib:run_where(Config), @@ -303,6 +496,29 @@ start_server_with_raw_key(erlang, PeerCA, OwnCa, Cert, Key, Config) -> {certfile, Cert}, {key, Key}]}]), {Server, ssl_test_lib:inet_port(Server)}. +start_server_ecc(erlang, PeerCA, OwnCa, Cert, Key, Expect, ECCOpts, Config) -> + CA = new_ca("erlang_server_ca", PeerCA, OwnCa), + {_, ServerNode, _} = ssl_test_lib:run_where(Config), + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, check_ecc, [server, Expect]}}, + {options, + ECCOpts ++ + [{verify, verify_peer}, {cacertfile, CA}, + {certfile, Cert}, {keyfile, Key}]}]), + {Server, ssl_test_lib:inet_port(Server)}. + +start_server_ecc_error(erlang, PeerCA, OwnCa, Cert, Key, ECCOpts, Config) -> + CA = new_ca("erlang_server_ca", PeerCA, OwnCa), + {_, ServerNode, _} = ssl_test_lib:run_where(Config), + Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, + {from, self()}, + {options, + ECCOpts ++ + [{verify, verify_peer}, {cacertfile, CA}, + {certfile, Cert}, {keyfile, Key}]}]), + {Server, ssl_test_lib:inet_port(Server)}. + check_result(Server, erlang, Client, erlang) -> ssl_test_lib:check_result(Server, ok, Client, ok); check_result(Server, erlang, _, _) -> @@ -362,3 +578,17 @@ new_openssl_ca(FileName, CA, OwnCa) -> file:write_file(FileName, Pem) end, FileName. + +supported_eccs(Opts) -> + ToCheck = proplists:get_value(eccs, Opts, []), + Supported = ssl:eccs(), + lists:all(fun(Curve) -> lists:member(Curve, Supported) end, ToCheck). + +check_ecc(SSL, Role, Expect) -> + {ok, Data} = ssl:connection_information(SSL), + case lists:keyfind(ecc, 1, Data) of + {ecc, {named_curve, Expect}} -> ok; + false when Expect =:= undefined -> ok; + Other -> {error, Role, Expect, Other} + end. + diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index 1be43c56c4..f8dea736ae 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -150,6 +150,7 @@ api_tests() -> peercert_with_client_cert, sockname, versions, + eccs, controlling_process, getstat, close_with_timeout, @@ -456,6 +457,15 @@ init_per_testcase(accept_pool, Config) -> init_per_testcase(controller_dies, Config) -> ct:timetrap({seconds, 10}), Config; +init_per_testcase(eccs, Config) -> + case ssl:eccs() of + [] -> + {skip, "named curves not supported"}; + [_|_] -> + ssl_test_lib:ct_log_supported_protocol_versions(Config), + ct:timetrap({seconds, 5}), + Config + end; init_per_testcase(_TestCase, Config) -> ssl_test_lib:ct_log_supported_protocol_versions(Config), ct:timetrap({seconds, 5}), @@ -1504,6 +1514,25 @@ versions(Config) when is_list(Config) -> [_|_] = Versions = ssl:versions(), ct:log("~p~n", [Versions]). + +%%-------------------------------------------------------------------- +eccs() -> + [{doc, "Test API functions eccs/0 and eccs/1"}]. + +eccs(Config) when is_list(Config) -> + [_|_] = All = ssl:eccs(), + [] = SSL3 = ssl:eccs({3,0}), + [_|_] = Tls = ssl:eccs({3,1}), + [_|_] = Tls1 = ssl:eccs({3,2}), + [_|_] = Tls2 = ssl:eccs({3,3}), + [] = SSL3 = ssl:eccs(sslv3), + [_|_] = Tls = ssl:eccs(tlsv1), + [_|_] = Tls1 = ssl:eccs('tlsv1.1'), + [_|_] = Tls2 = ssl:eccs('tlsv1.2'), + %% ordering is currently unverified by the test + true = lists:sort(All) =:= lists:usort(SSL3 ++ Tls ++ Tls1 ++ Tls2), + ok. + %%-------------------------------------------------------------------- send_recv() -> [{doc,""}]. diff --git a/lib/stdlib/test/base64_SUITE.erl b/lib/stdlib/test/base64_SUITE.erl index 9176a3664a..d0abe5c961 100644 --- a/lib/stdlib/test/base64_SUITE.erl +++ b/lib/stdlib/test/base64_SUITE.erl @@ -23,9 +23,7 @@ -include_lib("common_test/include/ct.hrl"). %% Test server specific exports --export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, - init_per_group/2,end_per_group/2, - init_per_testcase/2, end_per_testcase/2]). +-export([all/0, suite/0, groups/0, group/1]). %% Test cases must be exported. -export([base64_encode/1, base64_decode/1, base64_otp_5635/1, @@ -33,41 +31,26 @@ mime_decode_to_string/1, roundtrip_1/1, roundtrip_2/1, roundtrip_3/1, roundtrip_4/1]). -init_per_testcase(_, Config) -> - Config. - -end_per_testcase(_, _Config) -> - ok. - %%------------------------------------------------------------------------- %% Test cases starts here. %%------------------------------------------------------------------------- + suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap,{minutes,4}}]. -all() -> +all() -> [base64_encode, base64_decode, base64_otp_5635, base64_otp_6279, big, illegal, mime_decode, mime_decode_to_string, {group, roundtrip}]. -groups() -> +groups() -> [{roundtrip, [parallel], [roundtrip_1, roundtrip_2, roundtrip_3, roundtrip_4]}]. -init_per_suite(Config) -> - Config. - -end_per_suite(_Config) -> - ok. - -init_per_group(_GroupName, Config) -> - Config. - -end_per_group(_GroupName, Config) -> - Config. - - +group(roundtrip) -> + %% valgrind needs a lot of time + [{timetrap,{minutes,10}}]. %%------------------------------------------------------------------------- %% Test base64:encode/1. @@ -78,9 +61,9 @@ base64_encode(Config) when is_list(Config) -> %% One pad <<"SGVsbG8gV29ybGQ=">> = base64:encode(<<"Hello World">>), %% No pad - "QWxhZGRpbjpvcGVuIHNlc2Ft" = + "QWxhZGRpbjpvcGVuIHNlc2Ft" = base64:encode_to_string("Aladdin:open sesam"), - + "MDEyMzQ1Njc4OSFAIzBeJiooKTs6PD4sLiBbXXt9" = base64:encode_to_string(<<"0123456789!@#0^&*();:<>,. []{}">>), ok. @@ -93,7 +76,7 @@ base64_decode(Config) when is_list(Config) -> %% One pad <<"Hello World">> = base64:decode(<<"SGVsbG8gV29ybGQ=">>), %% No pad - <<"Aladdin:open sesam">> = + <<"Aladdin:open sesam">> = base64:decode("QWxhZGRpbjpvcGVuIHNlc2Ft"), Alphabet = list_to_binary(lists:seq(0, 255)), @@ -208,7 +191,7 @@ mime_decode_to_string(Config) when is_list(Config) -> %% One pad to ignore, followed by more text "Hello World!!" = base64:mime_decode_to_string(<<"SGVsb)(G8gV29ybGQ=h IQ= =">>), %% No pad - "Aladdin:open sesam" = + "Aladdin:open sesam" = base64:mime_decode_to_string("QWxhZGRpbjpvcG¤\")(VuIHNlc2Ft"), %% Encoded base 64 strings may be divided by non base 64 chars. %% In this cases whitespaces. @@ -314,7 +297,7 @@ interleaved_ws_roundtrip_1([], Base64List, Bin, List) -> random_byte_list(0, Acc) -> Acc; -random_byte_list(N, Acc) -> +random_byte_list(N, Acc) -> random_byte_list(N-1, [rand:uniform(255)|Acc]). make_big_binary(N) -> diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index b02d17bdb6..00e02a06cc 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -19,7 +19,7 @@ %% -module(ets_SUITE). --export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2]). -export([default/1,setbag/1,badnew/1,verybadnew/1,named/1,keypos2/1, privacy/1,privacy_owner/2]). @@ -31,15 +31,14 @@ -export([match_delete3/1]). -export([firstnext/1,firstnext_concurrent/1]). -export([slot/1]). --export([ match1/1, match2/1, match_object/1, match_object2/1]). --export([ dups/1, misc1/1, safe_fixtable/1, info/1, tab2list/1]). --export([ tab2file/1, tab2file2/1, tabfile_ext1/1, - tabfile_ext2/1, tabfile_ext3/1, tabfile_ext4/1, badfile/1]). --export([ heavy_lookup/1, heavy_lookup_element/1, heavy_concurrent/1]). --export([ lookup_element_mult/1]). --export([]). +-export([match1/1, match2/1, match_object/1, match_object2/1]). +-export([dups/1, misc1/1, safe_fixtable/1, info/1, tab2list/1]). +-export([tab2file/1, tab2file2/1, tabfile_ext1/1, + tabfile_ext2/1, tabfile_ext3/1, tabfile_ext4/1, badfile/1]). +-export([heavy_lookup/1, heavy_lookup_element/1, heavy_concurrent/1]). +-export([lookup_element_mult/1]). -export([foldl_ordered/1, foldr_ordered/1, foldl/1, foldr/1, fold_empty/1]). --export([t_delete_object/1, t_init_table/1, t_whitebox/1, +-export([t_delete_object/1, t_init_table/1, t_whitebox/1, t_delete_all_objects/1, t_insert_list/1, t_test_ms/1, t_select_delete/1,t_ets_dets/1]). @@ -61,8 +60,7 @@ -export([otp_7665/1]). -export([meta_wb/1]). -export([grow_shrink/1, grow_pseudo_deleted/1, shrink_pseudo_deleted/1]). --export([ - meta_lookup_unnamed_read/1, meta_lookup_unnamed_write/1, +-export([meta_lookup_unnamed_read/1, meta_lookup_unnamed_write/1, meta_lookup_named_read/1, meta_lookup_named_write/1, meta_newdel_unnamed/1, meta_newdel_named/1]). -export([smp_insert/1, smp_fixed_delete/1, smp_unfix_fix/1, smp_select_delete/1, @@ -95,7 +93,7 @@ rename_do/1, rename_unnamed_do/1, interface_equality_do/1, ordered_match_do/1, ordered_do/1, privacy_do/1, empty_do/1, badinsert_do/1, time_lookup_do/1, lookup_order_do/1, lookup_element_mult_do/1, delete_tab_do/1, delete_elem_do/1, - match_delete_do/1, match_delete3_do/1, firstnext_do/1, + match_delete_do/1, match_delete3_do/1, firstnext_do/1, slot_do/1, match1_do/1, match2_do/1, match_object_do/1, match_object2_do/1, misc1_do/1, safe_fixtable_do/1, info_do/1, dups_do/1, heavy_lookup_do/1, heavy_lookup_element_do/1, member_do/1, otp_5340_do/1, otp_7665_do/1, meta_wb_do/1, @@ -129,7 +127,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap,{minutes,5}}]. -all() -> +all() -> [{group, new}, {group, insert}, {group, lookup}, {group, delete}, firstnext, firstnext_concurrent, slot, {group, match}, t_match_spec_run, @@ -161,7 +159,7 @@ all() -> memory_check_summary]. % MUST BE LAST -groups() -> +groups() -> [{new, [], [default, setbag, badnew, verybadnew, named, keypos2, privacy]}, @@ -249,6 +247,7 @@ t_bucket_disappears_do(Opts) -> %% Check ets:match_spec_run/2. t_match_spec_run(Config) when is_list(Config) -> + ct:timetrap({minutes,30}), %% valgrind needs a lot init_externals(), EtsMem = etsmem(), @@ -709,7 +708,7 @@ adjust_xmem([_T1,_T2,_T3,_T4], {A0,B0,C0,D0} = _Mem0) -> {A0+TabDiff, B0+TabDiff, C0+TabDiff, D0+TabDiff}. %% Misc. whitebox tests -t_whitebox(Config) when is_list(Config) -> +t_whitebox(Config) when is_list(Config) -> EtsMem = etsmem(), repeat_for_opts(whitebox_1), repeat_for_opts(whitebox_1), @@ -1050,6 +1049,7 @@ do_reverse_chunked({L,C},Acc) -> %% Test the ets:select_delete/2 and ets:select_count/2 BIFs. t_select_delete(Config) when is_list(Config) -> + ct:timetrap({minutes,30}), %% valgrind needs a lot EtsMem = etsmem(), Tables = fill_sets_int(10000) ++ fill_sets_int(10000,[{write_concurrency,true}]), lists:foreach @@ -1495,15 +1495,15 @@ update_element(Config) when is_list(Config) -> verify_etsmem(EtsMem). update_element_opts(Opts) -> - TupleCases = [{{key,val}, 1 ,2}, - {{val,key}, 2, 1}, - {{key,val}, 1 ,[2]}, + TupleCases = [{{key,val}, 1 ,2}, + {{val,key}, 2, 1}, + {{key,val}, 1 ,[2]}, {{key,val,val}, 1, [2,3]}, {{val,key,val,val}, 2, [3,4,1]}, {{val,val,key,val}, 3, [1,4,1,2]}, % update pos1 twice {{val,val,val,key}, 4, [2,1,2,3]}],% update pos2 twice - lists:foreach(fun({Tuple,KeyPos,UpdPos}) -> update_element_opts(Tuple,KeyPos,UpdPos,Opts) end, + lists:foreach(fun({Tuple,KeyPos,UpdPos}) -> update_element_opts(Tuple,KeyPos,UpdPos,Opts) end, TupleCases), update_element_neg(Opts). @@ -1519,9 +1519,9 @@ update_element_opts(Tuple,KeyPos,UpdPos,Opts) -> true = ets:delete(OrdSet), ok. -update_element(T,Tuple,KeyPos,UpdPos) -> +update_element(T,Tuple,KeyPos,UpdPos) -> KeyList = [17,"seventeen",<<"seventeen">>,{17},list_to_binary(lists:seq(1,100)),make_ref(), self()], - lists:foreach(fun(Key) -> + lists:foreach(fun(Key) -> TupleWithKey = setelement(KeyPos,Tuple,Key), update_element_do(T,TupleWithKey,Key,UpdPos) end, @@ -1556,29 +1556,29 @@ update_element_do(Tab,Tuple,Key,UpdPos) -> {Pos, element(ToIx+1,Values)} % single {pos,value} arg end, - UpdateF = fun(ToIx,Rand) -> - PosValArg = PosValArgF(ToIx,[],UpdPos,Rand,PosValArgF), - %%io:format("update_element(~p)~n",[PosValArg]), - ArgHash = erlang:phash2({Tab,Key,PosValArg}), - true = ets:update_element(Tab, Key, PosValArg), - ArgHash = erlang:phash2({Tab,Key,PosValArg}), - NewTuple = update_tuple(PosValArg,Tuple), - [NewTuple] = ets:lookup(Tab,Key) + UpdateF = fun(ToIx,Rand) -> + PosValArg = PosValArgF(ToIx,[],UpdPos,Rand,PosValArgF), + %%io:format("update_element(~p)~n",[PosValArg]), + ArgHash = erlang:phash2({Tab,Key,PosValArg}), + true = ets:update_element(Tab, Key, PosValArg), + ArgHash = erlang:phash2({Tab,Key,PosValArg}), + NewTuple = update_tuple(PosValArg,Tuple), + [NewTuple] = ets:lookup(Tab,Key) end, - LoopF = fun(_FromIx, Incr, _Times, Checksum, _MeF) when Incr >= Length -> + LoopF = fun(_FromIx, Incr, _Times, Checksum, _MeF) when Incr >= Length -> Checksum; % done - (FromIx, Incr, 0, Checksum, MeF) -> + (FromIx, Incr, 0, Checksum, MeF) -> MeF(FromIx, Incr+1, Length, Checksum, MeF); - (FromIx, Incr, Times, Checksum, MeF) -> + (FromIx, Incr, Times, Checksum, MeF) -> ToIx = (FromIx + Incr) rem Length, UpdateF(ToIx,Checksum), - if + if Incr =:= 0 -> UpdateF(ToIx,Checksum); % extra update to same value true -> true - end, + end, MeF(ToIx, Incr, Times-1, Checksum+ToIx+1, MeF) end, @@ -1622,7 +1622,7 @@ update_element_neg_do(T) -> Object = {key, 0, "Hej"}, true = ets:insert(T,Object), - UpdateF = fun(Arg3) -> + UpdateF = fun(Arg3) -> ArgHash = erlang:phash2({T,key,Arg3}), {'EXIT',{badarg,_}} = (catch ets:update_element(T,key,Arg3)), ArgHash = erlang:phash2({T,key,Arg3}), @@ -1697,7 +1697,7 @@ update_counter_for(T) -> true = ets:lookup(T, b) =:= [setelement(1, NewObj, b)], ets:delete(T, b), Myself(NewObj,Times-1,Arg3,Myself) - end, + end, LoopF = fun(Obj, Times, Arg3) -> %%io:format("Loop start:\nObj = ~p\nArg3=~p\n",[Obj,Arg3]), @@ -1806,7 +1806,7 @@ uc_mimic(Obj, [Pits|Tail], Acc) -> uc_adder(Init, {_Pos, Add}) -> Init + Add; -uc_adder(Init, {_Pos, Add, Thres, Warp}) -> +uc_adder(Init, {_Pos, Add, Thres, Warp}) -> case Init + Add of X when X > Thres, Add > 0 -> Warp; @@ -1838,7 +1838,7 @@ update_counter_neg_for(T) -> Object = {key,0,false,1}, true = ets:insert(T,Object), - UpdateF = fun(Arg3) -> + UpdateF = fun(Arg3) -> ArgHash = erlang:phash2({T,key,Arg3}), {'EXIT',{badarg,_}} = (catch ets:update_counter(T,key,Arg3)), ArgHash = erlang:phash2({T,key,Arg3}), @@ -1978,15 +1978,16 @@ fixtable_next_do(Opts) -> verify_etsmem(EtsMem). do_fixtable_next(Tab) -> - F = fun(X,T,FF) -> case X of - 0 -> true; - _ -> - ets:insert(T, {X, - integer_to_list(X), - X rem 10}), - FF(X-1,T,FF) - end - end, + F = fun(X,T,FF) -> + case X of + 0 -> true; + _ -> + ets:insert(T, {X, + integer_to_list(X), + X rem 10}), + FF(X-1,T,FF) + end + end, F(100,Tab,F), ets:safe_fixtable(Tab,true), First = ets:first(Tab), @@ -2001,7 +2002,7 @@ do_fixtable_next(Tab) -> %% Check inserts of deleted keys in fixed bags. fixtable_insert(Config) when is_list(Config) -> - Combos = [[Type,{write_concurrency,WC}] || Type<- [bag,duplicate_bag], + Combos = [[Type,{write_concurrency,WC}] || Type<- [bag,duplicate_bag], WC <- [false,true]], lists:foreach(fun(Opts) -> fixtable_insert_do(Opts) end, Combos), @@ -2117,7 +2118,7 @@ heir_do(Opts) -> %% Different types of heir data and link/monitor relations TestFun = fun(Arg) -> {EtsMem,Arg} end, - Combos = [{Data,Mode} || Data<-[foo_data, <<"binary">>, + Combos = [{Data,Mode} || Data<-[foo_data, <<"binary">>, lists:seq(1,10), {17,TestFun,self()}, "The busy heir"], Mode<-[none,link,monitor]], @@ -2157,7 +2158,7 @@ heir_do(Opts) -> Founder4 ! {go, Heir4}, {'DOWN', MrefH4, process, Heir4, normal} = receive_any(), erts_debug:set_internal_state(next_pid, NextPidIx), - DoppelGanger = spawn_monitor_with_pid(Heir4, + DoppelGanger = spawn_monitor_with_pid(Heir4, fun()-> die_please = receive_any() end), Founder4 ! die_please, {'DOWN', MrefF4, process, Founder4, normal} = receive_any(), @@ -2170,12 +2171,12 @@ heir_do(Opts) -> failed -> io:format("Failed to spawn process with pid ~p\n", [Heir4]), true % try again - end + end end), verify_etsmem(EtsMem). -heir_founder(Master, HeirData, Opts) -> +heir_founder(Master, HeirData, Opts) -> {go,Heir} = receive_any(), HeirTpl = case Heir of none -> {heir,none}; @@ -2248,7 +2249,7 @@ heir_1(HeirData,Mode,Opts) -> {'DOWN', Mref, process, Heir, normal} = receive_any(). %% Test ets:give_way/3. -give_away(Config) when is_list(Config) -> +give_away(Config) when is_list(Config) -> repeat_for_opts(give_away_do). give_away_do(Opts) -> @@ -2387,7 +2388,7 @@ bad_table(Config) when is_list(Config) -> ok. bad_table_do(Opts, DummyFile) -> - Parent = self(), + Parent = self(), {Pid,Mref} = my_spawn_opt(fun()-> ets_new(priv,[private,named_table | Opts]), Priv = ets_new(priv,[private | Opts]), ets_new(prot,[protected,named_table | Opts]), @@ -2442,7 +2443,7 @@ bad_table_do(Opts, DummyFile) -> ], Info = {Opts, Priv, Prot}, lists:foreach(fun(Op) -> bad_table_op(Info, Op) end, - OpList), + OpList), Pid ! die_please, {'DOWN', Mref, process, Pid, normal} = receive_any(), ok. @@ -2577,14 +2578,14 @@ interface_equality_do(Opts) -> Set = ets_new(set,[set | Opts]), OrderedSet = ets_new(ordered_set,[ordered_set | Opts]), F = fun(X,T,FF) -> case X of - 0 -> true; - _ -> - ets:insert(T, {X, - integer_to_list(X), - X rem 10}), - FF(X-1,T,FF) - end - end, + 0 -> true; + _ -> + ets:insert(T, {X, + integer_to_list(X), + X rem 10}), + FF(X-1,T,FF) + end + end, F(100,Set,F), F(100,OrderedSet,F), equal_results(ets, insert, Set, OrderedSet, [{a,"a"}]), @@ -2653,20 +2654,20 @@ ordered_match_do(Opts) -> F(3000,T1,F), [[3,3],[3,3],[3,3]] = ets:match(T1, {'_','_','$1','$2',3}), F2 = fun(X,Rem,Res,FF) -> case X of - 0 -> []; - _ -> + 0 -> []; + _ -> case X rem Rem of Res -> FF(X-1,Rem,Res,FF) ++ [{X, - integer_to_list(X), + integer_to_list(X), X rem 10, X rem 100, X rem 1000}]; _ -> FF(X-1,Rem,Res,FF) end - end + end end, OL1 = F2(3000,100,2,F2), OL1 = ets:match_object(T1, {'_','_','_',2,'_'}), @@ -2744,7 +2745,7 @@ pick_all_backwards(T) -> %% Small test case for both set and bag type ets tables. -setbag(Config) when is_list(Config) -> +setbag(Config) when is_list(Config) -> EtsMem = etsmem(), Set = ets_new(set,[set]), Bag = ets_new(bag,[bag]), @@ -2821,7 +2822,7 @@ privacy_do(Opts) -> privacy_check(pub,prot,priv), - Owner ! {shift,1,{pub,prot,priv}}, + Owner ! {shift,1,{pub,prot,priv}}, receive {Pub1,Prot1,Priv1} -> ok = privacy_check(Pub1,Prot1,Priv1), @@ -2960,7 +2961,7 @@ badlookup(Config) when is_list(Config) -> verify_etsmem(EtsMem). %% Test that lookup returns objects in order of insertion for bag and dbag. -lookup_order(Config) when is_list(Config) -> +lookup_order(Config) when is_list(Config) -> EtsMem = etsmem(), repeat_for_opts(lookup_order_do, [write_concurrency,[bag,duplicate_bag]]), verify_etsmem(EtsMem), @@ -2982,7 +2983,7 @@ lookup_order_2(Opts, Fixed) -> case Fixed of true -> ets:safe_fixtable(T,true); false -> ok - end, + end, S10 = {T,[],key}, S20 = check_insert(S10,A), S30 = check_insert(S20,B), @@ -2994,7 +2995,7 @@ lookup_order_2(Opts, Fixed) -> S80 = check_delete(S70,D2b), S90 = check_insert(S80,D2a), SA0 = check_delete(S90,D3a), - SB0 = check_delete(SA0,D3b), + SB0 = check_delete(SA0,D3b), check_insert_new(SB0,D3b), true = ets:delete(T) @@ -3007,7 +3008,7 @@ check_insert({T,List0,Key},Val) -> ets:insert(T,{Key,Val}), List1 = case (ets:info(T,type) =:= bag andalso lists:member({Key,Val},List0)) of - true -> List0; + true -> List0; false -> [{Key,Val} | List0] end, check_check({T,List1,Key}). @@ -3040,8 +3041,6 @@ check_check(S={T,List,Key}) -> Items = length(List), S. - - fill_tab(Tab,Val) -> ets:insert(Tab,{key,Val}), ets:insert(Tab,{{a,144},Val}), @@ -3069,13 +3068,11 @@ lookup_element_mult_do(Opts) -> verify_etsmem(EtsMem). lem_data() -> - [ - {service,'eddie2@boromir',{150,236,14,103},httpd88,self()}, + [{service,'eddie2@boromir',{150,236,14,103},httpd88,self()}, {service,'eddie2@boromir',{150,236,14,103},httpd80,self()}, {service,'eddie3@boromir',{150,236,14,107},httpd88,self()}, {service,'eddie3@boromir',{150,236,14,107},httpd80,self()}, - {service,'eddie4@boromir',{150,236,14,108},httpd88,self()} - ]. + {service,'eddie4@boromir',{150,236,14,108},httpd88,self()}]. lem_crash(T) -> L = ets:lookup_element(T, 'eddie2@boromir', 3), @@ -3126,6 +3123,7 @@ delete_tab_do(Opts) -> %% Check that ets:delete/1 works and that other processes can run. delete_large_tab(Config) when is_list(Config) -> + ct:timetrap({minutes,30}), %% valgrind needs a lot Data = [{erlang:phash2(I, 16#ffffff),I} || I <- lists:seq(1, 200000)], EtsMem = etsmem(), repeat_for_opts(fun(Opts) -> delete_large_tab_do(Opts,Data) end), @@ -3148,7 +3146,7 @@ delete_large_tab_1(Name, Flags, Data, Fix) -> lists:foreach(fun({K,_}) -> ets:delete(Tab, K) end, Data) end, - {priority, Prio} = process_info(self(), priority), + {priority, Prio} = process_info(self(), priority), Deleter = self(), [SchedTracer] = start_loopers(1, @@ -3195,7 +3193,7 @@ delete_large_tab_1(Name, Flags, Data, Fix) -> %% Delete a large name table and try to create a new table with %% the same name in another process. -delete_large_named_table(Config) when is_list(Config) -> +delete_large_named_table(Config) when is_list(Config) -> Data = [{erlang:phash2(I, 16#ffffff),I} || I <- lists:seq(1, 200000)], EtsMem = etsmem(), repeat_for_opts(fun(Opts) -> delete_large_named_table_do(Opts,Data) end), @@ -3566,7 +3564,7 @@ dyn_lookup(T) -> dyn_lookup(T, ets:first(T)). dyn_lookup(_T, '$end_of_table') -> []; dyn_lookup(T, K) -> - NextKey=ets:next(T,K), + NextKey = ets:next(T,K), case ets:next(T,K) of NextKey -> dyn_lookup(T, NextKey); @@ -4085,9 +4083,9 @@ tabfile_ext2_do(Opts,Config) -> Name = make_ref(), [ets:insert(T,{X,integer_to_list(X)}) || X <- L], ok = ets:tab2file(T,FName,[{extended_info,[md5sum]}]), - true = lists:sort(ets:tab2list(T)) =:= + true = lists:sort(ets:tab2list(T)) =:= lists:sort(ets:tab2list(element(2,ets:file2tab(FName)))), - true = lists:sort(ets:tab2list(T)) =:= + true = lists:sort(ets:tab2list(T)) =:= lists:sort(ets:tab2list( element(2,ets:file2tab(FName,[{verify,true}])))), {ok, Name} = disk_log:open([{name,Name},{file,FName}]), @@ -4102,9 +4100,9 @@ tabfile_ext2_do(Opts,Config) -> ets:tab2list( element(2,ets:file2tab(FName2)))), {error,checksum_error} = ets:file2tab(FName2,[{verify,true}]), - {value,{extended_info,[md5sum]}} = + {value,{extended_info,[md5sum]}} = lists:keysearch(extended_info,1,element(2,ets:tabfile_info(FName2))), - {value,{extended_info,[md5sum]}} = + {value,{extended_info,[md5sum]}} = lists:keysearch(extended_info,1,element(2,ets:tabfile_info(FName))), file:delete(FName), file:delete(FName2), @@ -4149,15 +4147,14 @@ tabfile_ext4(Config) when is_list(Config) -> Name2 = make_ref(), [ets:insert(TL,{X,integer_to_list(X)}) || X <- LL], ok = ets:tab2file(TL,FName,[{extended_info,[md5sum]}]), - {ok, Name2} = disk_log:open([{name, Name2}, {file, FName}, + {ok, Name2} = disk_log:open([{name, Name2}, {file, FName}, {mode, read_only}]), {C,[_|_]} = disk_log:chunk(Name2,start), {_,[_|_]} = disk_log:chunk(Name2,C), disk_log:close(Name2), - true = lists:sort(ets:tab2list(TL)) =:= + true = lists:sort(ets:tab2list(TL)) =:= lists:sort(ets:tab2list(element(2,ets:file2tab(FName)))), - Res = [ - begin + Res = [begin {ok,FD} = file:open(FName,[binary,read,write]), {ok, Bin} = file:pread(FD,0,1000), <<B1:N/binary,Ch:8,B2/binary>> = Bin, @@ -4167,7 +4164,7 @@ tabfile_ext4(Config) when is_list(Config) -> ok = file:close(FD), X = case ets:file2tab(FName) of {ok,TL2} -> - true = lists:sort(ets:tab2list(TL)) =/= + true = lists:sort(ets:tab2list(TL)) =/= lists:sort(ets:tab2list(TL2)); _ -> totally_broken @@ -4175,7 +4172,7 @@ tabfile_ext4(Config) when is_list(Config) -> {error,Y} = ets:file2tab(FName,[{verify,true}]), ets:tab2file(TL,FName,[{extended_info,[md5sum]}]), {X,Y} - end || N <- lists:seq(500,600) ], + end || N <- lists:seq(500,600)], io:format("~p~n",[Res]), file:delete(FName), ok. @@ -4404,16 +4401,14 @@ member_do(Opts) -> build_table(L1,L2,Num) -> - T = ets_new(xxx, [ordered_set] - ), + T = ets_new(xxx, [ordered_set]), lists:foreach( fun(X1) -> lists:foreach( fun(X2) -> F = fun(FF,N) -> - ets:insert(T,{{X1,X2,N}, - X1, X2, N}), - case N of + ets:insert(T,{{X1,X2,N}, X1, X2, N}), + case N of 0 -> ok; _ -> @@ -4426,16 +4421,14 @@ build_table(L1,L2,Num) -> T. build_table2(L1,L2,Num) -> - T = ets_new(xxx, [ordered_set] - ), + T = ets_new(xxx, [ordered_set]), lists:foreach( fun(X1) -> lists:foreach( fun(X2) -> F = fun(FF,N) -> - ets:insert(T,{{N,X1,X2}, - N, X1, X2}), - case N of + ets:insert(T,{{N,X1,X2}, N, X1, X2}), + case N of 0 -> ok; _ -> @@ -4726,7 +4719,7 @@ del_one_by_one_dbag_3(T,From,To) -> N = (ets:info(T,size) + 1), Obj2 = {From, integer_to_list(From)}, ets:delete_object(T,Obj2), - N = (ets:info(T,size) + 2) + N = (ets:info(T,size) + 2) end, Next = if From < To -> @@ -4773,14 +4766,14 @@ gen_dets_filename(Config,N) -> filename:join(proplists:get_value(priv_dir,Config), "testdets_" ++ integer_to_list(N) ++ ".dets"). -otp_6842_select_1000(Config) when is_list(Config) -> +otp_6842_select_1000(Config) when is_list(Config) -> Tab = ets_new(xxx,[ordered_set]), [ets:insert(Tab,{X,X}) || X <- lists:seq(1,10000)], AllTrue = lists:duplicate(10,true), AllTrue = [ length( element(1, - ets:select(Tab,[{'_',[],['$_']}],X*1000))) =:= + ets:select(Tab,[{'_',[],['$_']}],X*1000))) =:= X*1000 || X <- lists:seq(1,10) ], Sequences = [[1000,1000,1000,1000,1000,1000,1000,1000,1000,1000], [2000,2000,2000,2000,2000], @@ -4806,7 +4799,13 @@ check_seq(A,B,C) -> false. otp_6338(Config) when is_list(Config) -> - L = binary_to_term(<<131,108,0,0,0,2,104,2,108,0,0,0,2,103,100,0,19,112,112,98,49,95,98,115,49,50,64,98,108,97,100,101,95,48,95,53,0,0,33,50,0,0,0,4,1,98,0,0,23,226,106,100,0,4,101,120,105,116,104,2,108,0,0,0,2,104,2,100,0,3,115,98,109,100,0,19,112,112,98,50,95,98,115,49,50,64,98,108,97,100,101,95,48,95,56,98,0,0,18,231,106,100,0,4,114,101,99,118,106>>), + L = binary_to_term(<<131,108,0,0,0,2,104,2,108,0,0,0,2,103,100,0,19,112,112, + 98,49,95,98,115,49,50,64,98,108,97,100,101,95,48,95,53, + 0,0,33,50,0,0,0,4,1,98,0,0,23,226,106,100,0,4,101,120, + 105,116,104,2,108,0,0,0,2,104,2,100,0,3,115,98,109,100, + 0,19,112,112,98,50,95,98,115,49,50,64,98,108,97,100, + 101,95,48,95,56,98,0,0,18,231,106,100,0,4,114,101,99, + 118,106>>), T = ets_new(xxx,[ordered_set]), lists:foreach(fun(X) -> ets:insert(T,X) end,L), [[4839,recv]] = ets:match(T,{[{sbm,ppb2_bs12@blade_0_8},'$1'],'$2'}), @@ -4825,7 +4824,7 @@ otp_5340_do(Opts) -> ets:delete(T). w(_,0, _) -> ok; -w(T,N, Id) -> +w(T,N, Id) -> ets:insert(T, {N, Id}), w(T,N-1,Id). @@ -4915,7 +4914,7 @@ meta_wb_new(Name, _, Tabs, Opts) -> case (catch ets_new(Name,[named_table|Opts])) of Name -> false = lists:member(Name, Tabs), - [Name | Tabs]; + [Name | Tabs]; {'EXIT',{badarg,_}} -> true = lists:member(Name, Tabs), Tabs @@ -5090,7 +5089,7 @@ meta_lookup_unnamed_read(Config) when is_list(Config) -> Tab end, ExecF = fun(Tab) -> [{key,data}] = ets:lookup(Tab,key), - Tab + Tab end, FiniF = fun(Tab) -> true = ets:delete(Tab) end, @@ -5114,7 +5113,7 @@ meta_lookup_named_read(Config) when is_list(Config) -> Tab end, ExecF = fun(Tab) -> [{key,data}] = ets:lookup(Tab,key), - Tab + Tab end, FiniF = fun(Tab) -> true = ets:delete(Tab) end, @@ -5173,9 +5172,9 @@ smp_fixed_delete_do() -> ets:safe_fixtable(T,true), Buckets = num_of_buckets(T), InitF = fun([ProcN,NumOfProcs|_]) -> {ProcN,NumOfProcs} end, - ExecF = fun({Key,_}) when Key > NumOfObjs -> + ExecF = fun({Key,_}) when Key > NumOfObjs -> [end_of_work]; - ({Key,Increment}) -> + ({Key,Increment}) -> true = ets:delete(T,Key), {Key+Increment,Increment} end, @@ -5204,7 +5203,7 @@ smp_unfix_fix_do() -> T = ets_new(foo,[public,{write_concurrency,true}]), %%Mem = ets:info(T,memory), NumOfObjs = 100000, - Deleted = 50000, + Deleted = 50000, filltabint(T,NumOfObjs), ets:safe_fixtable(T,true), Buckets = num_of_buckets(T), @@ -5217,7 +5216,7 @@ smp_unfix_fix_do() -> true = ets:info(T,fixed), Deleted = get_kept_objects(T), - {Child, Mref} = + {Child, Mref} = my_spawn_opt( fun()-> true = ets:info(T,fixed), @@ -5276,22 +5275,19 @@ otp_8166_do(WC) -> NumOfObjs = 3000, %% Need more than 1000 live objects for match_object to trap one time Deleted = NumOfObjs div 2, filltabint(T,NumOfObjs), - {ReaderPid, ReaderMref} = - my_spawn_opt(fun()-> otp_8166_reader(T,NumOfObjs) end, - [link, monitor, {scheduler,2}]), - {ZombieCrPid, ZombieCrMref} = - my_spawn_opt(fun()-> otp_8166_zombie_creator(T,Deleted) end, - [link, monitor, {scheduler,3}]), + {ReaderPid, ReaderMref} = my_spawn_opt(fun()-> otp_8166_reader(T,NumOfObjs) end, + [link, monitor, {scheduler,2}]), + {ZombieCrPid, ZombieCrMref} = my_spawn_opt(fun()-> otp_8166_zombie_creator(T,Deleted) end, + [link, monitor, {scheduler,3}]), repeat(fun() -> ZombieCrPid ! {loop, self()}, zombies_created = receive_any(), otp_8166_trapper(T, 10, ZombieCrPid) - end, - 100), + end, 100), ReaderPid ! quit, {'DOWN', ReaderMref, process, ReaderPid, normal} = receive_any(), - ZombieCrPid ! quit, + ZombieCrPid ! quit, {'DOWN', ZombieCrMref, process, ZombieCrPid, normal} = receive_any(), false = ets:info(T,fixed), 0 = get_kept_objects(T), @@ -5301,7 +5297,7 @@ otp_8166_do(WC) -> %% Keep reading the table otp_8166_reader(T, NumOfObjs) -> - repeat_while(fun(0) -> + repeat_while(fun(0) -> receive quit -> {false,done} after 0 -> {true,NumOfObjs} end; @@ -5315,14 +5311,14 @@ otp_8166_reader(T, NumOfObjs) -> otp_8166_trapper(T, Try, ZombieCrPid) -> [] = ets:match_object(T,{'_',"Pink Unicorn"}), case {ets:info(T,fixed),Try} of - {true,1} -> + {true,1} -> io:format("failed to provoke unsafe unfix, give up...\n",[]), ZombieCrPid ! unfix; - {true,_} -> + {true,_} -> io:format("trapper too fast, trying again...\n",[]), otp_8166_trapper(T, Try-1, ZombieCrPid); {false,_} -> done - end. + end. %% Fixate table and create some pseudo-deleted objects (zombies) @@ -5342,7 +5338,7 @@ otp_8166_zombie_creator(T,Deleted) -> repeat_while(fun() -> case ets:info(T,safe_fixed_monotonic_time) of {_,[_P1,_P2]} -> false; - _ -> + _ -> receive unfix -> false after 0 -> true end @@ -5399,7 +5395,7 @@ smp_select_delete(Config) when is_list(Config) -> Mod = 17, Zeros = erlang:make_tuple(Mod,0), InitF = fun(_) -> Zeros end, - ExecF = fun(Diffs0) -> + ExecF = fun(Diffs0) -> case rand:uniform(20) of 1 -> Mod = 17, @@ -5421,7 +5417,7 @@ smp_select_delete(Config) when is_list(Config) -> Diffs1; false -> Diffs0 end - end + end end, FiniF = fun(Result) -> Result end, Results = run_workers_do(InitF,ExecF,FiniF,20000), @@ -5432,7 +5428,7 @@ smp_select_delete(Config) when is_list(Config) -> 0, TotCnts), io:format("LeftInTab = ~p\n",[LeftInTab]), LeftInTab = ets:info(T,size), - lists:foldl(fun(Cnt,Eq) -> + lists:foldl(fun(Cnt,Eq) -> WasCnt = ets:select_count(T, [{{'_', '$1'}, [{'=:=', {'rem', '$1', Mod}, Eq}], @@ -5440,7 +5436,7 @@ smp_select_delete(Config) when is_list(Config) -> io:format("~p: ~p =?= ~p\n",[Eq,Cnt,WasCnt]), Cnt = WasCnt, Eq+1 - end, + end, 0, TotCnts), verify_table_load(T), LeftInTab = ets:select_delete(T, [{{'$1','$1'}, [], [true]}]), @@ -5478,8 +5474,8 @@ types_do(Opts) -> %% OTP-9932: Memory overwrite when inserting large integers in compressed bag. %% Will crash with segv on 64-bit opt if not fixed. otp_9932(Config) when is_list(Config) -> - T = ets:new(xxx, [bag, compressed]), - Fun = fun(N) -> + T = ets:new(xxx, [bag, compressed]), + Fun = fun(N) -> Key = {1316110174588445 bsl N,1316110174588583 bsl N}, S = {Key, Key}, true = ets:insert(T, S), @@ -5495,9 +5491,9 @@ otp_9932(Config) when is_list(Config) -> %% write_concurrency table. otp_9423(Config) when is_list(Config) -> InitF = fun(_) -> {0,0} end, - ExecF = fun({S,F}) -> - receive - stop -> + ExecF = fun({S,F}) -> + receive + stop -> io:format("~p got stop\n", [self()]), [end_of_work | {"Succeded=",S,"Failed=",F}] after 0 -> @@ -5593,12 +5589,12 @@ take(Config) when is_list(Config) -> %% Utility functions: %% -add_lists(L1,L2) -> +add_lists(L1,L2) -> add_lists(L1,L2,[]). add_lists([],[],Acc) -> lists:reverse(Acc); add_lists([E1|T1], [E2|T2], Acc) -> - add_lists(T1, T2, [E1+E2 | Acc]). + add_lists(T1, T2, [E1+E2 | Acc]). run_workers(InitF,ExecF,FiniF,Laps) -> run_workers(InitF,ExecF,FiniF,Laps, 0). @@ -5644,9 +5640,9 @@ worker_loop(infinite, ExecF, State) -> worker_loop(N, ExecF, State) -> worker_loop(N-1,ExecF,ExecF(State)). -wait_pids(Pids) -> +wait_pids(Pids) -> wait_pids(Pids,[]). -wait_pids([],Acc) -> +wait_pids([],Acc) -> Acc; wait_pids(Pids, Acc) -> receive @@ -5683,7 +5679,7 @@ etsmem() -> wait_for_memory_deallocations(), AllTabs = lists:map(fun(T) -> {T,ets:info(T,name),ets:info(T,size), - ets:info(T,memory),ets:info(T,type)} + ets:info(T,memory),ets:info(T,type)} end, ets:all()), EtsAllocInfo = erlang:system_info({allocator,ets_alloc}), @@ -5895,7 +5891,7 @@ receive_any() -> receive_any_spinning() -> receive_any_spinning(1000000). receive_any_spinning(Loops) -> - receive_any_spinning(Loops,Loops,1). + receive_any_spinning(Loops,Loops,1). receive_any_spinning(Loops,0,Tries) -> receive M -> io:format("Spinning process ~p got msg ~p after ~p tries\n", [self(),M,Tries]), diff --git a/lib/stdlib/test/rand_SUITE.erl b/lib/stdlib/test/rand_SUITE.erl index cb778c96d4..02b7cb10c2 100644 --- a/lib/stdlib/test/rand_SUITE.erl +++ b/lib/stdlib/test/rand_SUITE.erl @@ -18,7 +18,11 @@ %% %CopyrightEnd% -module(rand_SUITE). --export([all/0, suite/0,groups/0]). +-compile({nowarn_deprecated_function,[{random,seed,1}, + {random,uniform_s,1}, + {random,uniform_s,2}]}). + +-export([all/0, suite/0, groups/0, group/1]). -export([interval_int/1, interval_float/1, seed/1, api_eq/1, reference/1, @@ -47,18 +51,22 @@ groups() -> [{basic_stats, [parallel], [basic_stats_uniform_1, basic_stats_uniform_2, basic_stats_normal]}]. +group(basic_stats) -> + %% valgrind needs a lot of time + [{timetrap,{minutes,10}}]. + %% A simple helper to test without test_server during dev test() -> Tests = all(), lists:foreach(fun(Test) -> - try - ok = ?MODULE:Test([]), - io:format("~p: ok~n", [Test]) - catch _:Reason -> - io:format("Failed: ~p: ~p ~p~n", - [Test, Reason, erlang:get_stacktrace()]) - end - end, Tests). + try + ok = ?MODULE:Test([]), + io:format("~p: ok~n", [Test]) + catch _:Reason -> + io:format("Failed: ~p: ~p ~p~n", + [Test, Reason, erlang:get_stacktrace()]) + end + end, Tests). algs() -> [exs64, exsplus, exs1024]. diff --git a/system/doc/design_principles/applications.xml b/system/doc/design_principles/applications.xml index 0a1b65ea8e..c673fde07e 100644 --- a/system/doc/design_principles/applications.xml +++ b/system/doc/design_principles/applications.xml @@ -11,7 +11,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software @@ -19,7 +19,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - + </legalnotice> <title>Applications</title> @@ -172,31 +172,136 @@ ch_app:stop([])</code> </section> <section> - <marker id="app_dir"></marker> - <title>Directory Structure</title> - <p>When packaging code using <c>systools</c>, the code for each - application is placed in a separate directory, - <c>lib/Application-Vsn</c>, where <c>Vsn</c> is the version number.</p> - <p>This can be useful to know, even if <c>systools</c> is not used, - since Erlang/OTP is packaged according to the OTP principles - and thus comes with this directory structure. The code server - (see the <c>code(3)</c> manual page in Kernel) automatically - uses code from - the directory with the highest version number, if more than one - version of an application is present.</p> - <p>The application directory structure can also be used in the - development environment. The version number can then - be omitted from the name.</p> - <p>The application directory has the following sub-directories:</p> - <list type="bulleted"> - <item><c>src</c> - Contains the Erlang source code.</item> - <item><c>ebin</c> - Contains the Erlang object code, the - <c>beam</c> files. The <c>.app</c> file is also placed here.</item> - <item><c>priv</c> - Used for application specific files. For - example, C executables are placed here. The function - <c>code:priv_dir/1</c> is to be used to access this directory.</item> - <item><c>include</c> - Used for include files.</item> - </list> + <marker id="app_dir"></marker> + <title>Directory Structure</title> + <p>When packaging code using <c>systools</c>, the code for each + application is placed in a separate directory, + <c>lib/Application-Vsn</c>, where <c>Vsn</c> is the version number.</p> + <p>This can be useful to know, even if <c>systools</c> is not used, + since Erlang/OTP is packaged according to the OTP principles + and thus comes with a specific directory structure. The code server + (see the <seealso marker="kernel:code"><c>code(3)</c></seealso> manual + page in Kernel) automatically uses code from + the directory with the highest version number, if more than one + version of an application is present.</p> + <section> + <title>Directory Structure guidelines for a Development Environment</title> + <p>Any directory structure for development will suffice as long as the released directory structure + adhere to the <seealso marker="#app_dir_released">description below</seealso>, + but it is encouraged that the same directory structure + also be used in a development environment. The version number should be omitted from the + application directory name since this is an artifact of the release step. + </p> + <p> Some sub-directories are <em>required</em>. Some sub-directories are <em>optional</em>, meaning that it should + only be used if the application itself requires it. Finally, some sub-directories are <em>recommended</em>, + meaning it is encouraged that it is used and used as described here. For example, both documentation + and tests are encouraged to exist in an application for it to be deemed a proper OTP application.</p> +<code type="none"> + ─ ${application} + ├── doc + │ ├── internal + │ ├── examples + │ └── src + ├── include + ├── priv + ├── src + │ └── ${application}.app.src + └── test +</code> + <list type="bulleted"> + <item><c>src</c> - Required. Contains the Erlang source code, the source of the <c>.app</c> file + and internal include files used by the application itself. Additional sub-directories within + <c>src</c> can be used as namespaces to organize source files. These directories should never + be deeper than one level.</item> + <item><c>priv</c> - Optional. Used for application specific files. </item> + <item><c>include</c> - Optional. Used for public include files that must be reachable from + other applications.</item> + <item><c>doc</c> - Recommended. Any source documentation should be placed in sub-directories here.</item> + <item><c>doc/internal</c> - Recommended. Any documentation that describes implementation details about + this application, not intended for publication, should be placed here.</item> + <item><c>doc/examples</c> - Recommended. Source code for examples on how to use this application should + be placed here. It is encouraged that examples are sourced to the public documentation from + this directory.</item> + <item><c>doc/src</c> - Recommended. All source files for documentation, such as Markdown, AsciiDoc or + XML-files, should be placed here.</item> + <item><c>test</c> - Recommended. All files regarding tests, such as test suites and test specifications, + should be placed here. </item> + </list> + + <p>Other directories in the development environment may be needed. If source code from languages other + than Erlang is used, for instance C-code for NIFs, that code should be placed in a separate directory. + By convention it is recommended to prefix such directories with the language name, for example + <c>c_src</c> for C, <c>java_src</c> for Java or <c>go_src</c> for Go. Directories with <c>_src</c> + suffix indicates that it is a part of the application and the compilation step. The final build artifacts + should target the <c>priv/lib</c> or <c>priv/bin</c> directories.</p> + <p>The <c>priv</c> directory holds assets that the application needs during runtime. Executables should + reside in <c>priv/bin</c> and dynamically-linked libraries should reside in <c>priv/lib</c>. Other assets + are free to reside within the <c>priv</c> directory but it is recommended it does so in a structured manner.</p> + <p>Source files from other languages that generate Erlang code, such as ASN.1 or Mibs, should be placed + in directories, at the top level or in <c>src</c>, with the same name as the source language, for example + <c>asn1</c> and <c>mibs</c>. Build artifacts should be placed in their respective language directory, + such as <c>src</c> for Erlang code or <c>java_src</c> for Java code.</p> + <p>The <c>.app</c> file for release may reside in the <c>ebin</c>-directory in a development environment + but it is encouraged that this is an artifact of the build step. By convention a <c>.app.src</c> file + is used, which resides in the <c>src</c> directory. This file is nearly identical as the + <c>.app</c> file but certain fields may be replaced during the build step, such as the application version.</p> + <p>Directory names should not be capitalized.</p> + <p>It is encouraged to omit empty directories.</p> + + </section> + + <section> + <marker id="app_dir_released"></marker> + <title>Directory Structure for a Released System</title> + <p>A released application must follow a certain structure. + </p> +<code type="none"> + ─ ${application}-${version} + ├── bin + ├── doc + │ ├── html + │ ├── man[1-9] + │ ├── pdf + │ ├── internal + │ └── examples + ├── ebin + │ └── ${application}.app + ├── include + ├── priv + │ ├── lib + │ └── bin + └── src +</code> + <list type="bulleted"> + <item><c>src</c> - Optional. Contains the Erlang source code and internal include files + used by the application itself. This directory is no longer required in a released application.</item> + <item><c>ebin</c> - Required. Contains the Erlang object code, the <c>beam</c> files. + The <c>.app</c> file must also be placed here.</item> + <item><c>priv</c> - Optional. Used for application specific files. <c>code:priv_dir/1</c> + is to be used to access this directory.</item> + <item><c>priv/lib</c> - Recommended. Any shared-object files that are used by the application, + such as NIFs or linked-in-drivers, should be placed here.</item> + <item><c>priv/bin</c> - Recommended. Any executable that is used by the application, + such as port-programs, should be placed here.</item> + <item><c>include</c> - Optional. Used for public include files that must be reachable from + other applications.</item> + <item><c>bin</c> - Optional. Any executable that is a product of the application, + such as escripts or shell-scripts, should be placed here.</item> + <item><c>doc</c> - Optional. Any released documentation should be placed in + sub-directories here.</item> + <item><c>doc/man1</c> - Recommended. Man pages for Application executables.</item> + <item><c>doc/man3</c> - Recommended. Man pages for module APIs.</item> + <item><c>doc/man6</c> - Recommended. Man pages for Application overview.</item> + <item><c>doc/html</c> - Optional. HTML pages for the entire Application.</item> + <item><c>doc/pdf</c> - Optional. PDF documentation for the entire Application.</item> + </list> + + <p>The <c>src</c> directory could be useful to release for debugging purposes but is not required. + The <c>include</c> directory should only be released if the applications has public include files.</p> + <p>The only documentation that is recommended to be released in this way are the man pages. HTML and PDF + will normally be distributed in some other manner.</p> + <p>It is encouraged to omit empty directories.</p> + </section> </section> <section> @@ -381,4 +486,3 @@ application:start(Application, Type)</code> <c>shutdown</c>, not <c>normal</c>.</p> </section> </chapter> - |