diff options
124 files changed, 2584 insertions, 1336 deletions
diff --git a/erts/configure.in b/erts/configure.in index d7d56d45b6..eb3d975edc 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -381,7 +381,6 @@ if test X${enable_m64_build} = Xyes; then else if test X${enable_m32_build} = Xyes; then - enable_hipe=no; case $CFLAGS in *-m32*) ;; @@ -642,10 +641,6 @@ dnl Ditto between ultrasparc and sparc64. dnl AC_MSG_CHECKING(whether compilation mode forces ARCH adjustment) case "$ARCH-$ac_cv_sizeof_void_p" in -i386-8) - AC_MSG_RESULT(yes: adjusting ARCH=x86 to ARCH=amd64) - ARCH=amd64 - ;; x86-8) AC_MSG_RESULT(yes: adjusting ARCH=x86 to ARCH=amd64) ARCH=amd64 @@ -666,6 +661,14 @@ ppc64-4) AC_MSG_RESULT(yes: adjusting ARCH=ppc64 to ARCH=ppc) ARCH=ppc ;; +ppc-8) + AC_MSG_RESULT(yes: adjusting ARCH=ppc to ARCH=ppc64) + ARCH=ppc64 + ;; +arm-8) + AC_MSG_RESULT(yes: adjusting ARCH=arm to ARCH=noarch) + ARCH=noarch + ;; *) AC_MSG_RESULT(no) ;; diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index 1c1fd89825..f93b386392 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -32,6 +32,37 @@ <p>This document describes the changes made to the ERTS application.</p> +<section><title>Erts 8.3.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Trying to open a directory with file:read_file/1 on Unix + leaked a file descriptor. This bug has now been fixed.</p> + <p> + Own Id: OTP-14308 Aux Id: ERL-383 </p> + </item> + </list> + </section> + + + <section><title>Known Bugs and Problems</title> + <list> + <item> + <p> + Invoking <c>init:stop/0</c> via the SIGTERM signal, in a + non-SMP BEAM, could cause BEAM to terminate with fatal + error. This has now been fixed and the BEAM will + terminate normally when SIGTERM is received.</p> + <p> + Own Id: OTP-14290</p> + </item> + </list> + </section> + +</section> + <section><title>Erts 8.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 9a91fdce08..6010c17c17 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -3044,10 +3044,12 @@ do { \ GetArg2(2, Op1, Op2); if (is_both_small(Op1, Op2)) { /* - * We could extract the tag from one argument, but a tag extraction - * could mean a shift. Therefore, play it safe here. + * TAG ^ TAG == 0. + * + * Therefore, we perform the XOR operation on the tagged values, + * and OR in the tag bits. */ - Eterm result = make_small(signed_val(Op1) ^ signed_val(Op2)); + Eterm result = (Op1 ^ Op2) | make_small(0); StoreBifResult(4, result); } DO_OUTLINED_ARITH_2(bxor, Op1, Op2); diff --git a/erts/emulator/drivers/unix/unix_efile.c b/erts/emulator/drivers/unix/unix_efile.c index 3ff68a8859..0acc2432a7 100644 --- a/erts/emulator/drivers/unix/unix_efile.c +++ b/erts/emulator/drivers/unix/unix_efile.c @@ -430,6 +430,9 @@ efile_openfile(Efile_error* errInfo, /* Where to return error codes. */ if ( (stat("/dev/null", &nullstatbuf) < 0) || (statbuf.st_ino != nullstatbuf.st_ino) || (statbuf.st_dev != nullstatbuf.st_dev) ) { +#ifdef HAVE_FSTAT + efile_closefile(fd); +#endif errno = EISDIR; return check_error(-1, errInfo); } diff --git a/erts/epmd/src/epmd_cli.c b/erts/epmd/src/epmd_cli.c index 6fd27d46ea..b10b331cb5 100644 --- a/erts/epmd/src/epmd_cli.c +++ b/erts/epmd/src/epmd_cli.c @@ -46,10 +46,11 @@ void kill_epmd(EpmdVars *g) if ((rval = read_fill(fd,buf,2)) == 2) { if (buf[0] == 'O' && buf[1] == 'K') { printf("Killed\n"); + epmd_cleanup_exit(g,0); } else { printf("Killing not allowed - living nodes in database.\n"); + epmd_cleanup_exit(g,1); } - epmd_cleanup_exit(g,0); } else if (rval < 0) { printf("epmd: failed to read answer from local epmd\n"); epmd_cleanup_exit(g,1); diff --git a/erts/preloaded/ebin/erl_prim_loader.beam b/erts/preloaded/ebin/erl_prim_loader.beam Binary files differindex 5b03019d8e..4b2f07f1a4 100644 --- a/erts/preloaded/ebin/erl_prim_loader.beam +++ b/erts/preloaded/ebin/erl_prim_loader.beam diff --git a/erts/preloaded/ebin/erl_tracer.beam b/erts/preloaded/ebin/erl_tracer.beam Binary files differindex 4cf1b5ed82..c4e1e57c40 100644 --- a/erts/preloaded/ebin/erl_tracer.beam +++ b/erts/preloaded/ebin/erl_tracer.beam diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam Binary files differindex 5d6f36b222..c584fe6f97 100644 --- a/erts/preloaded/ebin/erlang.beam +++ b/erts/preloaded/ebin/erlang.beam diff --git a/erts/preloaded/ebin/erts_code_purger.beam b/erts/preloaded/ebin/erts_code_purger.beam Binary files differindex 0a318b70bb..af7028b01c 100644 --- a/erts/preloaded/ebin/erts_code_purger.beam +++ b/erts/preloaded/ebin/erts_code_purger.beam diff --git a/erts/preloaded/ebin/erts_dirty_process_code_checker.beam b/erts/preloaded/ebin/erts_dirty_process_code_checker.beam Binary files differindex 20ee82a134..a6ecba2a17 100644 --- a/erts/preloaded/ebin/erts_dirty_process_code_checker.beam +++ b/erts/preloaded/ebin/erts_dirty_process_code_checker.beam diff --git a/erts/preloaded/ebin/erts_internal.beam b/erts/preloaded/ebin/erts_internal.beam Binary files differindex c0ff53f503..c4b281163e 100644 --- a/erts/preloaded/ebin/erts_internal.beam +++ b/erts/preloaded/ebin/erts_internal.beam diff --git a/erts/preloaded/ebin/erts_literal_area_collector.beam b/erts/preloaded/ebin/erts_literal_area_collector.beam Binary files differindex e925636787..4552a34ae2 100644 --- a/erts/preloaded/ebin/erts_literal_area_collector.beam +++ b/erts/preloaded/ebin/erts_literal_area_collector.beam diff --git a/erts/preloaded/ebin/init.beam b/erts/preloaded/ebin/init.beam Binary files differindex 92eedd73d8..e90a988a07 100644 --- a/erts/preloaded/ebin/init.beam +++ b/erts/preloaded/ebin/init.beam diff --git a/erts/preloaded/ebin/otp_ring0.beam b/erts/preloaded/ebin/otp_ring0.beam Binary files differindex b91fa63e7d..77b0b7a537 100644 --- a/erts/preloaded/ebin/otp_ring0.beam +++ b/erts/preloaded/ebin/otp_ring0.beam diff --git a/erts/preloaded/ebin/prim_eval.beam b/erts/preloaded/ebin/prim_eval.beam Binary files differindex a011890c1a..ac9916cc86 100644 --- a/erts/preloaded/ebin/prim_eval.beam +++ b/erts/preloaded/ebin/prim_eval.beam diff --git a/erts/preloaded/ebin/prim_file.beam b/erts/preloaded/ebin/prim_file.beam Binary files differindex b5e5ff9f88..2d8a355c82 100644 --- a/erts/preloaded/ebin/prim_file.beam +++ b/erts/preloaded/ebin/prim_file.beam diff --git a/erts/preloaded/ebin/prim_inet.beam b/erts/preloaded/ebin/prim_inet.beam Binary files differindex 994677872c..e2559ac034 100644 --- a/erts/preloaded/ebin/prim_inet.beam +++ b/erts/preloaded/ebin/prim_inet.beam diff --git a/erts/preloaded/ebin/prim_zip.beam b/erts/preloaded/ebin/prim_zip.beam Binary files differindex 6f1c82509f..12cbb5bf7a 100644 --- a/erts/preloaded/ebin/prim_zip.beam +++ b/erts/preloaded/ebin/prim_zip.beam diff --git a/erts/preloaded/ebin/zlib.beam b/erts/preloaded/ebin/zlib.beam Binary files differindex eb9e07af7e..f5217eadf6 100644 --- a/erts/preloaded/ebin/zlib.beam +++ b/erts/preloaded/ebin/zlib.beam diff --git a/erts/preloaded/src/init.erl b/erts/preloaded/src/init.erl index 3dc6953b4c..c12a5b159a 100644 --- a/erts/preloaded/src/init.erl +++ b/erts/preloaded/src/init.erl @@ -263,21 +263,30 @@ boot(Start,Flags,Args) -> boot_loop(BootPid,State). %%% Convert a term to a printable string, if possible. -to_string(X) when is_list(X) -> % assume string +to_string(X, D) when is_list(X), D < 4 -> % assume string F = flatten(X, []), case printable_list(F) of - true -> F; - false -> "" + true when length(F) > 0 -> F; + _false -> + List = [to_string(E, D+1) || E <- X], + flatten(["[",join(List),"]"], []) end; -to_string(X) when is_atom(X) -> +to_string(X, _D) when is_list(X) -> + "[_]"; +to_string(X, _D) when is_atom(X) -> atom_to_list(X); -to_string(X) when is_pid(X) -> +to_string(X, _D) when is_pid(X) -> pid_to_list(X); -to_string(X) when is_float(X) -> +to_string(X, _D) when is_float(X) -> float_to_list(X); -to_string(X) when is_integer(X) -> +to_string(X, _D) when is_integer(X) -> integer_to_list(X); -to_string(_X) -> +to_string(X, D) when is_tuple(X), D < 4 -> + List = [to_string(E, D+1) || E <- tuple_to_list(X)], + flatten(["{",join(List),"}"], []); +to_string(X, _D) when is_tuple(X) -> + "{_}"; +to_string(_X, _D) -> "". % can't do anything with it %% This is an incorrect and narrow definition of printable characters. @@ -291,6 +300,13 @@ printable_list([$\t|T]) -> printable_list(T); printable_list([]) -> true; printable_list(_) -> false. +join([] = T) -> + T; +join([_Elem] = T) -> + T; +join([Elem|T]) -> + [Elem,","|join(T)]. + flatten([H|T], Tail) when is_list(H) -> flatten(H, flatten(T, Tail)); flatten([H|T], Tail) -> @@ -299,7 +315,7 @@ flatten([], Tail) -> Tail. things_to_string([X|Rest]) -> - " (" ++ to_string(X) ++ ")" ++ things_to_string(Rest); + " (" ++ to_string(X, 0) ++ ")" ++ things_to_string(Rest); things_to_string([]) -> "". @@ -307,9 +323,8 @@ halt_string(String, List) -> String ++ things_to_string(List). %% String = string() -%% List = [string() | atom() | pid() | number()] -%% Any other items in List, such as tuples, are ignored when creating -%% the string used as argument to erlang:halt/1. +%% List = [string() | atom() | pid() | number() | list() | tuple()] +%% Items in List are truncated if found to be too large -spec crash(_, _) -> no_return(). crash(String, List) -> halt(halt_string(String, List)). diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml index 6a0d87bcaf..c230148b29 100644 --- a/lib/common_test/doc/src/write_test_chapter.xml +++ b/lib/common_test/doc/src/write_test_chapter.xml @@ -1036,7 +1036,7 @@ Importance >= (100-VerbosityLevel)</pre> <p>Note that the category argument is not required in order to only specify the importance of a printout. Example:</p> <pre> -<c>ct:pal(?LOW_IMPORTANCE, "Info report: ~p", [Info])</c></pre> +ct:pal(?LOW_IMPORTANCE, "Info report: ~p", [Info])</pre> <p>Or perhaps in combination with constants:</p> <pre> -define(INFO, ?LOW_IMPORTANCE). diff --git a/lib/common_test/src/ct_slave.erl b/lib/common_test/src/ct_slave.erl index 571958ca03..4188bd7c3b 100644 --- a/lib/common_test/src/ct_slave.erl +++ b/lib/common_test/src/ct_slave.erl @@ -309,7 +309,12 @@ is_started(ENode) -> % make a Erlang node name from name and hostname enodename(Host, Node) -> - list_to_atom(atom_to_list(Node)++"@"++atom_to_list(Host)). + case lists:member($@, atom_to_list(Node)) of + true -> + Node; + false -> + list_to_atom(atom_to_list(Node)++"@"++atom_to_list(Host)) + end. % performs actual start of the "slave" node do_start(Host, Node, Options) -> diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c index b2f31870b9..1f4ce9a3da 100644 --- a/lib/crypto/c_src/crypto.c +++ b/lib/crypto/c_src/crypto.c @@ -425,10 +425,12 @@ static ERL_NIF_TERM hmac_final_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM static ERL_NIF_TERM cmac_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM block_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM aes_cfb_8_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM aes_cfb_128_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM aes_ige_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM aes_ctr_stream_init(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM aes_ctr_stream_encrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM strong_rand_bytes_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM strong_rand_range_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM rand_uniform_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM mod_exp_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM dss_verify_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); @@ -501,6 +503,7 @@ static ErlNifFunc nif_funcs[] = { {"aes_ctr_stream_encrypt", 2, aes_ctr_stream_encrypt}, {"aes_ctr_stream_decrypt", 2, aes_ctr_stream_encrypt}, {"strong_rand_bytes_nif", 1, strong_rand_bytes_nif}, + {"strong_rand_range_nif", 1, strong_rand_range_nif}, {"rand_uniform_nif", 2, rand_uniform_nif}, {"mod_exp_nif", 4, mod_exp_nif}, {"dss_verify_nif", 4, dss_verify_nif}, @@ -1736,17 +1739,20 @@ static ERL_NIF_TERM block_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM return enif_raise_exception(env, atom_notsup); } - if ((argv[0] == atom_aes_cfb8 || argv[0] == atom_aes_cfb128) - && (key.size == 24 || key.size == 32) -#ifdef FIPS_SUPPORT - && !FIPS_mode() -#endif - ) { + if (argv[0] == atom_aes_cfb8 + && (key.size == 24 || key.size == 32)) { /* Why do EVP_CIPHER_CTX_set_key_length() fail on these key sizes? * Fall back on low level API */ return aes_cfb_8_crypt(env, argc-1, argv+1); } + else if (argv[0] == atom_aes_cfb128 + && (key.size == 24 || key.size == 32)) { + /* Why do EVP_CIPHER_CTX_set_key_length() fail on these key sizes? + * Fall back on low level API + */ + return aes_cfb_128_crypt_nif(env, argc-1, argv+1); + } ivec_size = EVP_CIPHER_iv_length(cipher); @@ -1822,6 +1828,31 @@ static ERL_NIF_TERM aes_cfb_8_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM return ret; } +static ERL_NIF_TERM aes_cfb_128_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Key, IVec, Data, IsEncrypt) */ + ErlNifBinary key, ivec, text; + AES_KEY aes_key; + unsigned char ivec_clone[16]; /* writable copy */ + int new_ivlen = 0; + ERL_NIF_TERM ret; + + if (!enif_inspect_iolist_as_binary(env, argv[0], &key) + || !(key.size == 16 || key.size == 24 || key.size == 32) + || !enif_inspect_binary(env, argv[1], &ivec) || ivec.size != 16 + || !enif_inspect_iolist_as_binary(env, argv[2], &text)) { + return enif_make_badarg(env); + } + + memcpy(ivec_clone, ivec.data, 16); + AES_set_encrypt_key(key.data, key.size * 8, &aes_key); + AES_cfb128_encrypt((unsigned char *) text.data, + enif_make_new_binary(env, text.size, &ret), + text.size, &aes_key, ivec_clone, &new_ivlen, + (argv[3] != atom_true)); + CONSUME_REDS(env,text); + return ret; +} + static ERL_NIF_TERM aes_ige_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {/* (Key, IVec, Data, IsEncrypt) */ #ifdef HAVE_AES_IGE @@ -2331,6 +2362,27 @@ static ERL_NIF_TERM bin_from_bn(ErlNifEnv* env, const BIGNUM *bn) return term; } +static ERL_NIF_TERM strong_rand_range_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Range) */ + BIGNUM *bn_range, *bn_rand; + ERL_NIF_TERM ret; + + if(!get_bn_from_bin(env, argv[0], &bn_range)) { + return enif_make_badarg(env); + } + + bn_rand = BN_new(); + if (BN_rand_range(bn_rand, bn_range) != 1) { + ret = atom_false; + } + else { + ret = bin_from_bn(env, bn_rand); + } + BN_free(bn_rand); + BN_free(bn_range); + return ret; +} + static ERL_NIF_TERM rand_uniform_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {/* (Lo,Hi) */ BIGNUM *bn_from = NULL, *bn_to, *bn_rand; diff --git a/lib/crypto/doc/src/crypto.xml b/lib/crypto/doc/src/crypto.xml index d0deaceaaf..552d95d7dc 100644 --- a/lib/crypto/doc/src/crypto.xml +++ b/lib/crypto/doc/src/crypto.xml @@ -658,10 +658,11 @@ </type> <desc> <p>Set the seed for PRNG to the given binary. This calls the - RAND_seed function from openssl. Only use this if the system - you are running on does not have enough "randomness" built in. - Normally this is when <seealso marker="#strong_rand_bytes/1"> - strong_rand_bytes/1</seealso> returns <c>low_entropy</c></p> + RAND_seed function from openssl. Only use this if the system + you are running on does not have enough "randomness" built in. + Normally this is when + <seealso marker="#strong_rand_bytes/1">strong_rand_bytes/1</seealso> + throws <c>low_entropy</c></p> </desc> </func> @@ -728,6 +729,43 @@ failed due to lack of secure "randomness".</p> </desc> </func> + + <func> + <name>rand_seed() -> rand:state()</name> + <fsummary>Strong random number generation plugin state</fsummary> + <desc> + <p> + Creates state object for + <seealso marker="stdlib:rand">random number generation</seealso>, + in order to generate cryptographically strong random numbers + (based on OpenSSL's <c>BN_rand_range</c>), + and saves it on process dictionary before returning it as well. + See also + <seealso marker="stdlib:rand#seed-1">rand:seed/1</seealso>. + </p> + <p><em>Example</em></p> + <pre> +_ = crypto:rand_seed(), +_IntegerValue = rand:uniform(42), % [1; 42] +_FloatValue = rand:uniform(). % [0.0; 1.0[</pre> + </desc> + </func> + + <func> + <name>rand_seed_s() -> rand:state()</name> + <fsummary>Strong random number generation plugin state</fsummary> + <desc> + <p> + Creates state object for + <seealso marker="stdlib:rand">random number generation</seealso>, + in order to generate cryptographically strongly random numbers + (based on OpenSSL's <c>BN_rand_range</c>). + See also + <seealso marker="stdlib:rand#seed_s-1">rand:seed_s/1</seealso>. + </p> + </desc> + </func> + <func> <name>stream_init(Type, Key) -> State</name> <fsummary></fsummary> diff --git a/lib/crypto/doc/src/notes.xml b/lib/crypto/doc/src/notes.xml index 37997b649b..887aeca680 100644 --- a/lib/crypto/doc/src/notes.xml +++ b/lib/crypto/doc/src/notes.xml @@ -31,6 +31,22 @@ </header> <p>This document describes the changes made to the Crypto application.</p> +<section><title>Crypto 3.7.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix a bug with AES CFB 128 for 192 and 256 bit keys. + Thanks to kellymclaughlin !</p> + <p> + Own Id: OTP-14313 Aux Id: PR-1393 </p> + </item> + </list> + </section> + +</section> + <section><title>Crypto 3.7.3</title> <section><title>Improvements and New Features</title> diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index d322765dff..1287ec6176 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -30,6 +30,12 @@ -export([hmac/3, hmac/4, hmac_init/2, hmac_update/2, hmac_final/1, hmac_final_n/2]). -export([cmac/3, cmac/4]). -export([exor/2, strong_rand_bytes/1, mod_pow/3]). +-export([rand_seed/0]). +-export([rand_seed_s/0]). +-export([rand_plugin_next/1]). +-export([rand_plugin_uniform/1]). +-export([rand_plugin_uniform/2]). +-export([rand_plugin_jump/1]). -export([rand_uniform/2]). -export([block_encrypt/3, block_decrypt/3, block_encrypt/4, block_decrypt/4]). -export([next_iv/2, next_iv/3]). @@ -45,6 +51,9 @@ %% This should correspond to the similar macro in crypto.c -define(MAX_BYTES_TO_NIF, 20000). %% Current value is: erlang:system_info(context_reductions) * 10 +%% Used by strong_rand_float/0 +-define(HALF_DBL_EPSILON, 1.1102230246251565e-16). % math:pow(2, -53) + %%-type ecdsa_digest_type() :: 'md5' | 'sha' | 'sha256' | 'sha384' | 'sha512'. -type crypto_integer() :: binary() | integer(). %%-type ec_named_curve() :: atom(). @@ -286,9 +295,11 @@ stream_decrypt(State, Data0) -> stream_crypt(fun do_stream_decrypt/2, State, Data, erlang:byte_size(Data), MaxByts, []). %% -%% RAND - pseudo random numbers using RN_ functions in crypto lib +%% RAND - pseudo random numbers using RN_ and BN_ functions in crypto lib %% -spec strong_rand_bytes(non_neg_integer()) -> binary(). +-spec rand_seed() -> rand:state(). +-spec rand_seed_s() -> rand:state(). -spec rand_uniform(crypto_integer(), crypto_integer()) -> crypto_integer(). @@ -300,6 +311,46 @@ strong_rand_bytes(Bytes) -> strong_rand_bytes_nif(_Bytes) -> ?nif_stub. +rand_seed() -> + rand:seed(rand_seed_s()). + +rand_seed_s() -> + {#{ type => ?MODULE, + max => infinity, + next => fun ?MODULE:rand_plugin_next/1, + uniform => fun ?MODULE:rand_plugin_uniform/1, + uniform_n => fun ?MODULE:rand_plugin_uniform/2, + jump => fun ?MODULE:rand_plugin_jump/1}, + no_seed}. + +rand_plugin_next(Seed) -> + {bytes_to_integer(strong_rand_range(1 bsl 64)), Seed}. + +rand_plugin_uniform(State) -> + {strong_rand_float(), State}. + +rand_plugin_uniform(Max, State) -> + {bytes_to_integer(strong_rand_range(Max)) + 1, State}. + +rand_plugin_jump(State) -> + State. + +strong_rand_range(Range) when is_integer(Range), Range > 0 -> + BinRange = int_to_bin(Range), + strong_rand_range(BinRange); +strong_rand_range(BinRange) when is_binary(BinRange) -> + case strong_rand_range_nif(BinRange) of + false -> + erlang:error(low_entropy); + <<BinResult/binary>> -> + BinResult + end. +strong_rand_range_nif(_BinRange) -> ?nif_stub. + +strong_rand_float() -> + WholeRange = strong_rand_range(1 bsl 53), + ?HALF_DBL_EPSILON * bytes_to_integer(WholeRange). + rand_uniform(From,To) when is_binary(From), is_binary(To) -> case rand_uniform_nif(From,To) of <<Len:32/integer, MSB, Rest/binary>> when MSB > 127 -> @@ -328,6 +379,7 @@ rand_uniform_pos(_,_) -> rand_uniform_nif(_From,_To) -> ?nif_stub. + -spec rand_seed(binary()) -> ok. rand_seed(Seed) -> rand_seed_nif(Seed). diff --git a/lib/crypto/test/crypto_SUITE.erl b/lib/crypto/test/crypto_SUITE.erl index 1d7037d003..1b7456af18 100644 --- a/lib/crypto/test/crypto_SUITE.erl +++ b/lib/crypto/test/crypto_SUITE.erl @@ -36,7 +36,9 @@ all() -> {group, non_fips}, mod_pow, exor, - rand_uniform + rand_uniform, + rand_plugin, + rand_plugin_s ]. groups() -> @@ -486,6 +488,17 @@ rand_uniform(Config) when is_list(Config) -> 10 = byte_size(crypto:strong_rand_bytes(10)). %%-------------------------------------------------------------------- +rand_plugin() -> + [{doc, "crypto rand plugin testing (implicit state / process dictionary)"}]. +rand_plugin(Config) when is_list(Config) -> + rand_plugin_aux(implicit_state). + +rand_plugin_s() -> + [{doc, "crypto rand plugin testing (explicit state)"}]. +rand_plugin_s(Config) when is_list(Config) -> + rand_plugin_aux(explicit_state). + +%%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- hash(_, [], []) -> @@ -951,6 +964,101 @@ crypto_rand_uniform(L,H) -> ct:fail({"Not in interval", R1, L, H}) end. +foldallmap(_Fun, AccN, []) -> + {true, AccN}; +foldallmap(Fun, AccN, [H|T]) -> + case Fun(H, AccN) of + {true, AccM} -> foldallmap(Fun, AccM, T); + {{false, Result}, AccM} -> {Result, AccM} + end. + +allmap(_Fun, []) -> + true; +allmap(Fun, [H|T]) -> + case Fun(H) of + true -> allmap(Fun, T); + {false, Result} -> Result + end. + +rand_plugin_aux(StateType) -> + {Seeder, SeedExporter, FloatGenerator, IntegerGenerator} = rand_plugin_functions(StateType), + State0 = Seeder(), + {crypto, no_seed} = SeedExporter(State0), + {FloatTestResult, State1} = rand_plugin_aux_floats(State0, FloatGenerator), + case FloatTestResult of + true -> + {IntegerTestResult, _State2} = rand_plugin_aux_integers(State1, IntegerGenerator), + IntegerTestResult; + {false, _} -> + FloatTestResult + end. + +% returns {Seeder, SeedExporter, FloatGenerator, IntegerGenerator} with consistent signatures +rand_plugin_functions(implicit_state) -> + {fun () -> crypto:rand_seed(), implicit_state end, + fun (implicit_state) -> rand:export_seed() end, + fun (implicit_state) -> {rand:uniform(), implicit_state} end, + fun (N, implicit_state) -> {rand:uniform(N), implicit_state} end}; +rand_plugin_functions(explicit_state) -> + {fun crypto:rand_seed_s/0, + fun rand:export_seed_s/1, + fun rand:uniform_s/1, + fun rand:uniform_s/2}. + +rand_plugin_aux_floats(State0, FloatGenerator) -> + {FloatSamples, State1} = + lists:mapfoldl( + fun (_, StateAcc) -> + FloatGenerator(StateAcc) + end, + State0, + lists:seq(1, 10000)), + + {allmap( + fun (V) -> + (V >= 0.0 andalso V < 1.0) + orelse {false, ct:fail({"Float sample not in interval", V, 0.0, 1.0})} + end, + FloatSamples), + State1}. + +rand_plugin_aux_integers(State0, IntegerGenerator) -> + MaxIntegerCeiling = 1 bsl 32, + {IntegerCeilings, State1} = + lists:mapfoldl( + fun (_, StateAcc) -> + IntegerGenerator(MaxIntegerCeiling, StateAcc) + end, + State0, + lists:seq(1, 100)), + + foldallmap( + fun (Ceiling, StateAcc) -> + case Ceiling >= 1 andalso Ceiling =< MaxIntegerCeiling of + false -> + {{false, ct:fail({"Integer ceiling not in interval", + Ceiling, 1, MaxIntegerCeiling})}, + StateAcc}; + true -> + foldallmap( + fun (_, SubStateAcc) -> + {Sample, NewSubStateAcc} = IntegerGenerator(Ceiling, SubStateAcc), + case Sample >= 1 andalso Sample =< Ceiling of + false -> + {{false, ct:fail({"Integer sample not in interval", + Sample, 1, Ceiling})}, + NewSubStateAcc}; + true -> + {true, NewSubStateAcc} + end + end, + StateAcc, + lists:seq(1, 100)) + end + end, + State1, + IntegerCeilings). + %%-------------------------------------------------------------------- %% Test data ------------------------------------------------ %%-------------------------------------------------------------------- diff --git a/lib/crypto/vsn.mk b/lib/crypto/vsn.mk index 81cb2f8130..f3e0623ac9 100644 --- a/lib/crypto/vsn.mk +++ b/lib/crypto/vsn.mk @@ -1 +1 @@ -CRYPTO_VSN = 3.7.3 +CRYPTO_VSN = 3.7.4 diff --git a/lib/debugger/src/dbg_ieval.erl b/lib/debugger/src/dbg_ieval.erl index f5e079ef7e..88c7caacb0 100644 --- a/lib/debugger/src/dbg_ieval.erl +++ b/lib/debugger/src/dbg_ieval.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2016. All Rights Reserved. +%% Copyright Ericsson AB 1998-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1486,7 +1486,6 @@ guard_expr({map,_,E0,Fs0}, Bs) -> Value = lists:foldl(fun ({map_assoc,K,V}, Mi) -> maps:put(K,V,Mi); ({map_exact,K,V}, Mi) -> maps:update(K,V,Mi) end, E, Fs), - io:format("~p~n", [{E,Value}]), {value,Value}; guard_expr({bin,_,Flds}, Bs) -> {value,V,_Bs} = diff --git a/lib/debugger/src/dbg_wx_code.erl b/lib/debugger/src/dbg_wx_code.erl index 473963500a..bca8a0d241 100644 --- a/lib/debugger/src/dbg_wx_code.erl +++ b/lib/debugger/src/dbg_wx_code.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2016. All Rights Reserved. +%% Copyright Ericsson AB 2008-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -190,6 +190,6 @@ find(Ed, Str, Case, Next) -> keyWords() -> L = ["after","begin","case","try","cond","catch","andalso","orelse", - "end","fun","if","let","of","query","receive","when","bnot","not", + "end","fun","if","let","of","receive","when","bnot","not", "div","rem","band","and","bor","bxor","bsl","bsr","or","xor"], lists:flatten([K ++ " " || K <- L] ++ [0]). diff --git a/lib/debugger/src/dbg_wx_src_view.erl b/lib/debugger/src/dbg_wx_src_view.erl index 207c407fbc..ee8eb72407 100644 --- a/lib/debugger/src/dbg_wx_src_view.erl +++ b/lib/debugger/src/dbg_wx_src_view.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2016. All Rights Reserved. +%% Copyright Ericsson AB 2008-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -62,6 +62,6 @@ code_area(Parent, Sizer) -> keyWords() -> L = ["after","begin","case","try","cond","catch","andalso","orelse", - "end","fun","if","let","of","query","receive","when","bnot","not", + "end","fun","if","let","of","receive","when","bnot","not", "div","rem","band","and","bor","bxor","bsl","bsr","or","xor"], lists:flatten([K ++ " " || K <- L] ++ [0]). diff --git a/lib/debugger/src/dbg_wx_trace.erl b/lib/debugger/src/dbg_wx_trace.erl index 29c8e8cefb..f4ee30618c 100644 --- a/lib/debugger/src/dbg_wx_trace.erl +++ b/lib/debugger/src/dbg_wx_trace.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2016. All Rights Reserved. +%% Copyright Ericsson AB 2008-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -524,7 +524,8 @@ gui_cmd({edit, {Var, Value}}, State) -> cancel -> State; {Var, Term} -> - Cmd = atom_to_list(Var)++"="++io_lib:format("~w", [Term]), + %% The space after "=" is needed for handling "B= <<1>>". + Cmd = atom_to_list(Var)++"= "++io_lib:format("~w", [Term]), gui_cmd({user_command, lists:flatten(Cmd)}, State) end. diff --git a/lib/debugger/src/int.erl b/lib/debugger/src/int.erl index e5bade9abe..fdf5957182 100644 --- a/lib/debugger/src/int.erl +++ b/lib/debugger/src/int.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2016. All Rights Reserved. +%% Copyright Ericsson AB 1998-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -533,7 +533,9 @@ load({Mod, Src, Beam, BeamBin, Exp, Abst}, Dist) -> case erl_prim_loader:get_file(filename:absname(Src)) of {ok, SrcBin, _} -> MD5 = code:module_md5(BeamBin), - Bin = term_to_binary({interpreter_module,Exp,Abst,SrcBin,MD5}), + SrcBin1 = unicode:characters_to_binary(SrcBin, enc(SrcBin)), + true = is_binary(SrcBin1), + Bin = term_to_binary({interpreter_module,Exp,Abst,SrcBin1,MD5}), {module, Mod} = dbg_iserver:safe_call({load, Mod, Src, Bin}), _ = everywhere(Dist, fun() -> diff --git a/lib/edoc/src/edoc.erl b/lib/edoc/src/edoc.erl index 7276a57268..b641118c5d 100644 --- a/lib/edoc/src/edoc.erl +++ b/lib/edoc/src/edoc.erl @@ -197,7 +197,7 @@ application(App, Dir, Options) when is_atom(App) -> ?OVERVIEW_FILE), Opts = Options ++ [{source_path, [Src]}, subpackages, - {title, io_lib:fwrite("The ~s application", [App])}, + {title, io_lib:fwrite("The ~ts application", [App])}, {overview, Overview}, {dir, filename:join(Dir, ?EDOC_DIR)}, {includes, [filename:join(Dir, "include")]}], diff --git a/lib/edoc/src/edoc_doclet.erl b/lib/edoc/src/edoc_doclet.erl index 006b07574b..6e17ec0af0 100644 --- a/lib/edoc/src/edoc_doclet.erl +++ b/lib/edoc/src/edoc_doclet.erl @@ -152,7 +152,7 @@ title(App, Options) -> if App == ?NO_APP -> "Overview"; true -> - io_lib:fwrite("Application: ~s", [App]) + io_lib:fwrite("Application: ~ts", [App]) end). diff --git a/lib/edoc/src/edoc_extract.erl b/lib/edoc/src/edoc_extract.erl index 68edad1a3e..390851e9ef 100644 --- a/lib/edoc/src/edoc_extract.erl +++ b/lib/edoc/src/edoc_extract.erl @@ -488,8 +488,15 @@ find_names([P | Ps], Ns) -> find_names([P1 | Ps], Ns); record_expr -> A = erl_syntax:record_expr_type(P), - N = list_to_atom(capitalize(erl_syntax:atom_name(A))), - find_names(Ps, [N | Ns]); + AtomName = erl_syntax:atom_name(A), + Atom = list_to_atom(AtomName), + case AtomName =:= lists:flatten(io_lib:write_atom(Atom)) of + true -> + N = list_to_atom(capitalize(AtomName)), + find_names(Ps, [N | Ns]); + false -> + find_names(Ps, Ns) + end; infix_expr -> %% this can only be a '++' operation P1 = erl_syntax:infix_expr_right(P), @@ -540,6 +547,7 @@ tidy_name_1(Cs) -> [$_ | Cs]. %% Change initial character from lowercase to uppercase. capitalize([C | Cs]) when C >= $a, C =< $z -> [C - 32 | Cs]; +capitalize([C | Cs]) when C >= $\340, C =< $\376, C /= $\367 -> [C - 32 | Cs]; capitalize(Cs) -> Cs. %% Collects the tags belonging to each entry, checks them, expands diff --git a/lib/edoc/src/edoc_layout.erl b/lib/edoc/src/edoc_layout.erl index 5ef210980c..5b1889ab06 100644 --- a/lib/edoc/src/edoc_layout.erl +++ b/lib/edoc/src/edoc_layout.erl @@ -109,14 +109,20 @@ module(Element, Options) -> stylesheet, index_columns, sort_functions, + encoding, pretty_printer}). init_opts(Element, Options) -> + Encoding = case get_attrval(encoding, Element) of + "latin1" -> latin1; + _ -> utf8 + end, R = #opts{root = get_attrval(root, Element), index_columns = proplists:get_value(index_columns, Options, 1), sort_functions = proplists:get_value(sort_functions, Options, true), + encoding = Encoding, pretty_printer = proplists:get_value(pretty_printer, Options, '') }, @@ -183,8 +189,9 @@ layout_module(#xmlElement{name = module, content = Es}=E, Opts) -> Desc = get_content(description, Es), ShortDesc = get_content(briefDescription, Desc), FullDesc = get_content(fullDescription, Desc), - Functions = [{function_name(E), E} || E <- get_content(functions, Es)], - Types = [{type_name(E), E} || E <- get_content(typedecls, Es)], + Functions = [{function_name(E, Opts), E} || + E <- get_content(functions, Es)], + Types = [{type_name(E, Opts), E} || E <- get_content(typedecls, Es)], SortedFs = if Opts#opts.sort_functions -> lists:sort(Functions); true -> Functions end, @@ -198,7 +205,7 @@ layout_module(#xmlElement{name = module, content = Es}=E, Opts) -> ++ [?NL] ++ version(Es) ++ since(Es) - ++ behaviours(Es, Name) + ++ behaviours(Es, Name, Opts) ++ authors(Es) ++ references(Es) ++ sees(Es) @@ -215,7 +222,7 @@ layout_module(#xmlElement{name = module, content = Es}=E, Opts) -> ++ [hr, ?NL] ++ navigation("bottom") ++ footer()), - Encoding = get_attrval(encoding, E), + Encoding = Opts#opts.encoding, xhtml(Title, stylesheet(Opts), Body, Encoding). module_params(Es) -> @@ -367,7 +374,7 @@ function(Name, E=#xmlElement{content = Es}, Opts) -> case typespec(get_content(typespec, Es), Opts) of [] -> signature(get_content(args, Es), - get_attrval(name, E)); + atom(get_attrval(name, E), Opts)); Spec -> Spec end}, ?NL] @@ -387,8 +394,8 @@ function(Name, E=#xmlElement{content = Es}, Opts) -> ++ sees(Es) ++ todos(Es)). -function_name(E) -> - atom(get_attrval(name, E)) ++ "/" ++ get_attrval(arity, E). +function_name(E, Opts) -> + atom(get_attrval(name, E), Opts) ++ "/" ++ get_attrval(arity, E). function_header(Name, E, Private) -> case is_exported(E) of @@ -449,7 +456,7 @@ throws(Es, Opts) -> [] -> []; Es1 -> %% Doesn't use format_type; keep it short! - [{p, (["throws ", {tt, t_utype(get_elem(type, Es1))}] + [{p, (["throws ", {tt, t_utype(get_elem(type, Es1), Opts)}] ++ local_defs(get_elem(localdef, Es1), Opts))}, ?NL] end. @@ -458,7 +465,7 @@ throws(Es, Opts) -> typespec([], _Opts) -> []; typespec(Es, Opts) -> - Name = t_name(get_elem(erlangName, Es)), + Name = t_name(get_elem(erlangName, Es), Opts), Defs = get_elem(localdef, Es), [Type] = get_elem(type, Es), format_spec(Name, Type, Defs, Opts) ++ local_defs(Defs, Opts). @@ -479,12 +486,12 @@ typedecl(Name, E=#xmlElement{content = Es}, Opts) -> ++ [{p, typedef(get_content(typedef, Es), Opts)}, ?NL] ++ fulldesc(Es)). -type_name(#xmlElement{content = Es}) -> - t_name(get_elem(erlangName, get_content(typedef, Es))). +type_name(#xmlElement{content = Es}, Opts) -> + t_name(get_elem(erlangName, get_content(typedef, Es)), Opts). typedef(Es, Opts) -> - Name = ([t_name(get_elem(erlangName, Es)), "("] - ++ seq(fun t_utype_elem/1, get_content(argtypes, Es), [")"])), + Name = ([t_name(get_elem(erlangName, Es), Opts), "("] + ++ seq(t_utype_elem_fun(Opts), get_content(argtypes, Es), [")"])), (case get_elem(type, Es) of [] -> [{b, ["abstract datatype"]}, ": ", {tt, Name}]; Type -> format_type(Name, Name, Type, [], Opts) @@ -505,7 +512,9 @@ local_defs(Es0, Last, Opts) -> localdef(E = #xmlElement{content = Es}, Last, Opts) -> Name = case get_elem(typevar, Es) of [] -> - label_anchor(N0 = t_abstype(get_content(abstype, Es)), E); + label_anchor(N0 = t_abstype(get_content(abstype, Es), + Opts), + E); [V] -> N0 = t_var(V) end, @@ -516,97 +525,99 @@ localdef(E = #xmlElement{content = Es}, Last, Opts) -> %% (fast) Erlang pretty printer). format_spec(Name, Type, Defs, #opts{pretty_printer = erl_pp}=Opts) -> try - L = t_clause(Name, Type), - O = pp_clause(Name, Type), - {R, ".\n"} = etypef(L, O), + L = t_clause(Name, Type, Opts), + O = pp_clause(Name, Type, Opts), + {R, ".\n"} = etypef(L, O, Opts), [{pre, R}] catch _:_ -> %% Should not happen. format_spec(Name, Type, Defs, Opts#opts{pretty_printer=''}) end; -format_spec(Sep, Type, Defs, _Opts) -> +format_spec(Sep, Type, Defs, Opts) -> %% Very limited formatting. Br = if Defs =:= [] -> br; true -> [] end, - [{tt, t_clause(Sep, Type)}, Br]. + [{tt, t_clause(Sep, Type, Opts)}, Br]. -t_clause(Name, Type) -> +t_clause(Name, Type, Opts) -> #xmlElement{content = [#xmlElement{name = 'fun', content = C}]} = Type, - [Name] ++ t_fun(C). + [Name] ++ t_fun(C, Opts). -pp_clause(Pre, Type) -> +pp_clause(Pre, Type, Opts) -> Types = ot_utype([Type]), - Atom = lists:duplicate(iolist_size(Pre), $a), + Atom = lists:duplicate(string_length(Pre), $a), Attr = {attribute,0,spec,{{list_to_atom(Atom),0},[Types]}}, - L1 = erl_pp:attribute(erl_parse:new_anno(Attr)), + L1 = erl_pp:attribute(erl_parse:new_anno(Attr), + [{encoding, Opts#opts.encoding}]), "-spec " ++ L2 = lists:flatten(L1), L3 = Pre ++ lists:nthtail(length(Atom), L2), - re:replace(L3, "\n ", "\n", [{return,list},global]). + re:replace(L3, "\n ", "\n", [{return,list},global,unicode]). format_type(Prefix, Name, Type, Last, #opts{pretty_printer = erl_pp}=Opts) -> try - L = t_utype(Type), - O = pp_type(Name, Type), - {R, ".\n"} = etypef(L, O), + L = t_utype(Type, Opts), + O = pp_type(Name, Type, Opts), + {R, ".\n"} = etypef(L, O, Opts), [{pre, Prefix ++ [" = "] ++ R ++ Last}] catch _:_ -> %% Example: "t() = record(a)." format_type(Prefix, Name, Type, Last, Opts#opts{pretty_printer =''}) end; -format_type(Prefix, _Name, Type, Last, _Opts) -> - [{tt, Prefix ++ [" = "] ++ t_utype(Type) ++ Last}]. +format_type(Prefix, _Name, Type, Last, Opts) -> + [{tt, Prefix ++ [" = "] ++ t_utype(Type, Opts) ++ Last}]. -pp_type(Prefix, Type) -> - Atom = list_to_atom(lists:duplicate(iolist_size(Prefix), $a)), +pp_type(Prefix, Type, Opts) -> + Atom = list_to_atom(lists:duplicate(string_length(Prefix), $a)), Attr = {attribute,0,type,{Atom,ot_utype(Type),[]}}, - L1 = erl_pp:attribute(erl_parse:new_anno(Attr)), + L1 = erl_pp:attribute(erl_parse:new_anno(Attr), + [{encoding, Opts#opts.encoding}]), {L2,N} = case lists:dropwhile(fun(C) -> C =/= $: end, lists:flatten(L1)) of ":: " ++ L3 -> {L3,9}; % compensation for extra "()" and ":" "::\n" ++ L3 -> {"\n"++L3,6} end, Ss = lists:duplicate(N, $\s), - re:replace(L2, "\n"++Ss, "\n", [{return,list},global]). + re:replace(L2, "\n"++Ss, "\n", [{return,list},global,unicode]). -etypef(L, O0) -> - {R, O} = etypef(L, [], O0, []), +etypef(L, O0, Opts) -> + {R, O} = etypef(L, [], O0, [], Opts), {lists:reverse(R), O}. -etypef([C | L], St, [C | O], R) -> - etypef(L, St, O, [[C] | R]); -etypef(" "++L, St, O, R) -> - etypef(L, St, O, R); -etypef("", [Cs | St], O, R) -> - etypef(Cs, St, O, R); -etypef("", [], O, R) -> +etypef([C | L], St, [C | O], R, Opts) -> + etypef(L, St, O, [[C] | R], Opts); +etypef(" "++L, St, O, R, Opts) -> + etypef(L, St, O, R, Opts); +etypef("", [Cs | St], O, R, Opts) -> + etypef(Cs, St, O, R, Opts); +etypef("", [], O, R, _Opts) -> {R, O}; -etypef(L, St, " "++O, R) -> - etypef(L, St, O, [" " | R]); -etypef(L, St, "\n"++O, R) -> +etypef(L, St, " "++O, R, Opts) -> + etypef(L, St, O, [" " | R], Opts); +etypef(L, St, "\n"++O, R, Opts) -> Ss = lists:takewhile(fun(C) -> C =:= $\s end, O), - etypef(L, St, lists:nthtail(length(Ss), O), ["\n"++Ss | R]); -etypef([{a, HRef, S0} | L], St, O0, R) -> - {S, O} = etypef(S0, app_fix(O0)), - etypef(L, St, O, [{a, HRef, S} | R]); -etypef("="++L, St, "::"++O, R) -> + etypef(L, St, lists:nthtail(length(Ss), O), ["\n"++Ss | R], Opts); +etypef([{a, HRef, S0} | L], St, O0, R, Opts) -> + {S, O} = etypef(S0, app_fix(O0, Opts), Opts), + etypef(L, St, O, [{a, HRef, S} | R], Opts); +etypef("="++L, St, "::"++O, R, Opts) -> %% EDoc uses "=" for record field types; Erlang types use "::". %% Maybe there should be an option for this, possibly affecting %% other similar discrepancies. - etypef(L, St, O, ["=" | R]); -etypef([Cs | L], St, O, R) -> - etypef(Cs, [L | St], O, R). + etypef(L, St, O, ["=" | R], Opts); +etypef([Cs | L], St, O, R, Opts) -> + etypef(Cs, [L | St], O, R, Opts). -app_fix(L) -> +app_fix(L, Opts) -> try - {"//" ++ R1,L2} = app_fix(L, 1), + {"//" ++ R1,L2} = app_fix1(L, 1), [App, Mod] = string:tokens(R1, "/"), - "//" ++ atom(App) ++ "/" ++ atom(Mod) ++ L2 + "//" ++ atom(App, Opts) ++ "/" ++ atom(Mod, Opts) ++ L2 catch _:_ -> L end. -app_fix(L, I) -> % a bit slow +app_fix1(L, I) -> % a bit slow {L1, L2} = lists:split(I, L), case erl_scan:tokens([], L1 ++ ". ", 1) of {done, {ok,[{atom,_,Atom}|_],_}, _} -> {atom_to_list(Atom), L2}; - _ -> app_fix(L, I+1) + _ -> app_fix1(L, I+1) end. fulldesc(Es) -> @@ -703,7 +714,7 @@ deprecated(Es, S) -> ?NL] end. -behaviours(Es, Name) -> +behaviours(Es, Name, Opts) -> CBs = get_content(callbacks, Es), OCBs = get_content(optional_callbacks, Es), (case get_elem(behaviour, Es) of @@ -717,17 +728,18 @@ behaviours(Es, Name) -> if CBs =:= [], OCBs =:= [] -> []; true -> + CBFun = fun(E) -> callback(E, Opts) end, Req = if CBs =:= [] -> []; true -> [br, " Required callback functions: "] - ++ seq(fun callback/1, CBs, ["."]) + ++ seq(CBFun, CBs, ["."]) end, Opt = if OCBs =:= [] -> []; true -> [br, " Optional callback functions: "] - ++ seq(fun callback/1, OCBs, ["."]) + ++ seq(CBFun, OCBs, ["."]) end, [{p, ([{b, ["This module defines the ", {tt, [Name]}, " behaviour."]}] @@ -738,10 +750,10 @@ behaviours(Es, Name) -> behaviour(E=#xmlElement{content = Es}) -> see(E, [{tt, Es}]). -callback(E=#xmlElement{}) -> +callback(E=#xmlElement{}, Opts) -> Name = get_attrval(name, E), Arity = get_attrval(arity, E), - [{tt, [Name, "/", Arity]}]. + [{tt, [atom(Name, Opts), "/", Arity]}]. authors(Es) -> case get_elem(author, Es) of @@ -751,9 +763,26 @@ authors(Es) -> ?NL] end. -atom(String) -> +atom(String, #opts{encoding = latin1}) -> + io_lib:write_atom_as_latin1(list_to_atom(String)); +atom(String, #opts{encoding = utf8}) -> io_lib:write_atom(list_to_atom(String)). +-dialyzer({nowarn_function, string_length/1}). +string_length(Data) -> + try iolist_size(Data) + catch + _:_ -> + M = string, + F = length, + As = [Data], + try apply(M, F, As) + catch + _:_ -> + 20 + end + end. + %% <!ATTLIST author %% name CDATA #REQUIRED %% email CDATA #IMPLIED @@ -799,70 +828,73 @@ todos(Es) -> ?NL] end. -t_name([E]) -> +t_name([E], Opts) -> N = get_attrval(name, E), case get_attrval(module, E) of - "" -> atom(N); + "" -> atom(N, Opts); M -> - S = atom(M) ++ ":" ++ atom(N), + S = atom(M, Opts) ++ ":" ++ atom(N, Opts), case get_attrval(app, E) of "" -> S; - A -> "//" ++ atom(A) ++ "/" ++ S + A -> "//" ++ atom(A, Opts) ++ "/" ++ S end end. -t_utype([E]) -> - t_utype_elem(E). +t_utype([E], Opts) -> + t_utype_elem(E, Opts). -t_utype_elem(E=#xmlElement{content = Es}) -> +t_utype_elem_fun(Opts) -> + fun(E) -> t_utype_elem(E, Opts) end. + +t_utype_elem(E=#xmlElement{content = Es}, Opts) -> case get_attrval(name, E) of - "" -> t_type(Es); + "" -> t_type(Es, Opts); Name -> - T = t_type(Es), + T = t_type(Es, Opts), case T of [Name] -> T; % avoid generating "Foo::Foo" T -> [Name] ++ ["::"] ++ T end end. -t_type([E=#xmlElement{name = typevar}]) -> +t_type([E=#xmlElement{name = typevar}], _Opts) -> t_var(E); -t_type([E=#xmlElement{name = atom}]) -> - t_atom(E); -t_type([E=#xmlElement{name = integer}]) -> +t_type([E=#xmlElement{name = atom}], Opts) -> + t_atom(E, Opts); +t_type([E=#xmlElement{name = integer}], _Opts) -> t_integer(E); -t_type([E=#xmlElement{name = range}]) -> +t_type([E=#xmlElement{name = range}], _Opts) -> t_range(E); -t_type([E=#xmlElement{name = binary}]) -> +t_type([E=#xmlElement{name = binary}], _Opts) -> t_binary(E); -t_type([E=#xmlElement{name = float}]) -> +t_type([E=#xmlElement{name = float}], _Opts) -> t_float(E); -t_type([#xmlElement{name = nil}]) -> +t_type([#xmlElement{name = nil}], _Opts) -> t_nil(); -t_type([#xmlElement{name = paren, content = Es}]) -> - t_paren(Es); -t_type([#xmlElement{name = list, content = Es}]) -> - t_list(Es); -t_type([#xmlElement{name = nonempty_list, content = Es}]) -> - t_nonempty_list(Es); -t_type([#xmlElement{name = map, content = Es}]) -> - t_map(Es); -t_type([#xmlElement{name = tuple, content = Es}]) -> - t_tuple(Es); -t_type([#xmlElement{name = 'fun', content = Es}]) -> - ["fun("] ++ t_fun(Es) ++ [")"]; -t_type([E = #xmlElement{name = record, content = Es}]) -> - t_record(E, Es); -t_type([E = #xmlElement{name = abstype, content = Es}]) -> - t_abstype(E, Es); -t_type([#xmlElement{name = union, content = Es}]) -> - t_union(Es). +t_type([#xmlElement{name = paren, content = Es}], Opts) -> + t_paren(Es, Opts); +t_type([#xmlElement{name = list, content = Es}], Opts) -> + t_list(Es, Opts); +t_type([#xmlElement{name = nonempty_list, content = Es}], Opts) -> + t_nonempty_list(Es, Opts); +t_type([#xmlElement{name = map, content = Es}], Opts) -> + t_map(Es, Opts); +t_type([#xmlElement{name = tuple, content = Es}], Opts) -> + t_tuple(Es, Opts); +t_type([#xmlElement{name = 'fun', content = Es}], Opts) -> + ["fun("] ++ t_fun(Es, Opts) ++ [")"]; +t_type([E = #xmlElement{name = record, content = Es}], Opts) -> + t_record(E, Es, Opts); +t_type([E = #xmlElement{name = abstype, content = Es}], Opts) -> + t_abstype(E, Es, Opts); +t_type([#xmlElement{name = union, content = Es}], Opts) -> + t_union(Es, Opts). t_var(E) -> [get_attrval(name, E)]. -t_atom(E) -> - [get_attrval(value, E)]. +t_atom(E, Opts) -> + [atom(get_attrval(value, E), Opts)]. t_integer(E) -> [get_attrval(value, E)]. @@ -879,62 +911,64 @@ t_float(E) -> t_nil() -> ["[]"]. -t_paren(Es) -> - ["("] ++ t_utype(get_elem(type, Es)) ++ [")"]. +t_paren(Es, Opts) -> + ["("] ++ t_utype(get_elem(type, Es), Opts) ++ [")"]. -t_list(Es) -> - ["["] ++ t_utype(get_elem(type, Es)) ++ ["]"]. +t_list(Es, Opts) -> + ["["] ++ t_utype(get_elem(type, Es), Opts) ++ ["]"]. -t_nonempty_list(Es) -> - ["["] ++ t_utype(get_elem(type, Es)) ++ [", ...]"]. +t_nonempty_list(Es, Opts) -> + ["["] ++ t_utype(get_elem(type, Es), Opts) ++ [", ...]"]. -t_tuple(Es) -> - ["{"] ++ seq(fun t_utype_elem/1, Es, ["}"]). +t_tuple(Es, Opts) -> + ["{"] ++ seq(t_utype_elem_fun(Opts), Es, ["}"]). -t_fun(Es) -> - ["("] ++ seq(fun t_utype_elem/1, get_content(argtypes, Es), - [") -> "] ++ t_utype(get_elem(type, Es))). +t_fun(Es, Opts) -> + ["("] ++ seq(t_utype_elem_fun(Opts), get_content(argtypes, Es), + [") -> "] ++ t_utype(get_elem(type, Es), Opts)). -t_map(Es) -> +t_map(Es, Opts) -> Fs = get_elem(map_field, Es), - ["#{"] ++ seq(fun t_map_field/1, Fs, ["}"]). + ["#{"] ++ seq(fun(E) -> t_map_field(E, Opts) end, Fs, ["}"]). -t_map_field(#xmlElement{content = [K,V]}=E) -> - KElem = t_utype_elem(K), - VElem = t_utype_elem(V), +t_map_field(#xmlElement{content = [K,V]}=E, Opts) -> + KElem = t_utype_elem(K, Opts), + VElem = t_utype_elem(V, Opts), AS = case get_attrval(assoc_type, E) of "assoc" -> " => "; "exact" -> " := " end, KElem ++ [AS] ++ VElem. -t_record(E, Es) -> - Name = ["#"] ++ t_type(get_elem(atom, Es)), +t_record(E, Es, Opts) -> + Name = ["#"] ++ t_type(get_elem(atom, Es), Opts), case get_elem(field, Es) of [] -> see(E, [Name, "{}"]); Fs -> - see(E, Name) ++ ["{"] ++ seq(fun t_field/1, Fs, ["}"]) + see(E, Name) ++ ["{"] ++ seq(fun(F) -> t_field(F, Opts) end, + Fs, ["}"]) end. -t_field(#xmlElement{content = Es}) -> - t_type(get_elem(atom, Es)) ++ [" = "] ++ t_utype(get_elem(type, Es)). +t_field(#xmlElement{content = Es}, Opts) -> + (t_type(get_elem(atom, Es), Opts) ++ [" = "] ++ + t_utype(get_elem(type, Es), Opts)). -t_abstype(E, Es) -> - Name = t_name(get_elem(erlangName, Es)), +t_abstype(E, Es, Opts) -> + Name = t_name(get_elem(erlangName, Es), Opts), case get_elem(type, Es) of [] -> see(E, [Name, "()"]); Ts -> - see(E, [Name]) ++ ["("] ++ seq(fun t_utype_elem/1, Ts, [")"]) + see(E, [Name]) ++ ["("] ++ seq(t_utype_elem_fun(Opts), Ts, [")"]) end. -t_abstype(Es) -> - ([t_name(get_elem(erlangName, Es)), "("] - ++ seq(fun t_utype_elem/1, get_elem(type, Es), [")"])). +t_abstype(Es, Opts) -> + ([t_name(get_elem(erlangName, Es), Opts), "("] + ++ seq(t_utype_elem_fun(Opts), get_elem(type, Es), [")"])). -t_union(Es) -> - seq(fun t_utype_elem/1, Es, " | ", []). +t_union(Es, Opts) -> + seq(t_utype_elem_fun(Opts), Es, " | ", []). seq(F, Es) -> seq(F, Es, []). @@ -989,8 +1023,8 @@ local_label(R) -> xhtml(Title, CSS, Body, Encoding) -> EncString = case Encoding of - "latin1" -> "ISO-8859-1"; - _ -> "UTF-8" + latin1 -> "ISO-8859-1"; + utf8 -> "UTF-8" end, [{html, [?NL, {head, [?NL, @@ -1009,11 +1043,11 @@ xhtml(Title, CSS, Body, Encoding) -> %% --------------------------------------------------------------------- type(E) -> - type(E, []). + Opts = init_opts(E, []), + type(E, [], Opts). -type(E, Ds) -> - Opts = [], - xmerl:export_simple_content(t_utype_elem(E) ++ local_defs(Ds, Opts), +type(E, Ds, Opts) -> + xmerl:export_simple_content(t_utype_elem(E, Opts) ++ local_defs(Ds, Opts), ?HTML_EXPORT). overview(E=#xmlElement{name = overview, content = Es}, Options) -> @@ -1036,7 +1070,7 @@ overview(E=#xmlElement{name = overview, content = Es}, Options) -> ++ [?NL, hr] ++ navigation("bottom") ++ footer()), - Encoding = get_attrval(encoding, E), + Encoding = Opts#opts.encoding, XML = xhtml(Title, stylesheet(Opts), Body, Encoding), xmerl:export_simple(XML, ?HTML_EXPORT, []). @@ -1094,8 +1128,8 @@ ot_var(E) -> {var,0,list_to_atom(get_attrval(name, E))}. ot_atom(E) -> - {ok, [{atom,A,Name}], _} = erl_scan:string(get_attrval(value, E), 0), - {atom,erl_anno:line(A),Name}. + Name = list_to_atom(get_attrval(value, E)), + {atom,erl_anno:new(0),Name}. ot_integer(E) -> {integer,0,list_to_integer(get_attrval(value, E))}. diff --git a/lib/edoc/src/edoc_report.erl b/lib/edoc/src/edoc_report.erl index ed778c8112..76557ef483 100644 --- a/lib/edoc/src/edoc_report.erl +++ b/lib/edoc/src/edoc_report.erl @@ -94,7 +94,7 @@ where({File, footer}) -> where({File, header}) -> io_lib:fwrite("~ts, in header file: ", [File]); where({File, {F, A}}) -> - io_lib:fwrite("~ts, function ~s/~w: ", [File, F, A]); + io_lib:fwrite("~ts, function ~ts/~w: ", [File, F, A]); where([]) -> io_lib:fwrite("~s: ", [?APPLICATION]); where(File) when is_list(File) -> diff --git a/lib/edoc/src/edoc_scanner.erl b/lib/edoc/src/edoc_scanner.erl index f1d5e1d4b9..35d00c6c0e 100644 --- a/lib/edoc/src/edoc_scanner.erl +++ b/lib/edoc/src/edoc_scanner.erl @@ -86,6 +86,8 @@ scan1([C|Cs], Toks, Pos) when C >= 0, C =< $ -> % Skip blanks scan1(Cs, Toks, Pos); scan1([C|Cs], Toks, Pos) when C >= $a, C =< $z -> % Unquoted atom scan_atom(C, Cs, Toks, Pos); +scan1([C|Cs], Toks, Pos) when C >= $\337, C =< $\377, C /= $\367 -> + scan_atom(C, Cs, Toks, Pos); scan1([C|Cs], Toks, Pos) when C >= $0, C =< $9 -> % Numbers scan_number(C, Cs, Toks, Pos); scan1([$-,C| Cs], Toks, Pos) when C >= $0, C =< $9 -> % Signed numbers @@ -96,6 +98,8 @@ scan1([C|Cs], Toks, Pos) when C >= $A, C =< $Z -> % Variables scan_variable(C, Cs, Toks, Pos); scan1([$_|Cs], Toks, Pos) -> % Variables scan_variable($_, Cs, Toks, Pos); +scan1([C|Cs], Toks, Pos) when C >= $\300, C =< $\336, C /= $\327 -> + scan_variable(C, Cs, Toks, Pos); scan1([$$|Cs], Toks, Pos) -> % Character constant case scan_char_const(Cs, Toks, Pos) of {ok, Result} -> @@ -261,6 +265,15 @@ scan_char([], _Pos) -> %% The following conforms to Standard Erlang escape sequences. +-define(HEX(C), C >= $0 andalso C =< $9 orelse + C >= $A andalso C =< $F orelse + C >= $a andalso C =< $f). + +-define(UNICODE(C), + (C >= 0 andalso C < 16#D800 orelse + C > 16#DFFF andalso C < 16#FFFE orelse + C > 16#FFFF andalso C =< 16#10FFFF)). + scan_escape([O1, O2, O3 | Cs], Pos) when % \<1-3> octal digits O1 >= $0, O1 =< $3, O2 >= $0, O2 =< $7, O3 >= $0, O3 =< $7 -> Val = (O1*8 + O2)*8 + O3 - 73*$0, @@ -272,6 +285,11 @@ scan_escape([O1, O2 | Cs], Pos) when scan_escape([O1 | Cs], Pos) when O1 >= $0, O1 =< $7 -> {O1 - $0,Cs,Pos}; +scan_escape([$x, ${ | Cs], Pos) -> + scan_hex(Cs, Pos, []); +scan_escape([$x, H1, H2 | Cs], Pos) when ?HEX(H1), ?HEX(H2) -> + Val = (H1*16 + H2) - 17*$0, + {Val,Cs,Pos}; scan_escape([$^, C | Cs], Pos) -> % \^X -> CTL-X if C >= $\100, C =< $\137 -> {C - $\100,Cs,Pos}; @@ -285,6 +303,18 @@ scan_escape([C | Cs], Pos) -> scan_escape([], _Pos) -> {error, truncated_char}. +scan_hex([C | Cs], Pos, HCs) when ?HEX(C) -> + scan_hex(Cs, Pos, [C | HCs]); +scan_hex([$} | Cs], Pos, HCs) -> + case catch erlang:list_to_integer(lists:reverse(HCs), 16) of + Val when ?UNICODE(Val) -> + {Val,Cs,Pos}; + _ -> + {error, undefined_escape_sequence} + end; +scan_hex(_Cs, _Pos, _HCs) -> + {error, undefined_escape_sequence}. + %% Note that we return $\000 for undefined escapes. escape_char($b) -> $\010; % \b = BS escape_char($d) -> $\177; % \d = DEL diff --git a/lib/edoc/src/edoc_specs.erl b/lib/edoc/src/edoc_specs.erl index c15dfd328f..fb04bfce0e 100644 --- a/lib/edoc/src/edoc_specs.erl +++ b/lib/edoc/src/edoc_specs.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -274,12 +274,19 @@ parms([A | As], [D | Ds]) -> param(#t_paren{type = Type}, Default) -> param(Type, Default); -param(#t_record{name = #t_atom{val = Name}}, _Default) -> - list_to_atom(capitalize(atom_to_list(Name))); +param(#t_record{name = #t_atom{val = Name}}=T, Default) -> + AtomList = atom_to_list(Name), + case AtomList =:= lists:flatten(io_lib:write_atom(Name)) of + true -> + list_to_atom(capitalize(AtomList)); + false -> + arg_name(?t_ann(T), Default) + end; param(T, Default) -> arg_name(?t_ann(T), Default). capitalize([C | Cs]) when C >= $a, C =< $z -> [C - 32 | Cs]; +capitalize([C | Cs]) when C >= $\340, C =< $\376, C /= $\367 -> [C - 32 | Cs]; capitalize(Cs) -> Cs. %% Like edoc_types:arg_name/1 diff --git a/lib/edoc/test/edoc_SUITE.erl b/lib/edoc/test/edoc_SUITE.erl index 4d846ad63d..29ca9d1203 100644 --- a/lib/edoc/test/edoc_SUITE.erl +++ b/lib/edoc/test/edoc_SUITE.erl @@ -23,12 +23,13 @@ init_per_group/2,end_per_group/2]). %% Test cases --export([app/1,appup/1,build_std/1,build_map_module/1,otp_12008/1, build_app/1]). +-export([app/1,appup/1,build_std/1,build_map_module/1,otp_12008/1, + build_app/1, otp_14285/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [app,appup,build_std,build_map_module,otp_12008, build_app]. + [app,appup,build_std,build_map_module,otp_12008, build_app, otp_14285]. groups() -> []. @@ -113,3 +114,18 @@ build_app(Config) -> true = filelib:is_regular(filename:join(OutDir, "a.html")), true = filelib:is_regular(filename:join(OutDir, "b.html")), ok. + +otp_14285(Config) -> + DataDir = ?config(data_dir, Config), + PrivDir = ?config(priv_dir, Config), + Un1 = filename:join(DataDir, "un_atom1.erl"), + Un2 = filename:join(DataDir, "un_atom2.erl"), + %% epp_dodger + Opts1 = [{dir, PrivDir}], + ok = edoc:files([Un1], Opts1), + ok = edoc:files([Un2], Opts1), + %% epp + Opts2 = [{preprocess, true}, {dir, PrivDir}], + ok = edoc:files([Un1], Opts2), + ok = edoc:files([Un2], Opts2), + ok. diff --git a/lib/edoc/test/edoc_SUITE_data/un_atom1.erl b/lib/edoc/test/edoc_SUITE_data/un_atom1.erl new file mode 100644 index 0000000000..20ca50d5d2 --- /dev/null +++ b/lib/edoc/test/edoc_SUITE_data/un_atom1.erl @@ -0,0 +1,41 @@ +%% coding:latin-1 + +-module(un_atom1). + +-export(['\x{aaa}memory'/0, 'func-\x{400}'/1, func/1, �func/1]). + +-record('rec-\x{400}', {'field-\x{400}'}). + +-type cs() :: $\x{a} + | $\x{aa} + | $\x{aaa} + | $\xaa. + +-callback 'callback-\x{400}'() -> 'apa'. + +-type 'type-\x{400}'() :: 'atom-\x{400}' + | cs() + | #'rec-\x{400}'{'field-\x{400}' :: 'type-\x{400}'()}. + +-spec '\x{aaa}memory'() -> 'type-\x{400}'(). + +'\x{aaa}memory'() -> + apa:foo(). + +%% @deprecated Please use {@link m:f/1}. +-spec 'func-\x{400}'(#'rec-\x{400}'{}) -> #'rec-\x{400}'{}. + +'func-\x{400}'(_T) -> + foo:bar(#'rec-\x{400}'{}). + +-record(rec, {}). + +-spec func(#rec{}) -> #rec{}. + +func(#rec{}) -> #rec{}. + +-record(�rec, {}). + +-spec �func(#�rec{}) -> #�rec{}. + +�func(#�rec{}) -> #�rec{}. diff --git a/lib/edoc/test/edoc_SUITE_data/un_atom2.erl b/lib/edoc/test/edoc_SUITE_data/un_atom2.erl new file mode 100644 index 0000000000..66c83e30e0 --- /dev/null +++ b/lib/edoc/test/edoc_SUITE_data/un_atom2.erl @@ -0,0 +1,40 @@ +%% coding:utf-8 +-module(un_atom2). + +-export(['\x{aaa}memory'/0, 'func-\x{400}'/1, func/1, äfunc/1]). + +-record('rec-\x{400}', {'field-\x{400}'}). + +-type cs() :: $\x{a} + | $\x{aa} + | $\x{aaa} + | $\xaa. + +-callback 'callback-\x{400}'() -> 'apa'. + +-type 'type-\x{400}'() :: 'atom-\x{400}' + | cs() + | #'rec-\x{400}'{'field-\x{400}' :: 'type-\x{400}'()}. + +-spec '\x{aaa}memory'() -> 'type-\x{400}'(). + +'\x{aaa}memory'() -> + apa:foo(). + +%% @deprecated Please use {@link m:f/1}. +-spec 'func-\x{400}'(#'rec-\x{400}'{}) -> #'rec-\x{400}'{}. + +'func-\x{400}'(_T) -> + foo:bar(#'rec-\x{400}'{}). + +-record(rec, {}). + +-spec func(#rec{}) -> #rec{}. + +func(#rec{}) -> #rec{}. + +-record(ärec, {}). + +-spec äfunc(#ärec{}) -> #ärec{}. + +äfunc(#ärec{}) -> #ärec{}. diff --git a/lib/hipe/llvm/Makefile b/lib/hipe/llvm/Makefile index 88016a7d8b..e8d9a0e8bb 100644 --- a/lib/hipe/llvm/Makefile +++ b/lib/hipe/llvm/Makefile @@ -73,8 +73,7 @@ include ../native.mk ERL_COMPILE_FLAGS += -Werror +inline +warn_export_vars #+warn_missing_spec # if in 32 bit backend define BIT32 symbol -ARCH = $(shell echo $(TARGET) | sed 's/^\(x86_64\)-.*/64bit/') -ifneq ($(ARCH), 64bit) +ifneq ($(BITS64),yes) ERL_COMPILE_FLAGS += -DBIT32 endif diff --git a/lib/hipe/llvm/hipe_llvm_main.erl b/lib/hipe/llvm/hipe_llvm_main.erl index 0957dd4df2..4eec0c752b 100644 --- a/lib/hipe/llvm/hipe_llvm_main.erl +++ b/lib/hipe/llvm/hipe_llvm_main.erl @@ -108,8 +108,10 @@ llvm_llc(Dir, Filename, Ver, Options) -> OptLevel = trans_optlev_flag(llc, Options), VerFlags = llc_ver_flags(Ver), Align = find_stack_alignment(), + Target = llc_target_opt(), LlcFlags = [OptLevel, "-code-model=medium", "-stack-alignment=" ++ Align , "-tailcallopt", "-filetype=asm" %FIXME + , Target | VerFlags], Command = "llc " ++ fix_opts(LlcFlags) ++ " " ++ Source, %% io:format("LLC: ~s~n", [Command]), @@ -123,7 +125,8 @@ llvm_llc(Dir, Filename, Ver, Options) -> compile(Dir, Fun_Name, Compiler) -> Source = Dir ++ Fun_Name ++ ".s", Dest = Dir ++ Fun_Name ++ ".o", - Command = Compiler ++ " -c " ++ Source ++ " -o " ++ Dest, + Target = compiler_target_opt(), + Command = Compiler ++ " " ++ Target ++ " -c " ++ Source ++ " -o " ++ Dest, %% io:format("~s: ~s~n", [Compiler, Command]), case os:cmd(Command) of "" -> ok; @@ -137,6 +140,18 @@ find_stack_alignment() -> _ -> exit({?MODULE, find_stack_alignment, "Unimplemented architecture"}) end. +llc_target_opt() -> + case get(hipe_target_arch) of + x86 -> "-march=x86"; + amd64 -> "-march=x86-64" + end. + +compiler_target_opt() -> + case get(hipe_target_arch) of + x86 -> "-m32"; + amd64 -> "-m64" + end. + %% @doc Join options. fix_opts(Opts) -> string:join(Opts, " "). diff --git a/lib/inets/doc/src/notes.xml b/lib/inets/doc/src/notes.xml index 2ed02e021e..2aa48cd50a 100644 --- a/lib/inets/doc/src/notes.xml +++ b/lib/inets/doc/src/notes.xml @@ -33,7 +33,23 @@ <file>notes.xml</file> </header> - <section><title>Inets 6.3.6</title> + <section><title>Inets 6.3.7</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed a bug in ftp that made further operations after a + recv_chunk operation impossible.</p> + <p> + Own Id: OTP-14242</p> + </item> + </list> + </section> + +</section> + +<section><title>Inets 6.3.6</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/inets/src/ftp/ftp.erl b/lib/inets/src/ftp/ftp.erl index 23d6483291..de869e3204 100644 --- a/lib/inets/src/ftp/ftp.erl +++ b/lib/inets/src/ftp/ftp.erl @@ -108,7 +108,7 @@ -define(DBG(F,A), 'n/a'). %%-define(DBG(F,A), io:format(F,A)). -%%-define(DBG(F,A), if is_list(F) -> ct:pal(F,A); is_atom(F)->ct:pal(atom_to_list(F),A) end). +%%-define(DBG(F,A), ct:pal("~p:~p " ++ if is_list(F) -> F; is_atom(F) -> atom_to_list(F) end, [?MODULE,?LINE|A])). %%%========================================================================= %%% API - CLIENT FUNCTIONS @@ -1482,13 +1482,13 @@ handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, activate_ctrl_connection(State), {noreply, State#state{dsock = undefined, data = <<>>}}; -handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, client = From, +handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, caller = recv_chunk} = State) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> - gen_server:reply(From, ok), - {noreply, State#state{dsock = undefined, client = undefined, - data = <<>>, caller = undefined, - chunk = false}}; + activate_ctrl_connection(State), + {noreply, State#state{dsock = undefined, data = <<>>, + caller = recv_chunk_closed + }}; handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, caller = recv_bin, data = Data} = State) @@ -1601,13 +1601,13 @@ terminate(normal, State) -> %% If terminate reason =/= normal the progress reporting process will %% be killed by the exit signal. progress_report(stop, State), - do_termiante({error, econn}, State); + do_terminate({error, econn}, State); terminate(Reason, State) -> Report = io_lib:format("Ftp connection closed due to: ~p~n", [Reason]), error_logger:error_report(Report), - do_termiante({error, eclosed}, State). + do_terminate({error, eclosed}, State). -do_termiante(ErrorMsg, State) -> +do_terminate(ErrorMsg, State) -> close_data_connection(State), close_ctrl_connection(State), case State#state.client of @@ -2046,6 +2046,16 @@ handle_ctrl_result({pos_prel, _}, #state{client = From, end; %%-------------------------------------------------------------------------- +%% File handling - chunk_transfer complete +handle_ctrl_result({pos_compl, _}, #state{client = From, + caller = recv_chunk_closed} + = State0) -> + gen_server:reply(From, ok), + {noreply, State0#state{caller = undefined, + chunk = false, + client = undefined}}; + +%%-------------------------------------------------------------------------- %% File handling - recv_file handle_ctrl_result({pos_prel, _}, #state{caller = {recv_file, _}} = State0) -> case accept_data_connection(State0) of diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl index bd5f6df39e..418b6247b0 100644 --- a/lib/inets/src/http_client/httpc.erl +++ b/lib/inets/src/http_client/httpc.erl @@ -524,7 +524,7 @@ handle_request(Method, Url, Options = request_options(Options0), Sync = proplists:get_value(sync, Options), Stream = proplists:get_value(stream, Options), - Host2 = header_host(Scheme, Host, Port), + Host2 = http_request:normalize_host(Scheme, Host, Port), HeadersRecord = header_record(NewHeaders, Host2, HTTPOptions), Receiver = proplists:get_value(receiver, Options), SocketOpts = proplists:get_value(socket_opts, Options), @@ -1035,14 +1035,6 @@ bad_option(Option, BadValue) -> throw({error, {bad_option, Option, BadValue}}). -header_host(https, Host, 443 = _Port) -> - Host; -header_host(http, Host, 80 = _Port) -> - Host; -header_host(_Scheme, Host, Port) -> - Host ++ ":" ++ integer_to_list(Port). - - header_record(NewHeaders, Host, #http_options{version = Version}) -> header_record(NewHeaders, #http_request_h{}, Host, Version). diff --git a/lib/inets/src/http_client/httpc_response.erl b/lib/inets/src/http_client/httpc_response.erl index d24705a845..d81afde5fe 100644 --- a/lib/inets/src/http_client/httpc_response.erl +++ b/lib/inets/src/http_client/httpc_response.erl @@ -362,8 +362,9 @@ redirect(Response = {StatusLine, Headers, Body}, Request) -> {ok, error(Request, Reason), Data}; %% Automatic redirection {ok, {Scheme, _, Host, Port, Path, Query}} -> + HostPort = http_request:normalize_host(Scheme, Host, Port), NewHeaders = - (Request#request.headers)#http_request_h{host = Host++":"++integer_to_list(Port)}, + (Request#request.headers)#http_request_h{host = HostPort}, NewRequest = Request#request{redircount = Request#request.redircount+1, diff --git a/lib/inets/src/http_lib/http_request.erl b/lib/inets/src/http_lib/http_request.erl index c77b616f0d..4c50edb5ef 100644 --- a/lib/inets/src/http_lib/http_request.erl +++ b/lib/inets/src/http_lib/http_request.erl @@ -22,7 +22,7 @@ -include("http_internal.hrl"). --export([headers/2, http_headers/1, is_absolut_uri/1, key_value/1]). +-export([headers/2, http_headers/1, is_absolut_uri/1, key_value/1, normalize_host/3]). key_value(KeyValueStr) -> @@ -85,6 +85,22 @@ is_absolut_uri("https://" ++ _) -> is_absolut_uri(_) -> false. +%%------------------------------------------------------------------------- +%% normalize_host(Scheme, Host, Port) -> string() +%% Scheme - http | https +%% Host - string() +%% Port - integer() +%% +%% Description: returns a normalized Host header value, with the port +%% number omitted for well-known ports +%%------------------------------------------------------------------------- +normalize_host(https, Host, 443 = _Port) -> + Host; +normalize_host(http, Host, 80 = _Port) -> + Host; +normalize_host(_Scheme, Host, Port) -> + Host ++ ":" ++ integer_to_list(Port). + %%%======================================================================== %%% Internal functions %%%======================================================================== diff --git a/lib/inets/test/ftp_SUITE.erl b/lib/inets/test/ftp_SUITE.erl index e2dec0c42a..0053ba6edd 100644 --- a/lib/inets/test/ftp_SUITE.erl +++ b/lib/inets/test/ftp_SUITE.erl @@ -93,8 +93,10 @@ ftp_tests()-> append_chunk, recv, recv_3, - recv_bin, + recv_bin, + recv_bin_twice, recv_chunk, + recv_chunk_twice, type, quote, error_elogin, @@ -191,9 +193,22 @@ end_per_suite(Config) -> ok. %%-------------------------------------------------------------------- -init_per_group(_Group, Config) -> Config. - -end_per_group(_Group, Config) -> Config. +init_per_group(Group, Config) when Group == ftps_active, + Group == ftps_passive -> + catch crypto:stop(), + try crypto:start() of + ok -> + Config + catch + _:_ -> + {skip, "Crypto did not start"} + end; + +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, Config) -> + Config. %%-------------------------------------------------------------------- init_per_testcase(Case, Config0) -> @@ -533,15 +548,19 @@ append_chunk(Config0) -> recv() -> [{doc, "Receive a file using recv/2"}]. recv(Config0) -> - File = "f_dst.txt", + File1 = "f_dst1.txt", + File2 = "f_dst2.txt", SrcDir = "a_dir", - Contents = <<"ftp_SUITE test ...">>, - Config = set_state([reset, {mkfile,[SrcDir,File],Contents}], Config0), + Contents1 = <<"1 ftp_SUITE test ...">>, + Contents2 = <<"2 ftp_SUITE test ...">>, + Config = set_state([reset, {mkfile,[SrcDir,File1],Contents1}, {mkfile,[SrcDir,File2],Contents2}], Config0), Pid = proplists:get_value(ftp, Config), ok = ftp:cd(Pid, id2ftp(SrcDir,Config)), ok = ftp:lcd(Pid, id2ftp("",Config)), - ok = ftp:recv(Pid, File), - chk_file(File, Contents, Config), + ok = ftp:recv(Pid, File1), + chk_file(File1, Contents1, Config), + ok = ftp:recv(Pid, File2), + chk_file(File2, Contents2, Config), {error,epath} = ftp:recv(Pid, "non_existing_file"), ok. @@ -572,6 +591,25 @@ recv_bin(Config0) -> ok. %%------------------------------------------------------------------------- +recv_bin_twice() -> + [{doc, "Receive two files as a binaries."}]. +recv_bin_twice(Config0) -> + File1 = "f_dst1.txt", + File2 = "f_dst2.txt", + Contents1 = <<"1 ftp_SUITE test ...">>, + Contents2 = <<"2 ftp_SUITE test ...">>, + Config = set_state([reset, {mkfile,File1,Contents1}, {mkfile,File2,Contents2}], Config0), + ct:log("First transfer",[]), + Pid = proplists:get_value(ftp, Config), + {ok,Received1} = ftp:recv_bin(Pid, id2ftp(File1,Config)), + find_diff(Received1, Contents1), + ct:log("Second transfer",[]), + {ok,Received2} = ftp:recv_bin(Pid, id2ftp(File2,Config)), + find_diff(Received2, Contents2), + ct:log("Transfers ready!",[]), + {error,epath} = ftp:recv_bin(Pid, id2ftp("non_existing_file",Config)), + ok. +%%------------------------------------------------------------------------- recv_chunk() -> [{doc, "Receive a file using chunk-wise."}]. recv_chunk(Config0) -> @@ -584,6 +622,23 @@ recv_chunk(Config0) -> {ok, ReceivedContents, _Ncunks} = recv_chunk(Pid, <<>>), find_diff(ReceivedContents, Contents). +recv_chunk_twice() -> + [{doc, "Receive two files using chunk-wise."}]. +recv_chunk_twice(Config0) -> + File1 = "big_file1.txt", + File2 = "big_file2.txt", + Contents1 = list_to_binary( lists:duplicate(1000, lists:seq(0,255)) ), + Contents2 = crypto:strong_rand_bytes(1200), + Config = set_state([reset, {mkfile,File1,Contents1}, {mkfile,File2,Contents2}], Config0), + Pid = proplists:get_value(ftp, Config), + {{error, "ftp:recv_chunk_start/2 not called"},_} = recv_chunk(Pid, <<>>), + ok = ftp:recv_chunk_start(Pid, id2ftp(File1,Config)), + {ok, ReceivedContents1, _Ncunks1} = recv_chunk(Pid, <<>>), + ok = ftp:recv_chunk_start(Pid, id2ftp(File2,Config)), + {ok, ReceivedContents2, _Ncunks2} = recv_chunk(Pid, <<>>), + find_diff(ReceivedContents1, Contents1), + find_diff(ReceivedContents2, Contents2). + recv_chunk(Pid, Acc) -> recv_chunk(Pid, Acc, 0). diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index 8aea38037d..67aa78aa06 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -163,21 +163,17 @@ init_per_group(misc = Group, Config) -> ok = httpc:set_options([{ipfamily, Inet}]), Config; + init_per_group(Group, Config0) when Group =:= sim_https; Group =:= https-> - ct:timetrap({seconds, 30}), - start_apps(Group), - StartSsl = try ssl:start() + catch crypto:stop(), + try crypto:start() of + ok -> + ct:timetrap({seconds, 30}), + start_apps(Group), + do_init_per_group(Group, Config0) catch - Error:Reason -> - {skip, lists:flatten(io_lib:format("Failed to start apps for https Error=~p Reason=~p", [Error, Reason]))} - end, - case StartSsl of - {error, {already_started, _}} -> - do_init_per_group(Group, Config0); - ok -> - do_init_per_group(Group, Config0); - _ -> - StartSsl + _:_ -> + {skip, "Crypto did not start"} end; init_per_group(Group, Config0) -> diff --git a/lib/inets/test/httpd_SUITE.erl b/lib/inets/test/httpd_SUITE.erl index aae4ce5256..44b1e09cbc 100644 --- a/lib/inets/test/httpd_SUITE.erl +++ b/lib/inets/test/httpd_SUITE.erl @@ -197,7 +197,14 @@ init_per_group(Group, Config0) when Group == https_basic; Group == https_security; Group == https_reload -> - init_ssl(Group, Config0); + catch crypto:stop(), + try crypto:start() of + ok -> + init_ssl(Group, Config0) + catch + _:_ -> + {skip, "Crypto did not start"} + end; init_per_group(Group, Config0) when Group == http_basic; Group == http_limit; Group == http_custom; @@ -232,7 +239,14 @@ init_per_group(https_htaccess = Group, Config) -> Path = proplists:get_value(doc_root, Config), catch remove_htaccess(Path), create_htaccess_data(Path, proplists:get_value(address, Config)), - init_ssl(Group, Config); + catch crypto:stop(), + try crypto:start() of + ok -> + init_ssl(Group, Config) + catch + _:_ -> + {skip, "Crypto did not start"} + end; init_per_group(auth_api, Config) -> [{auth_prefix, ""} | Config]; init_per_group(auth_api_dets, Config) -> diff --git a/lib/inets/vsn.mk b/lib/inets/vsn.mk index 5637170c15..411bbfc043 100644 --- a/lib/inets/vsn.mk +++ b/lib/inets/vsn.mk @@ -19,6 +19,6 @@ # %CopyrightEnd% APPLICATION = inets -INETS_VSN = 6.3.6 +INETS_VSN = 6.3.7 PRE_VSN = APP_VSN = "$(APPLICATION)-$(INETS_VSN)$(PRE_VSN)" diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl index 19d36a7613..088d851b09 100644 --- a/lib/kernel/test/code_SUITE.erl +++ b/lib/kernel/test/code_SUITE.erl @@ -107,6 +107,10 @@ init_per_testcase(big_boot_embedded, Config) -> _Else -> {skip, "Needs crypto!"} end; +init_per_testcase(on_load_embedded, Config) -> + LibRoot = code:lib_dir(), + LinkName = filename:join(LibRoot, "on_load_app-1.0"), + [{link_name,LinkName}|Config]; init_per_testcase(_Func, Config) -> P = code:get_path(), [{code_path, P}|Config]. @@ -124,6 +128,10 @@ end_per_testcase(TC, Config) when TC == mult_lib_roots; NodeName = list_to_atom(atom_to_list(TC)++"@"++HostName), test_server:stop_node(NodeName), end_per_testcase(Config); +end_per_testcase(on_load_embedded, Config) -> + LinkName = proplists:get_value(link_name, Config), + _ = del_link(LinkName), + end_per_testcase(Config); end_per_testcase(_Func, Config) -> end_per_testcase(Config). @@ -1271,10 +1279,9 @@ on_load_embedded(Config) when is_list(Config) -> on_load_embedded_1(Config) -> DataDir = proplists:get_value(data_dir, Config), + LinkName = proplists:get_value(link_name, Config), %% Link the on_load_app application into the lib directory. - LibRoot = code:lib_dir(), - LinkName = filename:join(LibRoot, "on_load_app-1.0"), OnLoadApp = filename:join(DataDir, "on_load_app-1.0"), del_link(LinkName), io:format("LinkName :~p, OnLoadApp: ~p~n",[LinkName,OnLoadApp]), @@ -1308,8 +1315,7 @@ on_load_embedded_1(Config) -> ok = rpc:call(Node, on_load_embedded, status, []), %% Clean up. - stop_node(Node), - ok = del_link(LinkName). + stop_node(Node). del_link(LinkName) -> case file:delete(LinkName) of diff --git a/lib/observer/doc/src/etop.xml b/lib/observer/doc/src/etop.xml index d70d9d1d23..059f9f05d8 100644 --- a/lib/observer/doc/src/etop.xml +++ b/lib/observer/doc/src/etop.xml @@ -35,7 +35,7 @@ <file></file> </header> <module>etop</module> - <modulesummary>Erlang Top is a tool for presenting information about Erlang + <modulesummary>Erlang Top is a tool for presenting information about Erlang processes similar to the information presented by "top" in UNIX.</modulesummary> <description> @@ -60,11 +60,11 @@ <p>Value: <c>atom()</c></p> <p>Mandatory</p></item> <tag><c>setcookie</c></tag> - <item><p>Cookie to use for the <c>etop</c> node. Must be same as the + <item><p>Cookie to use for the <c>etop</c> node. Must be same as the cookie on the measured node.</p> <p>Value: <c>atom()</c></p></item> <tag><c>lines</c></tag> - <item><p>Number of lines (processes) to display.</p> + <item><p>Number of lines (processes) to display.</p> <p>Value: <c>integer()</c></p> <p>Default: <c>10</c></p></item> <tag><c>interval</c></tag> @@ -92,7 +92,7 @@ <p>Default: <c>on</c></p></item> </taglist> - <p>For detalis about Erlang Top, see the + <p>For details about Erlang Top, see the <seealso marker="etop_ug">User's Guide</seealso>.</p> </description> diff --git a/lib/observer/doc/src/observer.xml b/lib/observer/doc/src/observer.xml index 4d43ffe39f..fc6395d2c0 100644 --- a/lib/observer/doc/src/observer.xml +++ b/lib/observer/doc/src/observer.xml @@ -43,7 +43,7 @@ <seealso marker="ttb"><c>ttb</c></seealso>. </p> - <p>For detalis about how to get started, see the + <p>For details about how to get started, see the <seealso marker="observer_ug"><c>User's Guide</c></seealso>.</p> </description> <funcs> diff --git a/lib/parsetools/doc/src/leex.xml b/lib/parsetools/doc/src/leex.xml index 29d546105f..1227625287 100644 --- a/lib/parsetools/doc/src/leex.xml +++ b/lib/parsetools/doc/src/leex.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2009</year><year>2016</year> + <year>2009</year><year>2017</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -446,7 +446,8 @@ D = [0-9] </item> </taglist> - <p>The following examples define Erlang data types:</p> + <p>The following examples define simplified versions of a few + Erlang data types:</p> <code> Atoms [a-z][0-9a-zA-Z_]* diff --git a/lib/parsetools/src/leex.erl b/lib/parsetools/src/leex.erl index e0f37ae9df..e2e7d7359f 100644 --- a/lib/parsetools/src/leex.erl +++ b/lib/parsetools/src/leex.erl @@ -1548,22 +1548,23 @@ out_action_code(File, XrlFile, {_A,Code,_Vars,Name,Args,ArgsChars}) -> L = erl_scan:line(hd(Code)), output_file_directive(File, XrlFile, L-2), io:fwrite(File, "~s(~s) ->~n", [Name, ArgsChars]), - io:fwrite(File, " ~s\n", [pp_tokens(Code, L)]). + io:fwrite(File, " ~ts\n", [pp_tokens(Code, L, File)]). -%% pp_tokens(Tokens, Line) -> [char()]. +%% pp_tokens(Tokens, Line, File) -> [char()]. %% Prints the tokens keeping the line breaks of the original code. -pp_tokens(Tokens, Line0) -> pp_tokens(Tokens, Line0, none). +pp_tokens(Tokens, Line0, File) -> pp_tokens(Tokens, Line0, File, none). -pp_tokens([], _Line0, _) -> []; -pp_tokens([T | Ts], Line0, Prev) -> +pp_tokens([], _Line0, _, _) -> []; +pp_tokens([T | Ts], Line0, File, Prev) -> Line = erl_scan:line(T), - [pp_sep(Line, Line0, Prev, T), pp_symbol(T) | pp_tokens(Ts, Line, T)]. + [pp_sep(Line, Line0, Prev, T), + pp_symbol(T, File) | pp_tokens(Ts, Line, File, T)]. -pp_symbol({var,_,Var}) -> atom_to_list(Var); -pp_symbol({_,_,Symbol}) -> io_lib:fwrite("~p", [Symbol]); -pp_symbol({dot, _}) -> "."; -pp_symbol({Symbol, _}) -> atom_to_list(Symbol). +pp_symbol({var,_,Var}, _) -> atom_to_list(Var); +pp_symbol({_,_,Symbol}, File) -> format_symbol(Symbol, File); +pp_symbol({dot, _}, _) -> "."; +pp_symbol({Symbol, _}, _) -> atom_to_list(Symbol). pp_sep(Line, Line0, Prev, T) when Line > Line0 -> ["\n " | pp_sep(Line - 1, Line0, Prev, T)]; @@ -1622,17 +1623,17 @@ out_dfa_edges(File, DFA) -> end, orddict:new(), Pt), foreach(fun (T) -> Crs = orddict:fetch(T, Tdict), - Edgelab = dfa_edgelabel(Crs), + Edgelab = dfa_edgelabel(Crs, File), io:fwrite(File, " ~b -> ~b [label=\"~ts\"];~n", [S,T,Edgelab]) end, sort(orddict:fetch_keys(Tdict))) end, DFA). -dfa_edgelabel([C]) when is_integer(C) -> quote(C); -dfa_edgelabel(Cranges) -> +dfa_edgelabel([C], File) when is_integer(C) -> quote(C, File); +dfa_edgelabel(Cranges, File) -> %% io:fwrite("el: ~p\n", [Cranges]), - "[" ++ map(fun ({A,B}) -> [quote(A), "-", quote(B)]; - (C) -> [quote(C)] + "[" ++ map(fun ({A,B}) -> [quote(A, File), "-", quote(B, File)]; + (C) -> [quote(C, File)] end, Cranges) ++ "]". set_encoding(#leex{encoding = none}, File) -> @@ -1651,33 +1652,50 @@ output_file_directive(File, Filename, Line) -> format_filename(Filename0, File) -> Filename = filename:flatten(Filename0), + case enc(File) of + unicode -> io_lib:write_string(Filename); + latin1 -> io_lib:write_string_as_latin1(Filename) + end. + +format_symbol(Symbol, File) -> + Format = case enc(File) of + latin1 -> "~p"; + unicode -> "~tp" + end, + io_lib:fwrite(Format, [Symbol]). + +enc(File) -> case lists:keyfind(encoding, 1, io:getopts(File)) of - {encoding, unicode} -> io_lib:write_string(Filename); - _ -> io_lib:write_string_as_latin1(Filename) + false -> latin1; % should never happen + {encoding, Enc} -> Enc end. -quote($^) -> "\\^"; -quote($.) -> "\\."; -quote($$) -> "\\$"; -quote($-) -> "\\-"; -quote($[) -> "\\["; -quote($]) -> "\\]"; -quote($\s) -> "\\\\s"; -quote($\") -> "\\\""; -quote($\b) -> "\\\\b"; -quote($\f) -> "\\\\f"; -quote($\n) -> "\\\\n"; -quote($\r) -> "\\\\r"; -quote($\t) -> "\\\\t"; -quote($\e) -> "\\\\e"; -quote($\v) -> "\\\\v"; -quote($\d) -> "\\\\d"; -quote($\\) -> "\\\\"; -quote(C) when is_integer(C) -> +quote($^, _File) -> "\\^"; +quote($., _File) -> "\\."; +quote($$, _File) -> "\\$"; +quote($-, _File) -> "\\-"; +quote($[, _File) -> "\\["; +quote($], _File) -> "\\]"; +quote($\s, _File) -> "\\\\s"; +quote($\", _File) -> "\\\""; +quote($\b, _File) -> "\\\\b"; +quote($\f, _File) -> "\\\\f"; +quote($\n, _File) -> "\\\\n"; +quote($\r, _File) -> "\\\\r"; +quote($\t, _File) -> "\\\\t"; +quote($\e, _File) -> "\\\\e"; +quote($\v, _File) -> "\\\\v"; +quote($\d, _File) -> "\\\\d"; +quote($\\, _File) -> "\\\\"; +quote(C, File) when is_integer(C) -> %% Must remove the $ and get the \'s right. - case io_lib:write_char(C) of + S = case enc(File) of + unicode -> io_lib:write_char(C); + latin1 -> io_lib:write_char_as_latin1(C) + end, + case S of [$$,$\\|Cs] -> "\\\\" ++ Cs; [$$|Cs] -> Cs end; -quote(maxchar) -> +quote(maxchar, _File) -> "MAXCHAR". diff --git a/lib/parsetools/src/yecc.erl b/lib/parsetools/src/yecc.erl index 05446c1a85..48559ec402 100644 --- a/lib/parsetools/src/yecc.erl +++ b/lib/parsetools/src/yecc.erl @@ -154,13 +154,13 @@ compile(Input0, Output0, format_error(bad_declaration) -> io_lib:fwrite("unknown or bad declaration, ignored", []); format_error({bad_expect, SymName}) -> - io_lib:fwrite("argument ~s of Expect is not an integer", + io_lib:fwrite("argument ~ts of Expect is not an integer", [format_symbol(SymName)]); format_error({bad_rootsymbol, SymName}) -> - io_lib:fwrite("rootsymbol ~s is not a nonterminal", + io_lib:fwrite("rootsymbol ~ts is not a nonterminal", [format_symbol(SymName)]); format_error({bad_states, SymName}) -> - io_lib:fwrite("argument ~s of States is not an integer", + io_lib:fwrite("argument ~ts of States is not an integer", [format_symbol(SymName)]); format_error({conflict, Conflict}) -> format_conflict(Conflict); @@ -169,19 +169,19 @@ format_error({conflicts, SR, RR}) -> format_error({duplicate_declaration, Tag}) -> io_lib:fwrite("duplicate declaration of ~s", [atom_to_list(Tag)]); format_error({duplicate_nonterminal, Nonterminal}) -> - io_lib:fwrite("duplicate non-terminals ~s", + io_lib:fwrite("duplicate non-terminals ~ts", [format_symbol(Nonterminal)]); format_error({duplicate_precedence, Op}) -> - io_lib:fwrite("duplicate precedence operator ~s", + io_lib:fwrite("duplicate precedence operator ~ts", [format_symbol(Op)]); format_error({duplicate_terminal, Terminal}) -> - io_lib:fwrite("duplicate terminal ~s", + io_lib:fwrite("duplicate terminal ~ts", [format_symbol(Terminal)]); format_error({endsymbol_is_nonterminal, Symbol}) -> - io_lib:fwrite("endsymbol ~s is a nonterminal", + io_lib:fwrite("endsymbol ~ts is a nonterminal", [format_symbol(Symbol)]); format_error({endsymbol_is_terminal, Symbol}) -> - io_lib:fwrite("endsymbol ~s is a terminal", + io_lib:fwrite("endsymbol ~ts is a terminal", [format_symbol(Symbol)]); format_error({error, Module, Error}) -> Module:format_error(Error); @@ -192,7 +192,7 @@ format_error(illegal_empty) -> format_error({internal_error, Error}) -> io_lib:fwrite("internal yecc error: ~w", [Error]); format_error({missing_syntax_rule, Nonterminal}) -> - io_lib:fwrite("no syntax rule for non-terminal symbol ~s", + io_lib:fwrite("no syntax rule for non-terminal symbol ~ts", [format_symbol(Nonterminal)]); format_error({n_states, Exp, N}) -> io_lib:fwrite("expected ~w states, but got ~p states", [Exp, N]); @@ -201,31 +201,31 @@ format_error(no_grammar_rules) -> format_error(nonterminals_missing) -> io_lib:fwrite("Nonterminals is missing", []); format_error({precedence_op_is_endsymbol, SymName}) -> - io_lib:fwrite("precedence operator ~s is endsymbol", + io_lib:fwrite("precedence operator ~ts is endsymbol", [format_symbol(SymName)]); format_error({precedence_op_is_unknown, SymName}) -> - io_lib:fwrite("unknown precedence operator ~s", + io_lib:fwrite("unknown precedence operator ~ts", [format_symbol(SymName)]); format_error({reserved, N}) -> io_lib:fwrite("the use of ~w should be avoided", [N]); format_error({symbol_terminal_and_nonterminal, SymName}) -> - io_lib:fwrite("symbol ~s is both a terminal and nonterminal", + io_lib:fwrite("symbol ~ts is both a terminal and nonterminal", [format_symbol(SymName)]); format_error(rootsymbol_missing) -> io_lib:fwrite("Rootsymbol is missing", []); format_error(terminals_missing) -> io_lib:fwrite("Terminals is missing", []); format_error({undefined_nonterminal, Symbol}) -> - io_lib:fwrite("undefined nonterminal: ~s", [format_symbol(Symbol)]); + io_lib:fwrite("undefined nonterminal: ~ts", [format_symbol(Symbol)]); format_error({undefined_pseudo_variable, Atom}) -> io_lib:fwrite("undefined pseudo variable ~w", [Atom]); format_error({undefined_symbol, SymName}) -> - io_lib:fwrite("undefined rhs symbol ~s", [format_symbol(SymName)]); + io_lib:fwrite("undefined rhs symbol ~ts", [format_symbol(SymName)]); format_error({unused_nonterminal, Nonterminal}) -> - io_lib:fwrite("non-terminal symbol ~s not used", + io_lib:fwrite("non-terminal symbol ~ts not used", [format_symbol(Nonterminal)]); format_error({unused_terminal, Terminal}) -> - io_lib:fwrite("terminal symbol ~s not used", + io_lib:fwrite("terminal symbol ~ts not used", [format_symbol(Terminal)]); format_error({bad_symbol, String}) -> io_lib:fwrite("bad symbol ~ts", [String]); @@ -1809,9 +1809,9 @@ report_conflict(Conflict, St, ActionName, How) -> Formated = format_symbol(ActionName), case How of prec -> - io:fwrite(<<"Resolved in favor of ~s.\n\n">>, [Formated]); + io:fwrite(<<"Resolved in favor of ~ts.\n\n">>, [Formated]); default -> - io:fwrite(<<"Conflict resolved in favor of ~s.\n\n">>, + io:fwrite(<<"Conflict resolved in favor of ~ts.\n\n">>, [Formated]) end; true -> @@ -1856,7 +1856,7 @@ format_conflict({Symbol, N, _, {one_level_up, {L1, RuleN1, {P1, Ass1}}, {L2, RuleN2, {P2, Ass2}}}}) -> S1 = io_lib:fwrite(<<"Conflicting precedences of symbols when " - "scanning ~s in state ~w:\n">>, + "scanning ~ts in state ~w:\n">>, [format_symbol(Symbol), N]), S2 = io_lib:fwrite(<<" ~s ~w (rule ~w at line ~w)\n" " vs.\n">>, @@ -1866,26 +1866,26 @@ format_conflict({Symbol, N, _, {one_level_up, [S1, S2, S3]; format_conflict({Symbol, N, Reduce, Confl}) -> S1 = io_lib:fwrite(<<"Parse action conflict scanning symbol " - "~s in state ~w:\n">>, [format_symbol(Symbol), N]), + "~ts in state ~w:\n">>, [format_symbol(Symbol), N]), S2 = case Reduce of {[HR | TR], RuleNmbr, RuleLine} -> - io_lib:fwrite(<<" Reduce to ~s from ~s (rule ~w at " + io_lib:fwrite(<<" Reduce to ~ts from ~ts (rule ~w at " "line ~w)\n vs.\n">>, [format_symbol(HR), format_symbols(TR), RuleNmbr, RuleLine]) end, S3 = case Confl of {reduce, [HR2|TR2], RuleNmbr2, RuleLine2} -> - io_lib:fwrite(<<" reduce to ~s from ~s " + io_lib:fwrite(<<" reduce to ~ts from ~ts " "(rule ~w at line ~w).">>, [format_symbol(HR2), format_symbols(TR2), RuleNmbr2, RuleLine2]); {shift, NewState, Sym} -> io_lib:fwrite(<<" shift to state ~w, adding right " - "sisters to ~s.">>, + "sisters to ~ts.">>, [NewState, format_symbol(Sym)]); {accept, Rootsymbol} -> - io_lib:fwrite(<<" reduce to rootsymbol ~s.">>, + io_lib:fwrite(<<" reduce to rootsymbol ~ts.">>, [format_symbol(Rootsymbol)]) end, [S1, S2, S3]. @@ -1926,8 +1926,9 @@ format_conflict({Symbol, N, Reduce, Confl}) -> -define(CODE_VERSION, "1.4"). -define(YECC_BUG(M, A), - iolist_to_binary([" erlang:error({yecc_bug,\"",?CODE_VERSION,"\",", - io_lib:fwrite(M, A), "}).\n\n"])). + unicode:characters_to_binary( + [" erlang:error({yecc_bug,\"",?CODE_VERSION,"\",", + io_lib:fwrite(M, A), "}).\n\n"])). %% Returns number of newlines in included files. output_prelude(Outport, Inport, St0) when St0#yecc.includefile =:= [] -> @@ -1980,7 +1981,7 @@ output_header(St0) -> output_goto(St, [{_Nonterminal, []} | Go], StateInfo) -> output_goto(St, Go, StateInfo); output_goto(St0, [{Nonterminal, List} | Go], StateInfo) -> - F = function_name(yeccgoto, Nonterminal), + F = function_name(St0, yeccgoto, Nonterminal), St05 = fwrite(St0, <<"-dialyzer({nowarn_function, ~w/7}).\n">>, [F]), St10 = output_goto1(St05, List, F, StateInfo, true), St = output_goto_fini(F, Nonterminal, St10), @@ -2018,7 +2019,8 @@ output_goto_fini(F, NT, #yecc{includefile_version = {1,1}}=St0) -> St = fwrite(St10, <<"~w(State, _Cat, _Ss, _Stack, _T, _Ts, _Tzr) ->\n">>, [F]), fwrite(St, - ?YECC_BUG(<<"{~w, State, missing_in_goto_table}">>, [NT]), + ?YECC_BUG(<<"{~ts, State, missing_in_goto_table}">>, + [quoted_atom(St0, NT)]), []); output_goto_fini(_F, _NT, St) -> fwrite(St, <<".\n\n">>, []). @@ -2027,7 +2029,7 @@ output_goto_fini(_F, _NT, St) -> find_user_code(ParseActions, St) -> [#user_code{state = State, terminal = Terminal, - funname = inlined_function_name(State, Terminal), + funname = inlined_function_name(St, State, Terminal), action = Action} || {State, La_actions} <- ParseActions, {Action, Terminals, RuleNmbr, NmbrOfDaughters} @@ -2148,14 +2150,14 @@ output_action(St, State, Terminal, #reduce{}=Action, IsFirst, SI) -> output_reduce(St, State, Terminal, Action, IsFirst, SI); output_action(St0, State, Terminal, #shift{state = NewState}, IsFirst, _SI) -> St10 = delim(St0, IsFirst), - St = fwrite(St10, <<"yeccpars2_~w(S, ~s, Ss, Stack, T, Ts, Tzr) ->\n">>, - [State, quoted_atom(Terminal)]), + St = fwrite(St10, <<"yeccpars2_~w(S, ~ts, Ss, Stack, T, Ts, Tzr) ->\n">>, + [State, quoted_atom(St10, Terminal)]), output_call_to_includefile(NewState, St); output_action(St0, State, Terminal, accept, IsFirst, _SI) -> St10 = delim(St0, IsFirst), St = fwrite(St10, - <<"yeccpars2_~w(_S, ~s, _Ss, Stack, _T, _Ts, _Tzr) ->\n">>, - [State, quoted_atom(Terminal)]), + <<"yeccpars2_~w(_S, ~ts, _Ss, Stack, _T, _Ts, _Tzr) ->\n">>, + [State, quoted_atom(St10, Terminal)]), fwrite(St, <<" {ok, hd(Stack)}">>, []); output_action(St, _State, _Terminal, nonassoc, _IsFirst, _SI) -> St. @@ -2174,19 +2176,19 @@ output_state_actions_fini(State, IsFirst, St0) -> St = fwrite(St10, <<"yeccpars2_~w(_, _, _, _, T, _, _) ->\n">>, [State]), fwrite(St, <<" yeccerror(T).\n\n">>, []). -output_reduce(St0, State, Terminal0, +output_reduce(St0, State, Terminal, #reduce{rule_nmbr = RuleNmbr, head = Head, nmbr_of_daughters = NmbrOfDaughters}, IsFirst, StateInfo) -> St10 = delim(St0, IsFirst), - Terminal = if - is_atom(Terminal0) -> quoted_atom(Terminal0); - true -> Terminal0 - end, + QuotedTerminal = if + is_atom(Terminal) -> quoted_atom(St10, Terminal); + true -> Terminal + end, St20 = fwrite(St10, - <<"yeccpars2_~w(_S, ~s, Ss, Stack, T, Ts, Tzr) ->\n">>, - [State, Terminal]), + <<"yeccpars2_~w(_S, ~ts, Ss, Stack, T, Ts, Tzr) ->\n">>, + [State, QuotedTerminal]), St30 = if NmbrOfDaughters < 2 -> @@ -2205,7 +2207,7 @@ output_reduce(St0, State, Terminal0, _ -> NewStack = "NewStack", fwrite(St30, <<" NewStack = ~w(Stack),\n">>, - [inlined_function_name(State, Terminal0)]) + [inlined_function_name(St30, State, Terminal)]) end, if NmbrOfDaughters =:= 0 -> @@ -2221,13 +2223,13 @@ output_reduce(St0, State, Terminal0, St = fwrite(St40, <<"~s">>, [C]), %% Short-circuit call to yeccpars2: fwrite(St, - <<" yeccpars2_~w(~s, ~s, [~w | Ss], ~s, T, Ts, Tzr)">>, - [Repr, NextS, Terminal, State, NewStack]); + <<" yeccpars2_~w(~s, ~ts, [~w | Ss], ~s, T, Ts, Tzr)">>, + [Repr, NextS, QuotedTerminal, State, NewStack]); true -> fwrite(St40, - <<" ~w(hd(~s), ~s, ~s, ~s, T, Ts, Tzr)">>, - [function_name(yeccgoto, Head), Ns, - Terminal, Ns, NewStack]) + <<" ~w(hd(~s), ~ts, ~s, ~s, T, Ts, Tzr)">>, + [function_name(St40, yeccgoto, Head), Ns, + QuotedTerminal, Ns, NewStack]) end. delim(St, true) -> @@ -2235,8 +2237,10 @@ delim(St, true) -> delim(St, false) -> fwrite(St, <<";\n">>, []). -quoted_atom(Atom) -> - io_lib:fwrite(<<"~w">>, [Atom]). +quoted_atom(#yecc{encoding = latin1}, Atom) when is_atom(Atom) -> + io_lib:write_atom_as_latin1(Atom); +quoted_atom(_St, Atomic) -> + io_lib:write(Atomic). output_inlined(St, UserCodeActions, Infile) -> foldl(fun(#user_code{funname = InlinedFunctionName, @@ -2288,14 +2292,16 @@ output_inlined(St0, FunctionName, Reduce, Infile) -> fwrite(St, <<" [begin\n ~ts\n end | ~s].\n\n">>, [pp_tokens(Tokens, Line0, St#yecc.encoding), Stack]). -inlined_function_name(State, "Cat") -> - inlined_function_name(State, ""); -inlined_function_name(State, Terminal) -> - list_to_atom(concat([yeccpars2_, State, '_', Terminal])). +inlined_function_name(St, State, Terminal) -> + End = case Terminal of + "Cat" -> []; + _ -> [quoted_atom(St, Terminal)] + end, + list_to_atom(concat([yeccpars2_, State, '_'] ++ End)). --compile({nowarn_unused_function,function_name/2}). -function_name(Name, Suf) -> - list_to_atom(concat([Name, '_' | quoted_atom(Suf)])). +-compile({nowarn_unused_function,function_name/3}). +function_name(St, Name, Suf) -> + list_to_atom(concat([Name, '_'] ++ [quoted_atom(St, Suf)])). rule(RulePointer, St) -> #rule{n = N, anno = Anno, symbols = Symbols} = diff --git a/lib/parsetools/test/leex_SUITE.erl b/lib/parsetools/test/leex_SUITE.erl index 54602848ec..3f5d9fee3e 100644 --- a/lib/parsetools/test/leex_SUITE.erl +++ b/lib/parsetools/test/leex_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -45,7 +45,7 @@ pt/1, man/1, ex/1, ex2/1, not_yet/1, line_wrap/1, - otp_10302/1, otp_11286/1, unicode/1, otp_13916/1]). + otp_10302/1, otp_11286/1, unicode/1, otp_13916/1, otp_14285/1]). % Default timetrap timeout (set in init_per_testcase). -define(default_timeout, ?t:minutes(1)). @@ -67,7 +67,7 @@ all() -> groups() -> [{checks, [], [file, compile, syntax]}, {examples, [], [pt, man, ex, ex2, not_yet, unicode]}, - {tickets, [], [otp_10302, otp_11286, otp_13916]}, + {tickets, [], [otp_10302, otp_11286, otp_13916, otp_14285]}, {bugs, [], [line_wrap]}]. init_per_suite(Config) -> @@ -1131,6 +1131,45 @@ otp_13916(Config) when is_list(Config) -> ?line run(Config, Ts), ok. +otp_14285(Config) -> + Dir = ?privdir, + Filename = filename:join(Dir, "file.xrl"), + + Ts = [{otp_14285_1, + <<"%% encoding: latin-1\n" + "Definitions.\n" + "A = a\n" + "Z = z\n" + "L = [{A}-{Z}]\n" + "U = [\\x{400}]\n" + "Rules.\n" + "{L}+ : {token,l}.\n" + "{U}+ : {token,'\\x{400}'}.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + " {ok,['\\x{400}'],1} = string(\"\\x{400}\"), ok.\n">>, + default, + ok}, + {otp_14285_2, + <<"%% encoding: UTF-8\n" + "Definitions.\n" + "A = a\n" + "Z = z\n" + "L = [{A}-{Z}]\n" + "U = [\x{400}]\n" + "Rules.\n" + "{L}+ : {token,l}.\n" + "{U}+ : {token,'\x{400}'}.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + " {ok,['\x{400}'],1} = string(\"\x{400}\"), ok.\n">>, + default, + ok}], + run(Config, Ts), + ok. + start_node(Name, Args) -> [_,Host] = string:tokens(atom_to_list(node()), "@"), ct:log("Trying to start ~w@~s~n", [Name,Host]), diff --git a/lib/parsetools/test/yecc_SUITE.erl b/lib/parsetools/test/yecc_SUITE.erl index 2c37278d4b..a7166b91ed 100644 --- a/lib/parsetools/test/yecc_SUITE.erl +++ b/lib/parsetools/test/yecc_SUITE.erl @@ -50,7 +50,7 @@ otp_5369/1, otp_6362/1, otp_7945/1, otp_8483/1, otp_8486/1, otp_7292/1, otp_7969/1, otp_8919/1, otp_10302/1, otp_11269/1, - otp_11286/1]). + otp_11286/1, otp_14285/1]). % Default timetrap timeout (set in init_per_testcase). -define(default_timeout, ?t:minutes(1)). @@ -78,7 +78,7 @@ groups() -> {bugs, [], [otp_5369, otp_6362, otp_7945, otp_8483, otp_8486]}, {improvements, [], [otp_7292, otp_7969, otp_8919, otp_10302, - otp_11269, otp_11286]}]. + otp_11269, otp_11286, otp_14285]}]. init_per_suite(Config) -> Config. @@ -2048,6 +2048,90 @@ otp_11286(Config) when is_list(Config) -> true = test_server:stop_node(Node), ok. +otp_14285(Config) -> + Dir = ?privdir, + YeccPre = filename:join(Dir, "yeccpre.hrl"), + ?line ok = file:write_file(YeccPre, + [<<"-export([t/0]).\n">>,my_yeccpre()]), + + T0 = <<" + Nonterminals '\\x{400}'. + Terminals t. + Rootsymbol '\\x{400}'. + '\\x{400}' -> t : '$1'. + Erlang code. + t() -> + L = [{t, 1}], + {ok, R} = parse(L), + {t, 1} = R, + ok.">>, + Ts0 = [{otp_14285_1, + [<<"%% coding: Latin-1\n">>,T0],YeccPre,ok}, + {otp_14285_2, + [<<"%% coding: coding: UTF-8\n">>,T0],YeccPre,ok}], + run(Config, Ts0), + file:delete(YeccPre), + + T1 = <<" + Nonterminals '1\\x{400}' list 'unused\\x{400}'. + Terminals '2\\x{400}'. + Rootsymbol '1\\x{400}'. + + '1\\x{400}' -> list : '$1'. + + list -> '2\\x{400}' : '$1'. + list -> list '2\\x{400}' : {foo,'\\x{400}'}. + + Erlang code. + + -export([t/0]). + + t() -> + L = [{'2\\x{400}', 1}, {'2\\x{400}',2}], + {ok, R} = parse(L), + {foo,A} = R, + '\\x{400}' = A, + [1024] = atom_to_list(A), + ok.">>, + + Ts1 = [{otp_14285_3, + [<<"%% coding: Latin-1\n">>,T1],default,ok}, + {otp_14285_4, + [<<"%% coding: UTF-8\n">>,T1],default,ok}], + run(Config, Ts1), + + T2 = <<" + Nonterminals E. + Terminals '-' '+' '=' id. + Rootsymbol E. + Endsymbol '\\x{400}'. + + E -> E '=' E : {op, '=', '$1', '$3'}. + E -> E '+' E : {op, '+', '$1', '$3'}. + E -> '-' E : {op, '-', '$2'}. + E -> id : '$1'. + + Nonassoc 100 '='. + Right 200 '+' '-'. + + Erlang code. + + -export([t/0]). + + t() -> + {ok,{op,'=',{id,1},{op,'-',{op,'+',{id,4},{id,6}}}}} = + parse([{id,1},{'=',2},{'-',3},{id,4},{'+',5},{id,6}, + {'\\x{400}',1}]), + ok.">>, + + Ts2 = [{otp_14285_5, + [<<"%% coding: Latin-1\n">>,T2],default,ok}, + {otp_14285_6, + [<<"%% coding: UTF-8\n">>,T2],default,ok}], + run(Config, Ts2), + + ok. + start_node(Name, Args) -> [_,Host] = string:tokens(atom_to_list(node()), "@"), ct:log("Trying to start ~w@~s~n", [Name,Host]), diff --git a/lib/ssh/doc/src/notes.xml b/lib/ssh/doc/src/notes.xml index 02a39f030c..c8c6e61cc8 100644 --- a/lib/ssh/doc/src/notes.xml +++ b/lib/ssh/doc/src/notes.xml @@ -30,6 +30,22 @@ <file>notes.xml</file> </header> +<section><title>Ssh 4.4.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + ssh:daemon_info/1 crashed if the listening IP was not + 'any'</p> + <p> + Own Id: OTP-14298 Aux Id: seq13294 </p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.4.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssh/doc/src/ssh.xml b/lib/ssh/doc/src/ssh.xml index 968983c862..368261968d 100644 --- a/lib/ssh/doc/src/ssh.xml +++ b/lib/ssh/doc/src/ssh.xml @@ -379,7 +379,7 @@ on the given port.</fsummary> <type> <v>Port = integer()</v> - <v>HostAddress = ip_address() | any</v> + <v>HostAddress = ip_address() | any | loopback</v> <v>Options = [{Option, Value}]</v> <v>Option = atom()</v> <v>Value = term()</v> @@ -390,6 +390,26 @@ <p>Starts a server listening for SSH connections on the given port. If the <c>Port</c> is 0, a random free port is selected. See <seealso marker="#daemon_info/1">daemon_info/1</seealso> about how to find the selected port number.</p> + + <p>Please note that by historical reasons both the <c>HostAddress</c> argument and the inet socket option + <c>ip</c> set the listening address. This is a source of possible inconsistent settings.</p> + + <p>The rules for handling the two address passing options are:</p> + <list> + <item>if <c>HostAddress</c> is an IP-address, that IP-address is the listening address. + An 'ip'-option will be discarded if present.</item> + + <item>if <c>HostAddress</c> is <c>loopback</c>, the listening address + is <c>loopback</c> and an loopback address will be choosen by the underlying layers. + An 'ip'-option will be discarded if present.</item> + + <item>if <c>HostAddress</c> is <c>any</c> and no 'ip'-option is present, the listening address is + <c>any</c> and the socket will listen to all addresses</item> + + <item>if <c>HostAddress</c> is <c>any</c> and an 'ip'-option is present, the listening address is + set to the value of the 'ip'-option</item> + </list> + <p>Options:</p> <taglist> <tag><c><![CDATA[{inet, inet | inet6}]]></c></tag> @@ -699,6 +719,12 @@ <p><c>Peer</c> is in the format of <c>{Host,Port}</c>.</p> </item> + <tag><c><![CDATA[{idle_time, integer()}]]></c></tag> + <item> + <p>Sets a time-out on a connection when no channels are active. + Defaults to <c>infinity</c>.</p> + </item> + <tag><c><![CDATA[{ssh_msg_debug_fun, fun(ConnectionRef::ssh_connection_ref(), AlwaysDisplay::boolean(), Msg::binary(), LanguageTag::binary()) -> _}]]></c></tag> <item> <p>Provide a fun to implement your own logging of the SSH message SSH_MSG_DEBUG. The last three parameters are from the message, see RFC4253, section 11.3. The <c>ConnectionRef</c> is the reference to the connection on which the message arrived. The return value from the fun is not checked.</p> diff --git a/lib/ssh/doc/src/ssh_app.xml b/lib/ssh/doc/src/ssh_app.xml index 5f710decc1..515b0639d5 100644 --- a/lib/ssh/doc/src/ssh_app.xml +++ b/lib/ssh/doc/src/ssh_app.xml @@ -109,7 +109,7 @@ </section> <section> <title>Host Keys</title> - <p>RSA and DSA host keys are supported and are + <p>RSA, DSA and ECDSA host keys are supported and are expected to be found in files named <c>ssh_host_rsa_key</c>, <c>ssh_host_dsa_key</c> and <c>ssh_host_ecdsa_key</c>. </p> @@ -160,7 +160,7 @@ <item>ecdsa-sha2-nistp384</item> <item>ecdsa-sha2-nistp521</item> <item>ssh-rsa</item> - <item>(ssh-dss, retired: can be enabled with the <c>preferred_algorithms</c> option)</item> + <item>ssh-dss</item> </list> </item> diff --git a/lib/ssh/src/ssh.erl b/lib/ssh/src/ssh.erl index e2a289d737..3e80a04b70 100644 --- a/lib/ssh/src/ssh.erl +++ b/lib/ssh/src/ssh.erl @@ -26,6 +26,7 @@ -include("ssh_connect.hrl"). -include_lib("public_key/include/public_key.hrl"). -include_lib("kernel/include/file.hrl"). +-include_lib("kernel/include/inet.hrl"). -export([start/0, start/1, stop/0, connect/2, connect/3, connect/4, @@ -108,7 +109,7 @@ connect(Socket, UserOptions, Timeout) when is_port(Socket), case valid_socket_to_use(Socket, ?GET_OPT(transport,Options)) of ok -> {ok, {Host,_Port}} = inet:sockname(Socket), - Opts = ?PUT_INTERNAL_OPT([{user_pid,self()}, {host,fmt_host(Host)}], Options), + Opts = ?PUT_INTERNAL_OPT([{user_pid,self()}, {host,Host}], Options), ssh_connection_handler:start_connection(client, Socket, Opts, Timeout); {error,SockError} -> {error,SockError} @@ -120,7 +121,7 @@ connect(Host, Port, UserOptions) when is_integer(Port), is_list(UserOptions) -> connect(Host, Port, UserOptions, infinity). -connect(Host, Port, UserOptions, Timeout) when is_integer(Port), +connect(Host0, Port, UserOptions, Timeout) when is_integer(Port), Port>0, is_list(UserOptions) -> case ssh_options:handle_options(client, UserOptions) of @@ -130,6 +131,7 @@ connect(Host, Port, UserOptions, Timeout) when is_integer(Port), {_, Transport, _} = TransportOpts = ?GET_OPT(transport, Options), ConnectionTimeout = ?GET_OPT(connect_timeout, Options), SocketOpts = [{active,false} | ?GET_OPT(socket_options,Options)], + Host = mangle_connect_address(Host0, SocketOpts), try Transport:connect(Host, Port, SocketOpts, ConnectionTimeout) of {ok, Socket} -> Opts = ?PUT_INTERNAL_OPT([{user_pid,self()}, {host,Host}], Options), @@ -138,7 +140,6 @@ connect(Host, Port, UserOptions, Timeout) when is_integer(Port), {error, Reason} catch exit:{function_clause, _F} -> - io:format('function_clause ~p~n',[_F]), {error, {options, {transport, TransportOpts}}}; exit:badarg -> {error, {options, {socket_options, SocketOpts}}} @@ -183,16 +184,88 @@ daemon(Port) -> daemon(Port, []). -daemon(Port, UserOptions) when is_integer(Port), Port >= 0 -> - daemon(any, Port, UserOptions); - daemon(Socket, UserOptions) when is_port(Socket) -> - daemon(socket, Socket, UserOptions). + try + #{} = Options = ssh_options:handle_options(server, UserOptions), + case valid_socket_to_use(Socket, ?GET_OPT(transport,Options)) of + ok -> + {ok, {IP,Port}} = inet:sockname(Socket), + finalize_start(IP, Port, ?GET_OPT(profile, Options), + ?PUT_INTERNAL_OPT({connected_socket, Socket}, Options), + fun(Opts, DefaultResult) -> + try ssh_acceptor:handle_established_connection( + IP, Port, Opts, Socket) + of + {error,Error} -> + {error,Error}; + _ -> + DefaultResult + catch + C:R -> + {error,{could_not_start_connection,{C,R}}} + end + end); + {error,SockError} -> + {error,SockError} + end + catch + throw:bad_fd -> + {error,bad_fd}; + throw:bad_socket -> + {error,bad_socket}; + error:{badmatch,{error,Error}} -> + {error,Error}; + error:Error -> + {error,Error}; + _C:_E -> + {error,{cannot_start_daemon,_C,_E}} + end; + +daemon(Port, UserOptions) when 0 =< Port, Port =< 65535 -> + daemon(any, Port, UserOptions). + + +daemon(Host0, Port0, UserOptions0) when 0 =< Port0, Port0 =< 65535, + Host0 == any ; Host0 == loopback ; is_tuple(Host0) -> + try + {Host1, UserOptions} = handle_daemon_args(Host0, UserOptions0), + #{} = Options0 = ssh_options:handle_options(server, UserOptions), + + {{Host,Port}, ListenSocket} = + open_listen_socket(Host1, Port0, Options0), + + %% Now Host,Port is what to use for the supervisor to register its name, + %% and ListenSocket is for listening on connections. But it is still owned + %% by self()... + + finalize_start(Host, Port, ?GET_OPT(profile, Options0), + ?PUT_INTERNAL_OPT({lsocket,{ListenSocket,self()}}, Options0), + fun(Opts, Result) -> + {_, Callback, _} = ?GET_OPT(transport, Opts), + receive + {request_control, ListenSocket, ReqPid} -> + ok = Callback:controlling_process(ListenSocket, ReqPid), + ReqPid ! {its_yours,ListenSocket}, + Result + end + end) + catch + throw:bad_fd -> + {error,bad_fd}; + throw:bad_socket -> + {error,bad_socket}; + error:{badmatch,{error,Error}} -> + {error,Error}; + error:Error -> + {error,Error}; + _C:_E -> + {error,{cannot_start_daemon,_C,_E}} + end; + +daemon(_, _, _) -> + {error, badarg}. -daemon(Host0, Port, UserOptions0) -> - {Host, UserOptions} = handle_daemon_args(Host0, UserOptions0), - start_daemon(Host, Port, ssh_options:handle_options(server, UserOptions)). %%-------------------------------------------------------------------- -spec daemon_info(daemon_ref()) -> ok_error( [{atom(), term()}] ). @@ -200,11 +273,17 @@ daemon(Host0, Port, UserOptions0) -> daemon_info(Pid) -> case catch ssh_system_sup:acceptor_supervisor(Pid) of AsupPid when is_pid(AsupPid) -> - [{ListenAddr,Port,Profile}] = - [{LA,Prt,Prf} || {{ssh_acceptor_sup,LA,Prt,Prf}, - _WorkerPid,worker,[ssh_acceptor]} <- supervisor:which_children(AsupPid)], + [{IP,Port,Profile}] = + [{IP,Prt,Prf} + || {{ssh_acceptor_sup,Hst,Prt,Prf},_Pid,worker,[ssh_acceptor]} + <- supervisor:which_children(AsupPid), + IP <- [case inet:parse_strict_address(Hst) of + {ok,IP} -> IP; + _ -> Hst + end] + ], {ok, [{port,Port}, - {listen_address,ListenAddr}, + {ip,IP}, {profile,Profile} ]}; _ -> @@ -222,8 +301,14 @@ stop_listener(SysSup) -> ssh_system_sup:stop_listener(SysSup). stop_listener(Address, Port) -> stop_listener(Address, Port, ?DEFAULT_PROFILE). +stop_listener(any, Port, Profile) -> + map_ip(fun(IP) -> + ssh_system_sup:stop_listener(IP, Port, Profile) + end, [{0,0,0,0},{0,0,0,0,0,0,0,0}]); stop_listener(Address, Port, Profile) -> - ssh_system_sup:stop_listener(Address, Port, Profile). + map_ip(fun(IP) -> + ssh_system_sup:stop_listener(IP, Port, Profile) + end, {address,Address}). %%-------------------------------------------------------------------- -spec stop_daemon(daemon_ref()) -> ok. @@ -236,9 +321,15 @@ stop_listener(Address, Port, Profile) -> stop_daemon(SysSup) -> ssh_system_sup:stop_system(SysSup). stop_daemon(Address, Port) -> - ssh_system_sup:stop_system(Address, Port, ?DEFAULT_PROFILE). + stop_daemon(Address, Port, ?DEFAULT_PROFILE). +stop_daemon(any, Port, Profile) -> + map_ip(fun(IP) -> + ssh_system_sup:stop_system(IP, Port, Profile) + end, [{0,0,0,0},{0,0,0,0,0,0,0,0}]); stop_daemon(Address, Port, Profile) -> - ssh_system_sup:stop_system(Address, Port, Profile). + map_ip(fun(IP) -> + ssh_system_sup:stop_system(IP, Port, Profile) + end, {address,Address}). %%-------------------------------------------------------------------- -spec shell(inet:socket() | string()) -> _. @@ -292,35 +383,34 @@ default_algorithms() -> %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- -handle_daemon_args(Host, UserOptions0) -> - case Host of - socket -> - {Host, UserOptions0}; - any -> - {ok, Host0} = inet:gethostname(), - Inet = proplists:get_value(inet, UserOptions0, inet), - {Host0, [Inet | UserOptions0]}; - {_,_,_,_} -> - {Host, [inet, {ip,Host} | UserOptions0]}; - {_,_,_,_,_,_,_,_} -> - {Host, [inet6, {ip,Host} | UserOptions0]}; - _ -> - error(badarg) +%% The handle_daemon_args/2 function basically only sets the ip-option in Opts +%% so that it is correctly set when opening the listening socket. + +handle_daemon_args(any, Opts) -> + case proplists:get_value(ip, Opts) of + undefined -> {any, Opts}; + IP -> {IP, Opts} + end; + +handle_daemon_args(IPaddr, Opts) when is_tuple(IPaddr) ; IPaddr == loopback -> + case proplists:get_value(ip, Opts) of + undefined -> {IPaddr, [{ip,IPaddr}|Opts]}; + IPaddr -> {IPaddr, Opts}; + IP -> {IPaddr, [{ip,IPaddr}|Opts--[{ip,IP}]]} %% Backward compatibility end. %%%---------------------------------------------------------------- valid_socket_to_use(Socket, {tcp,_,_}) -> %% Is this tcp-socket a valid socket? - case {is_tcp_socket(Socket), - {ok,[{active,false}]} == inet:getopts(Socket, [active]) - } + try {is_tcp_socket(Socket), + {ok,[{active,false}]} == inet:getopts(Socket, [active]) + } of - {true, true} -> - ok; - {true, false} -> - {error, not_passive_mode}; - _ -> - {error, not_tcp_socket} + {true, true} -> ok; + {true, false} -> {error, not_passive_mode}; + _ -> {error, not_tcp_socket} + catch + _:_ -> {error, bad_socket} end; valid_socket_to_use(_, {L4,_,_}) -> @@ -334,158 +424,62 @@ is_tcp_socket(Socket) -> end. %%%---------------------------------------------------------------- -start_daemon(_, _, {error,Error}) -> - {error,Error}; - -start_daemon(socket, Socket, Options) -> - case valid_socket_to_use(Socket, ?GET_OPT(transport,Options)) of - ok -> - try - do_start_daemon(Socket, Options) - catch - throw:bad_fd -> {error,bad_fd}; - throw:bad_socket -> {error,bad_socket}; - _C:_E -> {error,{cannot_start_daemon,_C,_E}} - end; - {error,SockError} -> - {error,SockError} - end; +open_listen_socket(_Host0, Port0, Options0) -> + {ok,LSock} = + case ?GET_SOCKET_OPT(fd, Options0) of + undefined -> + ssh_acceptor:listen(Port0, Options0); + Fd when is_integer(Fd) -> + %% Do gen_tcp:listen with the option {fd,Fd}: + ssh_acceptor:listen(0, Options0) + end, + {ok,{LHost,LPort}} = inet:sockname(LSock), + {{LHost,LPort}, LSock}. -start_daemon(Host, Port, Options) -> +%%%---------------------------------------------------------------- +finalize_start(Host, Port, Profile, Options0, F) -> try - do_start_daemon(Host, Port, Options) + sshd_sup:start_child(Host, Port, Profile, Options0) + of + {error, {already_started, _}} -> + {error, eaddrinuse}; + {error, Error} -> + {error, Error}; + Result = {ok,_} -> + F(Options0, Result) catch - throw:bad_fd -> {error,bad_fd}; - throw:bad_socket -> {error,bad_socket}; - _C:_E -> {error,{cannot_start_daemon,_C,_E}} + exit:{noproc, _} -> + {error, ssh_not_started} end. +%%%---------------------------------------------------------------- +map_ip(Fun, {address,IP}) when is_tuple(IP) -> + Fun(IP); +map_ip(Fun, {address,Address}) -> + IPs = try {ok,#hostent{h_addr_list=IP0s}} = inet:gethostbyname(Address), + IP0s + catch + _:_ -> [] + end, + map_ip(Fun, IPs); +map_ip(Fun, IPs) -> + lists:map(Fun, IPs). -do_start_daemon(Socket, Options) -> - {ok, {IP,Port}} = - try {ok,_} = inet:sockname(Socket) - catch - _:_ -> throw(bad_socket) - end, - Host = fmt_host(IP), - Opts = ?PUT_INTERNAL_OPT([{asocket, Socket}, - {asock_owner,self()}, - {address, Host}, - {port, Port}, - {role, server}], Options), - - Profile = ?GET_OPT(profile, Options), - case ssh_system_sup:system_supervisor(Host, Port, Profile) of - undefined -> - try sshd_sup:start_child(Opts) of - {error, {already_started, _}} -> - {error, eaddrinuse}; - Result = {ok,_} -> - call_ssh_acceptor_handle_connection(Host, Port, Opts, Socket, Result); - Result = {error, _} -> - Result - catch - exit:{noproc, _} -> - {error, ssh_not_started} - end; - Sup -> - AccPid = ssh_system_sup:acceptor_supervisor(Sup), - case ssh_acceptor_sup:start_child(AccPid, Opts) of - {error, {already_started, _}} -> - {error, eaddrinuse}; - {ok, _} -> - call_ssh_acceptor_handle_connection(Host, Port, Opts, Socket, {ok,Sup}); - Other -> - Other - end - end. - -do_start_daemon(Host0, Port0, Options0) -> - {Host,Port1} = - try - case ?GET_SOCKET_OPT(fd, Options0) of - undefined -> - {Host0,Port0}; - Fd when Port0==0 -> - find_hostport(Fd) - end - catch - _:_ -> throw(bad_fd) - end, - {Port, WaitRequestControl, Options1} = - case Port1 of - 0 -> %% Allocate the socket here to get the port number... - {ok,LSock} = ssh_acceptor:callback_listen(0, Options0), - {ok,{_,LPort}} = inet:sockname(LSock), - {LPort, - LSock, - ?PUT_INTERNAL_OPT({lsocket,{LSock,self()}}, Options0) - }; - _ -> - {Port1, false, Options0} - end, - Options = ?PUT_INTERNAL_OPT([{address, Host}, - {port, Port}, - {role, server}], Options1), - Profile = ?GET_OPT(profile, Options0), - case ssh_system_sup:system_supervisor(Host, Port, Profile) of - undefined -> - try sshd_sup:start_child(Options) of - {error, {already_started, _}} -> - {error, eaddrinuse}; - Result = {ok,_} -> - sync_request_control(WaitRequestControl, Options), - Result; - Result = {error, _} -> - Result - catch - exit:{noproc, _} -> - {error, ssh_not_started} - end; - Sup -> - AccPid = ssh_system_sup:acceptor_supervisor(Sup), - case ssh_acceptor_sup:start_child(AccPid, Options) of - {error, {already_started, _}} -> - {error, eaddrinuse}; - {ok, _} -> - sync_request_control(WaitRequestControl, Options), - {ok, Sup}; - Other -> - Other - end - end. - -call_ssh_acceptor_handle_connection(Host, Port, Options, Socket, DefaultResult) -> - {_, Callback, _} = ?GET_OPT(transport, Options), - try ssh_acceptor:handle_connection(Callback, Host, Port, Options, Socket) - of - {error,Error} -> {error,Error}; - _ -> DefaultResult - catch - C:R -> {error,{could_not_start_connection,{C,R}}} - end. - - -sync_request_control(false, _Options) -> - ok; -sync_request_control(LSock, Options) -> - {_, Callback, _} = ?GET_OPT(transport, Options), - receive - {request_control,LSock,ReqPid} -> - ok = Callback:controlling_process(LSock, ReqPid), - ReqPid ! {its_yours,LSock}, - ok +%%%---------------------------------------------------------------- +mangle_connect_address(A, SockOpts) -> + mangle_connect_address1(A, proplists:get_value(inet6,SockOpts,false)). + +loopback(true) -> {0,0,0,0,0,0,0,1}; +loopback(false) -> {127,0,0,1}. + +mangle_connect_address1( loopback, V6flg) -> loopback(V6flg); +mangle_connect_address1( any, V6flg) -> loopback(V6flg); +mangle_connect_address1({0,0,0,0}, _) -> loopback(false); +mangle_connect_address1({0,0,0,0,0,0,0,0}, _) -> loopback(true); +mangle_connect_address1( IP, _) when is_tuple(IP) -> IP; +mangle_connect_address1(A, _) -> + case catch inet:parse_address(A) of + {ok, {0,0,0,0}} -> loopback(false); + {ok, {0,0,0,0,0,0,0,0}} -> loopback(true); + _ -> A end. - -find_hostport(Fd) -> - %% Using internal functions inet:open/8 and inet:close/0. - %% Don't try this at home unless you know what you are doing! - {ok,S} = inet:open(Fd, {0,0,0,0}, 0, [], tcp, inet, stream, inet_tcp), - {ok, HostPort} = inet:sockname(S), - ok = inet:close(S), - HostPort. - -fmt_host({A,B,C,D}) -> - lists:concat([A,".",B,".",C,".",D]); -fmt_host(T={_,_,_,_,_,_,_,_}) -> - lists:flatten(string:join([io_lib:format("~.16B",[A]) || A <- tuple_to_list(T)], ":")). diff --git a/lib/ssh/src/ssh.hrl b/lib/ssh/src/ssh.hrl index c1ba58ed40..315310f700 100644 --- a/lib/ssh/src/ssh.hrl +++ b/lib/ssh/src/ssh.hrl @@ -75,9 +75,12 @@ %% Option access macros -define(do_get_opt(C,K,O), ssh_options:get_value(C,K,O, ?MODULE,?LINE)). --define(do_get_opt(C,K,O,D), ssh_options:get_value(C,K,O,D,?MODULE,?LINE)). +-define(do_get_opt(C,K,O,D), ssh_options:get_value(C,K,O,?LAZY(D),?MODULE,?LINE)). + +-define(LAZY(D), fun()-> D end). -define(GET_OPT(Key,Opts), ?do_get_opt(user_options, Key,Opts ) ). +-define(GET_OPT(Key,Opts,Def), ?do_get_opt(user_options, Key,Opts,Def) ). -define(GET_INTERNAL_OPT(Key,Opts), ?do_get_opt(internal_options,Key,Opts ) ). -define(GET_INTERNAL_OPT(Key,Opts,Def), ?do_get_opt(internal_options,Key,Opts,Def) ). -define(GET_SOCKET_OPT(Key,Opts), ?do_get_opt(socket_options, Key,Opts ) ). @@ -89,6 +92,10 @@ -define(PUT_INTERNAL_OPT(KeyVal,Opts), ?do_put_opt(internal_options,KeyVal,Opts) ). -define(PUT_SOCKET_OPT(KeyVal,Opts), ?do_put_opt(socket_options, KeyVal,Opts) ). +-define(do_del_opt(C,K,O), ssh_options:delete_key(C,K,O, ?MODULE,?LINE)). +-define(DELETE_INTERNAL_OPT(Key,Opts), ?do_del_opt(internal_options,Key,Opts) ). + + %% Types -type role() :: client | server . -type ok_error(SuccessType) :: {ok, SuccessType} | {error, any()} . @@ -109,12 +116,25 @@ -type double_algs() :: list( {client2serverlist,simple_algs()} | {server2client,simple_algs()} ) | simple_algs() . +-type options() :: #{socket_options := socket_options(), + internal_options := internal_options(), + option_key() => any() + }. + +-type socket_options() :: proplists:proplist(). +-type internal_options() :: #{option_key() => any()}. + +-type option_key() :: atom(). + + %% Records -record(ssh, { - role, %% client | server - peer, %% string version of peer address + role :: client | role(), + peer :: undefined | + {inet:hostname(), + {inet:ip_adress(),inet:port_number()}}, %% string version of peer address c_vsn, %% client version {Major,Minor} s_vsn, %% server version {Major,Minor} diff --git a/lib/ssh/src/ssh_acceptor.erl b/lib/ssh/src/ssh_acceptor.erl index 42be18f2ad..f7fbd7ccad 100644 --- a/lib/ssh/src/ssh_acceptor.erl +++ b/lib/ssh/src/ssh_acceptor.erl @@ -27,8 +27,8 @@ %% Internal application API -export([start_link/4, number_of_connections/1, - callback_listen/2, - handle_connection/5]). + listen/2, + handle_established_connection/4]). %% spawn export -export([acceptor_init/5, acceptor_loop/6]). @@ -42,41 +42,57 @@ start_link(Port, Address, Options, AcceptTimeout) -> Args = [self(), Port, Address, Options, AcceptTimeout], proc_lib:start_link(?MODULE, acceptor_init, Args). +%%%---------------------------------------------------------------- +number_of_connections(SystemSup) -> + length([X || + {R,X,supervisor,[ssh_subsystem_sup]} <- supervisor:which_children(SystemSup), + is_pid(X), + is_reference(R) + ]). + +%%%---------------------------------------------------------------- +listen(Port, Options) -> + {_, Callback, _} = ?GET_OPT(transport, Options), + SockOpts = [{active, false}, {reuseaddr,true} | ?GET_OPT(socket_options, Options)], + case Callback:listen(Port, SockOpts) of + {error, nxdomain} -> + Callback:listen(Port, lists:delete(inet6, SockOpts)); + {error, enetunreach} -> + Callback:listen(Port, lists:delete(inet6, SockOpts)); + {error, eafnosupport} -> + Callback:listen(Port, lists:delete(inet6, SockOpts)); + Other -> + Other + end. + +%%%---------------------------------------------------------------- +handle_established_connection(Address, Port, Options, Socket) -> + {_, Callback, _} = ?GET_OPT(transport, Options), + handle_connection(Callback, Address, Port, Options, Socket). + %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- acceptor_init(Parent, Port, Address, Opts, AcceptTimeout) -> - {_, Callback, _} = ?GET_OPT(transport, Opts), try - {LSock0,SockOwner0} = ?GET_INTERNAL_OPT(lsocket, Opts), - true = is_pid(SockOwner0), - {ok,{_,Port}} = inet:sockname(LSock0), - {LSock0, SockOwner0} + ?GET_INTERNAL_OPT(lsocket, Opts) of {LSock, SockOwner} -> - %% Use existing socket - proc_lib:init_ack(Parent, {ok, self()}), - request_ownership(LSock, SockOwner), - acceptor_loop(Callback, Port, Address, Opts, LSock, AcceptTimeout) - catch - error:{badkey,lsocket} -> - %% Open new socket - try - socket_listen(Port, Opts) - of - {ok, ListenSocket} -> + case inet:sockname(LSock) of + {ok,{_,Port}} -> % A usable, open LSock proc_lib:init_ack(Parent, {ok, self()}), - {_, Callback, _} = ?GET_OPT(transport, Opts), - acceptor_loop(Callback, - Port, Address, Opts, ListenSocket, AcceptTimeout); - {error,Error} -> - proc_lib:init_ack(Parent, Error), - {error,Error} - catch - _:_ -> - {error,listen_socket_failed} - end; + request_ownership(LSock, SockOwner), + {_, Callback, _} = ?GET_OPT(transport, Opts), + acceptor_loop(Callback, Port, Address, Opts, LSock, AcceptTimeout); + {error,_} -> % Not open, a restart + {ok,NewLSock} = listen(Port, Opts), + proc_lib:init_ack(Parent, {ok, self()}), + Opts1 = ?DELETE_INTERNAL_OPT(lsocket, Opts), + {_, Callback, _} = ?GET_OPT(transport, Opts1), + acceptor_loop(Callback, Port, Address, Opts1, NewLSock, AcceptTimeout) + end + catch _:_ -> {error,use_existing_socket_failed} end. @@ -88,30 +104,7 @@ request_ownership(LSock, SockOwner) -> {its_yours,LSock} -> ok end. - -socket_listen(Port0, Opts) -> - Port = case ?GET_SOCKET_OPT(fd, Opts) of - undefined -> Port0; - _ -> 0 - end, - callback_listen(Port, Opts). - - -callback_listen(Port, Opts0) -> - {_, Callback, _} = ?GET_OPT(transport, Opts0), - Opts = ?PUT_SOCKET_OPT([{active, false}, {reuseaddr,true}], Opts0), - SockOpts = ?GET_OPT(socket_options, Opts), - case Callback:listen(Port, SockOpts) of - {error, nxdomain} -> - Callback:listen(Port, lists:delete(inet6, SockOpts)); - {error, enetunreach} -> - Callback:listen(Port, lists:delete(inet6, SockOpts)); - {error, eafnosupport} -> - Callback:listen(Port, lists:delete(inet6, SockOpts)); - Other -> - Other - end. - +%%%---------------------------------------------------------------- acceptor_loop(Callback, Port, Address, Opts, ListenSocket, AcceptTimeout) -> case (catch Callback:accept(ListenSocket, AcceptTimeout)) of {ok, Socket} -> @@ -128,6 +121,7 @@ acceptor_loop(Callback, Port, Address, Opts, ListenSocket, AcceptTimeout) -> ListenSocket, AcceptTimeout) end. +%%%---------------------------------------------------------------- handle_connection(Callback, Address, Port, Options, Socket) -> Profile = ?GET_OPT(profile, Options), SystemSup = ssh_system_sup:system_supervisor(Address, Port, Profile), @@ -135,7 +129,8 @@ handle_connection(Callback, Address, Port, Options, Socket) -> MaxSessions = ?GET_OPT(max_sessions, Options), case number_of_connections(SystemSup) < MaxSessions of true -> - {ok, SubSysSup} = ssh_system_sup:start_subsystem(SystemSup, Options), + {ok, SubSysSup} = + ssh_system_sup:start_subsystem(SystemSup, server, Address, Port, Profile, Options), ConnectionSup = ssh_subsystem_sup:connection_supervisor(SubSysSup), NegTimeout = ?GET_OPT(negotiation_timeout, Options), ssh_connection_handler:start_connection(server, Socket, @@ -159,7 +154,7 @@ handle_connection(Callback, Address, Port, Options, Socket) -> {error,max_sessions} end. - +%%%---------------------------------------------------------------- handle_error(timeout) -> ok; @@ -186,10 +181,3 @@ handle_error(Reason) -> error_logger:error_report(String), exit({accept_failed, String}). - -number_of_connections(SystemSup) -> - length([X || - {R,X,supervisor,[ssh_subsystem_sup]} <- supervisor:which_children(SystemSup), - is_pid(X), - is_reference(R) - ]). diff --git a/lib/ssh/src/ssh_acceptor_sup.erl b/lib/ssh/src/ssh_acceptor_sup.erl index 77f7826918..26defcfdbd 100644 --- a/lib/ssh/src/ssh_acceptor_sup.erl +++ b/lib/ssh/src/ssh_acceptor_sup.erl @@ -29,7 +29,7 @@ -include("ssh.hrl"). --export([start_link/1, start_child/2, stop_child/4]). +-export([start_link/4, start_child/5, stop_child/4]). %% Supervisor callback -export([init/1]). @@ -41,19 +41,19 @@ %%%========================================================================= %%% API %%%========================================================================= -start_link(Servers) -> - supervisor:start_link(?MODULE, [Servers]). +start_link(Address, Port, Profile, Options) -> + supervisor:start_link(?MODULE, [Address, Port, Profile, Options]). -start_child(AccSup, Options) -> - Spec = child_spec(Options), +start_child(AccSup, Address, Port, Profile, Options) -> + Spec = child_spec(Address, Port, Profile, Options), case supervisor:start_child(AccSup, Spec) of {error, already_present} -> - Address = ?GET_INTERNAL_OPT(address, Options), - Port = ?GET_INTERNAL_OPT(port, Options), - Profile = ?GET_OPT(profile, Options), + %% Is this ever called? stop_child(AccSup, Address, Port, Profile), supervisor:start_child(AccSup, Spec); Reply -> + %% Reply = {ok,SystemSupPid} when the user calls ssh:daemon + %% after having called ssh:stop_listening Reply end. @@ -69,34 +69,29 @@ stop_child(AccSup, Address, Port, Profile) -> %%%========================================================================= %%% Supervisor callback %%%========================================================================= -init([Options]) -> - RestartStrategy = one_for_one, - MaxR = 10, - MaxT = 3600, - Children = [child_spec(Options)], - {ok, {{RestartStrategy, MaxR, MaxT}, Children}}. +init([Address, Port, Profile, Options]) -> + %% Initial start of ssh_acceptor_sup for this port or new start after + %% ssh:stop_daemon + SupFlags = #{strategy => one_for_one, + intensity => 10, + period => 3600 + }, + ChildSpecs = [child_spec(Address, Port, Profile, Options)], + {ok, {SupFlags,ChildSpecs}}. %%%========================================================================= %%% Internal functions %%%========================================================================= -child_spec(Options) -> - Address = ?GET_INTERNAL_OPT(address, Options), - Port = ?GET_INTERNAL_OPT(port, Options), +child_spec(Address, Port, Profile, Options) -> Timeout = ?GET_INTERNAL_OPT(timeout, Options, ?DEFAULT_TIMEOUT), - Profile = ?GET_OPT(profile, Options), - Name = id(Address, Port, Profile), - StartFunc = {ssh_acceptor, start_link, [Port, Address, Options, Timeout]}, - Restart = transient, - Shutdown = brutal_kill, - Modules = [ssh_acceptor], - Type = worker, - {Name, StartFunc, Restart, Shutdown, Type, Modules}. + #{id => id(Address, Port, Profile), + start => {ssh_acceptor, start_link, [Port, Address, Options, Timeout]}, + restart => transient, + shutdown => 5500, %brutal_kill, + type => worker, + modules => [ssh_acceptor] + }. id(Address, Port, Profile) -> - case is_list(Address) of - true -> - {ssh_acceptor_sup, any, Port, Profile}; - false -> - {ssh_acceptor_sup, Address, Port, Profile} - end. + {ssh_acceptor_sup, Address, Port, Profile}. diff --git a/lib/ssh/src/ssh_connection_handler.erl b/lib/ssh/src/ssh_connection_handler.erl index b9c643c77e..ff94e5dfb6 100644 --- a/lib/ssh/src/ssh_connection_handler.erl +++ b/lib/ssh/src/ssh_connection_handler.erl @@ -400,7 +400,9 @@ init_process_state(Role, Socket, Opts) -> timer:apply_after(?REKEY_DATA_TIMOUT, gen_statem, cast, [self(), data_size]), cache_init_idle_timer(D); server -> - D#data{connection_state = init_connection(Role, C, Opts)} + cache_init_idle_timer( + D#data{connection_state = init_connection(Role, C, Opts)} + ) end. @@ -411,14 +413,8 @@ init_connection(server, C = #connection{}, Opts) -> SubSystemSup = proplists:get_value(subsystem_sup, Sups), ConnectionSup = proplists:get_value(connection_sup, Sups), - Shell = ?GET_OPT(shell, Opts), - Exec = ?GET_OPT(exec, Opts), - CliSpec = case ?GET_OPT(ssh_cli, Opts) of - undefined -> {ssh_cli, [Shell]}; - Spec -> Spec - end, - C#connection{cli_spec = CliSpec, - exec = Exec, + C#connection{cli_spec = ?GET_OPT(ssh_cli, Opts, {ssh_cli,[?GET_OPT(shell, Opts)]}), + exec = ?GET_OPT(exec, Opts), system_supervisor = SystemSup, sub_system_supervisor = SubSystemSup, connection_supervisor = ConnectionSup @@ -444,7 +440,12 @@ init_ssh_record(Role, Socket, Opts) -> {Vsn, Version} = ssh_transport:versions(Role, Opts), case Role of client -> - PeerName = ?GET_INTERNAL_OPT(host, Opts), + PeerName = case ?GET_INTERNAL_OPT(host, Opts) of + PeerIP when is_tuple(PeerIP) -> + inet_parse:ntoa(PeerIP); + PeerName0 -> + PeerName0 + end, S0#ssh{c_vsn = Vsn, c_version = Version, io_cb = case ?GET_OPT(user_interaction, Opts) of @@ -919,6 +920,9 @@ handle_event(internal, Msg=#ssh_msg_channel_extended_data{}, StateName, D) - handle_event(internal, Msg=#ssh_msg_channel_eof{}, StateName, D) -> handle_connection_msg(Msg, StateName, D); +handle_event(internal, Msg=#ssh_msg_channel_close{}, {connected,server} = StateName, D) -> + handle_connection_msg(Msg, StateName, cache_request_idle_timer_check(D)); + handle_event(internal, Msg=#ssh_msg_channel_close{}, StateName, D) -> handle_connection_msg(Msg, StateName, D); diff --git a/lib/ssh/src/ssh_connection_sup.erl b/lib/ssh/src/ssh_connection_sup.erl index 0f54053f52..fad796f196 100644 --- a/lib/ssh/src/ssh_connection_sup.erl +++ b/lib/ssh/src/ssh_connection_sup.erl @@ -45,19 +45,17 @@ start_child(Sup, Args) -> %%%========================================================================= %%% Supervisor callback %%%========================================================================= --spec init( [term()] ) -> {ok,{supervisor:sup_flags(),[supervisor:child_spec()]}} | ignore . - init(_) -> - RestartStrategy = simple_one_for_one, - MaxR = 0, - MaxT = 3600, - - Name = undefined, % As simple_one_for_one is used. - StartFunc = {ssh_connection_handler, start_link, []}, - Restart = temporary, % E.g. should not be restarted - Shutdown = 4000, - Modules = [ssh_connection_handler], - Type = worker, - - ChildSpec = {Name, StartFunc, Restart, Shutdown, Type, Modules}, - {ok, {{RestartStrategy, MaxR, MaxT}, [ChildSpec]}}. + SupFlags = #{strategy => simple_one_for_one, + intensity => 0, + period => 3600 + }, + ChildSpecs = [#{id => undefined, % As simple_one_for_one is used. + start => {ssh_connection_handler, start_link, []}, + restart => temporary, + shutdown => 4000, + type => worker, + modules => [ssh_connection_handler] + } + ], + {ok, {SupFlags,ChildSpecs}}. diff --git a/lib/ssh/src/ssh_file.erl b/lib/ssh/src/ssh_file.erl index 898b4cc5c4..88f4d10792 100644 --- a/lib/ssh/src/ssh_file.erl +++ b/lib/ssh/src/ssh_file.erl @@ -221,6 +221,8 @@ file_name(Type, Name, Opts) -> %% in: "host" out: "host,1.2.3.4. +add_ip(IP) when is_tuple(IP) -> + ssh_connection:encode_ip(IP); add_ip(Host) -> case inet:getaddr(Host, inet) of {ok, Addr} -> diff --git a/lib/ssh/src/ssh_options.erl b/lib/ssh/src/ssh_options.erl index a882a01eaf..ee3cdbb8a0 100644 --- a/lib/ssh/src/ssh_options.erl +++ b/lib/ssh/src/ssh_options.erl @@ -28,6 +28,7 @@ -export([default/1, get_value/5, get_value/6, put_value/5, + delete_key/5, handle_options/2 ]). @@ -37,16 +38,6 @@ %%%================================================================ %%% Types --type options() :: #{socket_options := socket_options(), - internal_options := internal_options(), - option_key() => any() - }. - --type socket_options() :: proplists:proplist(). --type internal_options() :: #{option_key() => any()}. - --type option_key() :: atom(). - -type option_in() :: proplists:property() | proplists:proplist() . -type option_class() :: internal_options | socket_options | user_options . @@ -75,22 +66,23 @@ get_value(Class, Key, Opts, _CallerMod, _CallerLine) when is_map(Opts) -> user_options -> maps:get(Key, Opts) end; get_value(Class, Key, Opts, _CallerMod, _CallerLine) -> - io:format("*** Bad Opts GET OPT ~p ~p:~p Key=~p,~n Opts=~p~n",[Class,_CallerMod,_CallerLine,Key,Opts]), error({bad_options,Class, Key, Opts, _CallerMod, _CallerLine}). --spec get_value(option_class(), option_key(), options(), any(), +-spec get_value(option_class(), option_key(), options(), fun(() -> any()), atom(), non_neg_integer()) -> any() | no_return(). -get_value(socket_options, Key, Opts, Def, _CallerMod, _CallerLine) when is_map(Opts) -> - proplists:get_value(Key, maps:get(socket_options,Opts), Def); -get_value(Class, Key, Opts, Def, CallerMod, CallerLine) when is_map(Opts) -> +get_value(socket_options, Key, Opts, DefFun, _CallerMod, _CallerLine) when is_map(Opts) -> + proplists:get_value(Key, maps:get(socket_options,Opts), DefFun); +get_value(Class, Key, Opts, DefFun, CallerMod, CallerLine) when is_map(Opts) -> try get_value(Class, Key, Opts, CallerMod, CallerLine) + of + undefined -> DefFun(); + Value -> Value catch - error:{badkey,Key} -> Def + error:{badkey,Key} -> DefFun() end; -get_value(Class, Key, Opts, _Def, _CallerMod, _CallerLine) -> - io:format("*** Bad Opts GET OPT ~p ~p:~p Key=~p,~n Opts=~p~n",[Class,_CallerMod,_CallerLine,Key,Opts]), +get_value(Class, Key, Opts, _DefFun, _CallerMod, _CallerLine) -> error({bad_options,Class, Key, Opts, _CallerMod, _CallerLine}). @@ -136,6 +128,19 @@ put_socket_value(A, SockOpts) when is_atom(A) -> %%%================================================================ %%% +%%% Delete an option +%%% + +-spec delete_key(option_class(), option_key(), options(), + atom(), non_neg_integer()) -> options(). + +delete_key(internal_options, Key, Opts, _CallerMod, _CallerLine) when is_map(Opts) -> + InternalOpts = maps:get(internal_options,Opts), + Opts#{internal_options := maps:remove(Key, InternalOpts)}. + + +%%%================================================================ +%%% %%% Initialize the options %%% @@ -490,12 +495,6 @@ default(client) -> class => user_options }, - {idle_time, def} => - #{default => infinity, - chk => fun check_timeout/1, - class => user_options - }, - %%%%% Undocumented {keyboard_interact_fun, def} => #{default => undefined, @@ -548,6 +547,12 @@ default(common) -> class => user_options }, + {idle_time, def} => + #{default => infinity, + chk => fun check_timeout/1, + class => user_options + }, + %% This is a "SocketOption"... %% {fd, def} => %% #{default => undefined, @@ -792,18 +797,17 @@ read_moduli_file(D, I, Acc) -> check_silently_accept_hosts(B) when is_boolean(B) -> true; check_silently_accept_hosts(F) when is_function(F,2) -> true; -check_silently_accept_hosts({S,F}) when is_atom(S), - is_function(F,2) -> - lists:member(S, ?SHAs) andalso - lists:member(S, proplists:get_value(hashs,crypto:supports())); -check_silently_accept_hosts({L,F}) when is_list(L), - is_function(F,2) -> - lists:all(fun(S) -> - lists:member(S, ?SHAs) andalso - lists:member(S, proplists:get_value(hashs,crypto:supports())) - end, L); +check_silently_accept_hosts({false,S}) when is_atom(S) -> valid_hash(S); +check_silently_accept_hosts({S,F}) when is_function(F,2) -> valid_hash(S); check_silently_accept_hosts(_) -> false. + +valid_hash(S) -> valid_hash(S, proplists:get_value(hashs,crypto:supports())). + +valid_hash(S, Ss) when is_atom(S) -> lists:member(S, ?SHAs) andalso lists:member(S, Ss); +valid_hash(L, Ss) when is_list(L) -> lists:all(fun(S) -> valid_hash(S,Ss) end, L); +valid_hash(X, _) -> error_in_check(X, "Expect atom or list in fingerprint spec"). + %%%---------------------------------------------------------------- check_preferred_algorithms(Algs) -> try alg_duplicates(Algs, [], []) @@ -871,6 +875,7 @@ handle_pref_alg(Key, Vs, _) -> chk_alg_vs(OptKey, Values, SupportedValues) -> case (Values -- SupportedValues) of [] -> Values; + [none] -> [none]; % for testing only Bad -> error_in_check({OptKey,Bad}, "Unsupported value(s) found") end. diff --git a/lib/ssh/src/ssh_sftp.erl b/lib/ssh/src/ssh_sftp.erl index 140856c8e3..f1f7b57e8d 100644 --- a/lib/ssh/src/ssh_sftp.erl +++ b/lib/ssh/src/ssh_sftp.erl @@ -1063,15 +1063,6 @@ attr_to_info(A) when is_record(A, ssh_xfer_attr) -> gid = A#ssh_xfer_attr.group}. -%% Added workaround for sftp timestam problem. (Timestamps should be -%% in UTC but they where not) . The workaround uses a deprecated -%% function i calandar. This will work as expected most of the time -%% but has problems for the same reason as -%% calendar:local_time_to_universal_time/1. We consider it better that -%% the timestamps work as expected most of the time instead of none of -%% the time. Hopfully the file-api will be updated so that we can -%% solve this problem in a better way in the future. - unix_to_datetime(undefined) -> undefined; unix_to_datetime(UTCSecs) -> diff --git a/lib/ssh/src/ssh_sftpd.erl b/lib/ssh/src/ssh_sftpd.erl index 9352046795..b879116393 100644 --- a/lib/ssh/src/ssh_sftpd.erl +++ b/lib/ssh/src/ssh_sftpd.erl @@ -34,8 +34,7 @@ %%-------------------------------------------------------------------- %% External exports --export([subsystem_spec/1, - listen/1, listen/2, listen/3, stop/1]). +-export([subsystem_spec/1]). -export([init/1, handle_ssh_msg/2, handle_msg/2, terminate/2]). @@ -76,29 +75,6 @@ subsystem_spec(Options) -> {"sftp", {?MODULE, Options}}. -%%% DEPRECATED START %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -%%-------------------------------------------------------------------- -%% Function: listen() -> Pid | {error,Error} -%% Description: Starts the server -%%-------------------------------------------------------------------- -listen(Port) -> - listen(any, Port, []). -listen(Port, Options) -> - listen(any, Port, Options). -listen(Addr, Port, Options) -> - SubSystems = [subsystem_spec(Options)], - ssh:daemon(Addr, Port, [{subsystems, SubSystems} |Options]). - -%%-------------------------------------------------------------------- -%% Function: stop(Pid) -> ok -%% Description: Stops the listener -%%-------------------------------------------------------------------- -stop(Pid) -> - ssh:stop_listener(Pid). - - -%%% DEPRECATED END %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%==================================================================== %% subsystem callbacks diff --git a/lib/ssh/src/ssh_subsystem_sup.erl b/lib/ssh/src/ssh_subsystem_sup.erl index cf82db458f..cf409ade6b 100644 --- a/lib/ssh/src/ssh_subsystem_sup.erl +++ b/lib/ssh/src/ssh_subsystem_sup.erl @@ -28,7 +28,7 @@ -include("ssh.hrl"). --export([start_link/1, +-export([start_link/5, connection_supervisor/1, channel_supervisor/1 ]). @@ -39,8 +39,8 @@ %%%========================================================================= %%% API %%%========================================================================= -start_link(Options) -> - supervisor:start_link(?MODULE, [Options]). +start_link(Role, Address, Port, Profile, Options) -> + supervisor:start_link(?MODULE, [Role, Address, Port, Profile, Options]). connection_supervisor(SupPid) -> Children = supervisor:which_children(SupPid), @@ -53,49 +53,40 @@ channel_supervisor(SupPid) -> %%%========================================================================= %%% Supervisor callback %%%========================================================================= --spec init( [term()] ) -> {ok,{supervisor:sup_flags(),[supervisor:child_spec()]}} | ignore . - -init([Options]) -> - RestartStrategy = one_for_all, - MaxR = 0, - MaxT = 3600, - Children = child_specs(Options), - {ok, {{RestartStrategy, MaxR, MaxT}, Children}}. +init([Role, Address, Port, Profile, Options]) -> + SupFlags = #{strategy => one_for_all, + intensity => 0, + period => 3600 + }, + ChildSpecs = child_specs(Role, Address, Port, Profile, Options), + {ok, {SupFlags,ChildSpecs}}. %%%========================================================================= %%% Internal functions %%%========================================================================= -child_specs(Options) -> - case ?GET_INTERNAL_OPT(role, Options) of - client -> - []; - server -> - [ssh_channel_child_spec(Options), ssh_connectinon_child_spec(Options)] - end. +child_specs(client, _Address, _Port, _Profile, _Options) -> + []; +child_specs(server, Address, Port, Profile, Options) -> + [ssh_channel_child_spec(server, Address, Port, Profile, Options), + ssh_connection_child_spec(server, Address, Port, Profile, Options)]. -ssh_connectinon_child_spec(Options) -> - Address = ?GET_INTERNAL_OPT(address, Options), - Port = ?GET_INTERNAL_OPT(port, Options), - Role = ?GET_INTERNAL_OPT(role, Options), - Name = id(Role, ssh_connection_sup, Address, Port), - StartFunc = {ssh_connection_sup, start_link, [Options]}, - Restart = temporary, - Shutdown = 5000, - Modules = [ssh_connection_sup], - Type = supervisor, - {Name, StartFunc, Restart, Shutdown, Type, Modules}. - -ssh_channel_child_spec(Options) -> - Address = ?GET_INTERNAL_OPT(address, Options), - Port = ?GET_INTERNAL_OPT(port, Options), - Role = ?GET_INTERNAL_OPT(role, Options), - Name = id(Role, ssh_channel_sup, Address, Port), - StartFunc = {ssh_channel_sup, start_link, [Options]}, - Restart = temporary, - Shutdown = infinity, - Modules = [ssh_channel_sup], - Type = supervisor, - {Name, StartFunc, Restart, Shutdown, Type, Modules}. +ssh_connection_child_spec(Role, Address, Port, _Profile, Options) -> + #{id => id(Role, ssh_connection_sup, Address, Port), + start => {ssh_connection_sup, start_link, [Options]}, + restart => temporary, + shutdown => 5000, + type => supervisor, + modules => [ssh_connection_sup] + }. + +ssh_channel_child_spec(Role, Address, Port, _Profile, Options) -> + #{id => id(Role, ssh_channel_sup, Address, Port), + start => {ssh_channel_sup, start_link, [Options]}, + restart => temporary, + shutdown => infinity, + type => supervisor, + modules => [ssh_channel_sup] + }. id(Role, Sup, Address, Port) -> {Role, Sup, Address, Port}. diff --git a/lib/ssh/src/ssh_sup.erl b/lib/ssh/src/ssh_sup.erl index 8b57387589..26574763e4 100644 --- a/lib/ssh/src/ssh_sup.erl +++ b/lib/ssh/src/ssh_sup.erl @@ -31,63 +31,20 @@ %%%========================================================================= %%% Supervisor callback %%%========================================================================= --spec init( [term()] ) -> {ok,{supervisor:sup_flags(),[supervisor:child_spec()]}} | ignore . - -init([]) -> - SupFlags = {one_for_one, 10, 3600}, - Children = children(), - {ok, {SupFlags, Children}}. - -%%%========================================================================= -%%% Internal functions -%%%========================================================================= -get_services() -> - case (catch application:get_env(ssh, services)) of - {ok, Services} -> - Services; - _ -> - [] - end. - -children() -> - Services = get_services(), - Clients = [Service || Service <- Services, is_client(Service)], - Servers = [Service || Service <- Services, is_server(Service)], - - [server_child_spec(Servers), client_child_spec(Clients)]. - -server_child_spec(Servers) -> - Name = sshd_sup, - StartFunc = {sshd_sup, start_link, [Servers]}, - Restart = permanent, - Shutdown = infinity, - Modules = [sshd_sup], - Type = supervisor, - {Name, StartFunc, Restart, Shutdown, Type, Modules}. - -client_child_spec(Clients) -> - Name = sshc_sup, - StartFunc = {sshc_sup, start_link, [Clients]}, - Restart = permanent, - Shutdown = infinity, - Modules = [sshc_sup], - Type = supervisor, - {Name, StartFunc, Restart, Shutdown, Type, Modules}. - -is_server({sftpd, _}) -> - true; -is_server({shelld, _}) -> - true; -is_server(_) -> - false. - -is_client({sftpc, _}) -> - true; -is_client({shellc, _}) -> - true; -is_client(_) -> - false. - - - +init(_) -> + SupFlags = #{strategy => one_for_one, + intensity => 10, + period => 3600 + }, + ChildSpecs = [#{id => Module, + start => {Module, start_link, []}, + restart => permanent, + shutdown => 4000, %brutal_kill, + type => supervisor, + modules => [Module] + } + || Module <- [sshd_sup, + sshc_sup] + ], + {ok, {SupFlags,ChildSpecs}}. diff --git a/lib/ssh/src/ssh_system_sup.erl b/lib/ssh/src/ssh_system_sup.erl index b0bbd3aae5..84b4cd3241 100644 --- a/lib/ssh/src/ssh_system_sup.erl +++ b/lib/ssh/src/ssh_system_sup.erl @@ -21,7 +21,7 @@ %% %%---------------------------------------------------------------------- %% Purpose: The ssh server instance supervisor, an instans of this supervisor -%% exists for every ip-address and port combination, hangs under +%% exists for every ip-address and port combination, hangs under %% sshd_sup. %%---------------------------------------------------------------------- @@ -31,64 +31,103 @@ -include("ssh.hrl"). --export([start_link/1, stop_listener/1, +-export([start_link/4, stop_listener/1, stop_listener/3, stop_system/1, stop_system/3, system_supervisor/3, - subsystem_supervisor/1, channel_supervisor/1, - connection_supervisor/1, - acceptor_supervisor/1, start_subsystem/2, restart_subsystem/3, - restart_acceptor/3, stop_subsystem/2]). + subsystem_supervisor/1, channel_supervisor/1, + connection_supervisor/1, + acceptor_supervisor/1, start_subsystem/6, + stop_subsystem/2]). %% Supervisor callback -export([init/1]). %%%========================================================================= -%%% Internal API +%%% API %%%========================================================================= -start_link(Options) -> - Address = ?GET_INTERNAL_OPT(address, Options), - Port = ?GET_INTERNAL_OPT(port, Options), - Profile = ?GET_OPT(profile, Options), +start_link(Address, Port, Profile, Options) -> Name = make_name(Address, Port, Profile), - supervisor:start_link({local, Name}, ?MODULE, [Options]). + supervisor:start_link({local, Name}, ?MODULE, [Address, Port, Profile, Options]). -stop_listener(SysSup) -> - stop_acceptor(SysSup). +%%%========================================================================= +%%% Supervisor callback +%%%========================================================================= +init([Address, Port, Profile, Options]) -> + SupFlags = #{strategy => one_for_one, + intensity => 0, + period => 3600 + }, + ChildSpecs = + case ?GET_INTERNAL_OPT(connected_socket,Options,undefined) of + undefined -> + [#{id => id(ssh_acceptor_sup, Address, Port, Profile), + start => {ssh_acceptor_sup, start_link, [Address, Port, Profile, Options]}, + restart => transient, + shutdown => infinity, + type => supervisor, + modules => [ssh_acceptor_sup] + }]; + _ -> + [] + end, + {ok, {SupFlags,ChildSpecs}}. + +%%%========================================================================= +%%% Service API +%%%========================================================================= +stop_listener(SystemSup) -> + {Name, AcceptorSup, _, _} = lookup(ssh_acceptor_sup, SystemSup), + case supervisor:terminate_child(AcceptorSup, Name) of + ok -> + supervisor:delete_child(AcceptorSup, Name); + Error -> + Error + end. stop_listener(Address, Port, Profile) -> - Name = make_name(Address, Port, Profile), - stop_acceptor(whereis(Name)). - + stop_listener( + system_supervisor(Address, Port, Profile)). + + stop_system(SysSup) -> - Name = sshd_sup:system_name(SysSup), - spawn(fun() -> sshd_sup:stop_child(Name) end), + spawn(fun() -> sshd_sup:stop_child(SysSup) end), ok. -stop_system(Address, Port, Profile) -> +stop_system(Address, Port, Profile) -> spawn(fun() -> sshd_sup:stop_child(Address, Port, Profile) end), ok. + system_supervisor(Address, Port, Profile) -> Name = make_name(Address, Port, Profile), whereis(Name). subsystem_supervisor(SystemSup) -> - ssh_subsystem_sup(supervisor:which_children(SystemSup)). + {_, Child, _, _} = lookup(ssh_subsystem_sup, SystemSup), + Child. channel_supervisor(SystemSup) -> - SubSysSup = ssh_subsystem_sup(supervisor:which_children(SystemSup)), - ssh_subsystem_sup:channel_supervisor(SubSysSup). + ssh_subsystem_sup:channel_supervisor( + subsystem_supervisor(SystemSup)). connection_supervisor(SystemSup) -> - SubSysSup = ssh_subsystem_sup(supervisor:which_children(SystemSup)), - ssh_subsystem_sup:connection_supervisor(SubSysSup). + ssh_subsystem_sup:connection_supervisor( + subsystem_supervisor(SystemSup)). acceptor_supervisor(SystemSup) -> - ssh_acceptor_sup(supervisor:which_children(SystemSup)). + {_, Child, _, _} = lookup(ssh_acceptor_sup, SystemSup), + Child. -start_subsystem(SystemSup, Options) -> - Spec = ssh_subsystem_child_spec(Options), - supervisor:start_child(SystemSup, Spec). + +start_subsystem(SystemSup, Role, Address, Port, Profile, Options) -> + SubsystemSpec = + #{id => make_ref(), + start => {ssh_subsystem_sup, start_link, [Role, Address, Port, Profile, Options]}, + restart => temporary, + shutdown => infinity, + type => supervisor, + modules => [ssh_subsystem_sup]}, + supervisor:start_child(SystemSup, SubsystemSpec). stop_subsystem(SystemSup, SubSys) -> case catch lists:keyfind(SubSys, 2, supervisor:which_children(SystemSup)) of @@ -106,100 +145,21 @@ stop_subsystem(SystemSup, SubSys) -> ok end. - -restart_subsystem(Address, Port, Profile) -> - SysSupName = make_name(Address, Port, Profile), - SubSysName = id(ssh_subsystem_sup, Address, Port, Profile), - case supervisor:terminate_child(SysSupName, SubSysName) of - ok -> - supervisor:restart_child(SysSupName, SubSysName); - Error -> - Error - end. - -restart_acceptor(Address, Port, Profile) -> - SysSupName = make_name(Address, Port, Profile), - AcceptorName = id(ssh_acceptor_sup, Address, Port, Profile), - supervisor:restart_child(SysSupName, AcceptorName). - -%%%========================================================================= -%%% Supervisor callback -%%%========================================================================= --spec init( [term()] ) -> {ok,{supervisor:sup_flags(),[supervisor:child_spec()]}} | ignore . - -init([Options]) -> - RestartStrategy = one_for_one, - MaxR = 0, - MaxT = 3600, - Children = case ?GET_INTERNAL_OPT(asocket,Options,undefined) of - undefined -> child_specs(Options); - _ -> [] - end, - {ok, {{RestartStrategy, MaxR, MaxT}, Children}}. - %%%========================================================================= %%% Internal functions %%%========================================================================= -child_specs(Options) -> - [ssh_acceptor_child_spec(Options)]. - -ssh_acceptor_child_spec(Options) -> - Address = ?GET_INTERNAL_OPT(address, Options), - Port = ?GET_INTERNAL_OPT(port, Options), - Profile = ?GET_OPT(profile, Options), - Name = id(ssh_acceptor_sup, Address, Port, Profile), - StartFunc = {ssh_acceptor_sup, start_link, [Options]}, - Restart = transient, - Shutdown = infinity, - Modules = [ssh_acceptor_sup], - Type = supervisor, - {Name, StartFunc, Restart, Shutdown, Type, Modules}. - -ssh_subsystem_child_spec(Options) -> - Name = make_ref(), - StartFunc = {ssh_subsystem_sup, start_link, [Options]}, - Restart = temporary, - Shutdown = infinity, - Modules = [ssh_subsystem_sup], - Type = supervisor, - {Name, StartFunc, Restart, Shutdown, Type, Modules}. - - id(Sup, Address, Port, Profile) -> - case is_list(Address) of - true -> - {Sup, any, Port, Profile}; - false -> - {Sup, Address, Port, Profile} - end. + {Sup, Address, Port, Profile}. make_name(Address, Port, Profile) -> - case is_list(Address) of - true -> - list_to_atom(lists:flatten(io_lib:format("ssh_system_~p_~p_~p_sup", - [any, Port, Profile]))); - false -> - list_to_atom(lists:flatten(io_lib:format("ssh_system_~p_~p_~p_sup", - [Address, Port, Profile]))) - end. + list_to_atom(lists:flatten(io_lib:format("ssh_system_~s_~p_~p_sup", [fmt_host(Address), Port, Profile]))). -ssh_subsystem_sup([{_, Child, _, [ssh_subsystem_sup]} | _]) -> - Child; -ssh_subsystem_sup([_ | Rest]) -> - ssh_subsystem_sup(Rest). +fmt_host(IP) when is_tuple(IP) -> inet:ntoa(IP); +fmt_host(A) when is_atom(A) -> A; +fmt_host(S) when is_list(S) -> S. -ssh_acceptor_sup([{_, Child, _, [ssh_acceptor_sup]} | _]) -> - Child; -ssh_acceptor_sup([_ | Rest]) -> - ssh_acceptor_sup(Rest). -stop_acceptor(Sup) -> - [{Name, AcceptorSup}] = - [{SupName, ASup} || {SupName, ASup, _, [ssh_acceptor_sup]} <- - supervisor:which_children(Sup)], - case supervisor:terminate_child(AcceptorSup, Name) of - ok -> - supervisor:delete_child(AcceptorSup, Name); - Error -> - Error - end. +lookup(SupModule, SystemSup) -> + lists:keyfind([SupModule], 4, + supervisor:which_children(SystemSup)). + diff --git a/lib/ssh/src/ssh_transport.erl b/lib/ssh/src/ssh_transport.erl index 02c995399a..54ea80c727 100644 --- a/lib/ssh/src/ssh_transport.erl +++ b/lib/ssh/src/ssh_transport.erl @@ -200,9 +200,6 @@ is_valid_mac(Mac, Data, #ssh{recv_mac = Algorithm, recv_mac_key = Key, recv_sequence = SeqNum}) -> Mac == mac(Algorithm, Key, SeqNum, Data). -yes_no(Ssh, Prompt) -> - (Ssh#ssh.io_cb):yes_no(Prompt, Ssh). - format_version({Major,Minor}, SoftwareVersion) -> "SSH-" ++ integer_to_list(Major) ++ "." ++ integer_to_list(Minor) ++ "-" ++ SoftwareVersion. @@ -755,16 +752,44 @@ public_algo({#'ECPoint'{},{namedCurve,OID}}) -> accepted_host(Ssh, PeerName, Public, Opts) -> case ?GET_OPT(silently_accept_hosts, Opts) of - F when is_function(F,2) -> + + %% Original option values; User question and no host key fingerprints known. + %% Keep the original question unchanged: + false -> yes == yes_no(Ssh, "New host " ++ PeerName ++ " accept"); + true -> true; + + %% Variant: User question but with host key fingerprint in the question: + {false,Alg} -> + HostKeyAlg = (Ssh#ssh.algorithms)#alg.hkey, + Prompt = io_lib:format("The authenticity of the host can't be established.~n" + "~s host key fingerprint is ~s.~n" + "New host ~p accept", + [fmt_hostkey(HostKeyAlg), + public_key:ssh_hostkey_fingerprint(Alg,Public), + PeerName]), + yes == yes_no(Ssh, Prompt); + + %% Call-back alternatives: A user provided fun is called for the decision: + F when is_function(F,2) -> true == (catch F(PeerName, public_key:ssh_hostkey_fingerprint(Public))); + {DigestAlg,F} when is_function(F,2) -> - true == (catch F(PeerName, public_key:ssh_hostkey_fingerprint(DigestAlg,Public))); - true -> - true; - false -> - yes == yes_no(Ssh, "New host " ++ PeerName ++ " accept") + true == (catch F(PeerName, public_key:ssh_hostkey_fingerprint(DigestAlg,Public))) + end. + +yes_no(Ssh, Prompt) -> + (Ssh#ssh.io_cb):yes_no(Prompt, Ssh#ssh.opts). + + +fmt_hostkey('ssh-rsa') -> "RSA"; +fmt_hostkey('ssh-dss') -> "DSA"; +fmt_hostkey(A) when is_atom(A) -> fmt_hostkey(atom_to_list(A)); +fmt_hostkey("ecdsa"++_) -> "ECDSA"; +fmt_hostkey(X) -> X. + + known_host_key(#ssh{opts = Opts, key_cb = {KeyCb,KeyCbOpts}, peer = {PeerName,_}} = Ssh, Public, Alg) -> UserOpts = ?GET_OPT(user_options, Opts), diff --git a/lib/ssh/src/sshc_sup.erl b/lib/ssh/src/sshc_sup.erl index 15858f36e1..c71b81dc6d 100644 --- a/lib/ssh/src/sshc_sup.erl +++ b/lib/ssh/src/sshc_sup.erl @@ -27,23 +27,25 @@ -behaviour(supervisor). --export([start_link/1, start_child/1, stop_child/1]). +-export([start_link/0, start_child/1, stop_child/1]). %% Supervisor callback -export([init/1]). +-define(SSHC_SUP, ?MODULE). + %%%========================================================================= %%% API %%%========================================================================= -start_link(Args) -> - supervisor:start_link({local, ?MODULE}, ?MODULE, [Args]). +start_link() -> + supervisor:start_link({local,?SSHC_SUP}, ?MODULE, []). start_child(Args) -> supervisor:start_child(?MODULE, Args). stop_child(Client) -> spawn(fun() -> - ClientSup = whereis(?MODULE), + ClientSup = whereis(?SSHC_SUP), supervisor:terminate_child(ClientSup, Client) end), ok. @@ -51,22 +53,17 @@ stop_child(Client) -> %%%========================================================================= %%% Supervisor callback %%%========================================================================= --spec init( [term()] ) -> {ok,{supervisor:sup_flags(),[supervisor:child_spec()]}} | ignore . - -init(Args) -> - RestartStrategy = simple_one_for_one, - MaxR = 0, - MaxT = 3600, - {ok, {{RestartStrategy, MaxR, MaxT}, [child_spec(Args)]}}. - -%%%========================================================================= -%%% Internal functions -%%%========================================================================= -child_spec(_) -> - Name = undefined, % As simple_one_for_one is used. - StartFunc = {ssh_connection_handler, start_link, []}, - Restart = temporary, - Shutdown = 4000, - Modules = [ssh_connection_handler], - Type = worker, - {Name, StartFunc, Restart, Shutdown, Type, Modules}. +init(_) -> + SupFlags = #{strategy => simple_one_for_one, + intensity => 0, + period => 3600 + }, + ChildSpecs = [#{id => undefined, % As simple_one_for_one is used. + start => {ssh_connection_handler, start_link, []}, + restart => temporary, + shutdown => 4000, + type => worker, + modules => [ssh_connection_handler] + } + ], + {ok, {SupFlags,ChildSpecs}}. diff --git a/lib/ssh/src/sshd_sup.erl b/lib/ssh/src/sshd_sup.erl index 14f1937abd..449ba20d02 100644 --- a/lib/ssh/src/sshd_sup.erl +++ b/lib/ssh/src/sshd_sup.erl @@ -19,7 +19,7 @@ %% %% %%---------------------------------------------------------------------- -%% Purpose: The top supervisor for ssh servers hangs under +%% Purpose: The top supervisor for ssh servers hangs under %% ssh_sup. %%---------------------------------------------------------------------- @@ -29,90 +29,79 @@ -include("ssh.hrl"). --export([start_link/1, start_child/1, stop_child/1, - stop_child/3, system_name/1]). +-export([start_link/0, + start_child/4, + stop_child/1, + stop_child/3 +]). %% Supervisor callback -export([init/1]). +-define(SSHD_SUP, ?MODULE). + %%%========================================================================= %%% API %%%========================================================================= -start_link(Servers) -> - supervisor:start_link({local, ?MODULE}, ?MODULE, [Servers]). +start_link() -> + %% No children are start now. We wait until the user calls ssh:daemon + %% and uses start_child/4 to create the children + supervisor:start_link({local,?SSHD_SUP}, ?MODULE, []). -start_child(Options) -> - Address = ?GET_INTERNAL_OPT(address, Options), - Port = ?GET_INTERNAL_OPT(port, Options), - Profile = ?GET_OPT(profile, Options), +start_child(Address, Port, Profile, Options) -> case ssh_system_sup:system_supervisor(Address, Port, Profile) of undefined -> - Spec = child_spec(Address, Port, Options), - case supervisor:start_child(?MODULE, Spec) of - {error, already_present} -> - Name = id(Address, Port, Profile), - supervisor:delete_child(?MODULE, Name), - supervisor:start_child(?MODULE, Spec); - Reply -> - Reply - end; + %% Here we start listening on a new Host/Port/Profile + Spec = child_spec(Address, Port, Profile, Options), + supervisor:start_child(?SSHD_SUP, Spec); Pid -> + %% Here we resume listening on a new Host/Port/Profile after + %% haveing stopped listening to he same with ssh:stop_listen(Pid) AccPid = ssh_system_sup:acceptor_supervisor(Pid), - ssh_acceptor_sup:start_child(AccPid, Options) + ssh_acceptor_sup:start_child(AccPid, Address, Port, Profile, Options), + {ok,Pid} end. -stop_child(Name) -> - supervisor:terminate_child(?MODULE, Name). +stop_child(ChildId) when is_tuple(ChildId) -> + supervisor:terminate_child(?SSHD_SUP, ChildId); +stop_child(ChildPid) when is_pid(ChildPid)-> + stop_child(system_name(ChildPid)). -stop_child(Address, Port, Profile) -> - Name = id(Address, Port, Profile), - stop_child(Name). -system_name(SysSup) -> - Children = supervisor:which_children(sshd_sup), - system_name(SysSup, Children). +stop_child(Address, Port, Profile) -> + Id = id(Address, Port, Profile), + stop_child(Id). %%%========================================================================= %%% Supervisor callback %%%========================================================================= --spec init( [term()] ) -> {ok,{supervisor:sup_flags(),[supervisor:child_spec()]}} | ignore . - -init([Servers]) -> - RestartStrategy = one_for_one, - MaxR = 10, - MaxT = 3600, - Fun = fun(ServerOpts) -> - Address = ?GET_INTERNAL_OPT(address, ServerOpts), - Port = ?GET_INTERNAL_OPT(port, ServerOpts), - child_spec(Address, Port, ServerOpts) - end, - Children = lists:map(Fun, Servers), - {ok, {{RestartStrategy, MaxR, MaxT}, Children}}. +init(_) -> + SupFlags = #{strategy => one_for_one, + intensity => 10, + period => 3600 + }, + ChildSpecs = [ + ], + {ok, {SupFlags,ChildSpecs}}. %%%========================================================================= %%% Internal functions %%%========================================================================= -child_spec(Address, Port, Options) -> - Profile = ?GET_OPT(profile, Options), - Name = id(Address, Port,Profile), - StartFunc = {ssh_system_sup, start_link, [Options]}, - Restart = temporary, - Shutdown = infinity, - Modules = [ssh_system_sup], - Type = supervisor, - {Name, StartFunc, Restart, Shutdown, Type, Modules}. +child_spec(Address, Port, Profile, Options) -> + #{id => id(Address, Port, Profile), + start => {ssh_system_sup, start_link, [Address, Port, Profile, Options]}, + restart => temporary, + shutdown => infinity, + type => supervisor, + modules => [ssh_system_sup] + }. id(Address, Port, Profile) -> - case is_list(Address) of - true -> - {server, ssh_system_sup, any, Port, Profile}; - false -> - {server, ssh_system_sup, Address, Port, Profile} + {server, ssh_system_sup, Address, Port, Profile}. + +system_name(SysSup) -> + case lists:keyfind(SysSup, 2, supervisor:which_children(?SSHD_SUP)) of + {Name, SysSup, _, _} -> Name; + false -> undefind end. -system_name([], _ ) -> - undefined; -system_name(SysSup, [{Name, SysSup, _, _} | _]) -> - Name; -system_name(SysSup, [_ | Rest]) -> - system_name(SysSup, Rest). diff --git a/lib/ssh/test/ssh_algorithms_SUITE.erl b/lib/ssh/test/ssh_algorithms_SUITE.erl index 6f75d83c4a..2990d1e02a 100644 --- a/lib/ssh/test/ssh_algorithms_SUITE.erl +++ b/lib/ssh/test/ssh_algorithms_SUITE.erl @@ -235,13 +235,12 @@ sshc_simple_exec_os_cmd(Config) -> Parent = self(), Client = spawn( fun() -> - Cmd = lists:concat(["ssh -p ",Port, - " -C" - " -o UserKnownHostsFile=",KnownHosts, - " -o StrictHostKeyChecking=no" - " ",Host," 1+1."]), - Result = os:cmd(Cmd), - ct:log("~p~n = ~p",[Cmd, Result]), + Result = ssh_test_lib:open_sshc(Host, Port, + [" -C" + " -o UserKnownHostsFile=",KnownHosts, + " -o StrictHostKeyChecking=no" + ], + " 1+1."), Parent ! {result, self(), Result, "2"} end), receive diff --git a/lib/ssh/test/ssh_basic_SUITE.erl b/lib/ssh/test/ssh_basic_SUITE.erl index cdf6cf9ae1..089d191fea 100644 --- a/lib/ssh/test/ssh_basic_SUITE.erl +++ b/lib/ssh/test/ssh_basic_SUITE.erl @@ -46,7 +46,8 @@ exec_key_differs2/1, exec_key_differs3/1, exec_key_differs_fail/1, - idle_time/1, + idle_time_client/1, + idle_time_server/1, inet6_option/1, inet_option/1, internal_error/1, @@ -139,7 +140,7 @@ basic_tests() -> exec, exec_compressed, shell, shell_no_unicode, shell_unicode_string, cli, known_hosts, - idle_time, openssh_zlib_basic_test, + idle_time_client, idle_time_server, openssh_zlib_basic_test, misc_ssh_options, inet_option, inet6_option]. @@ -522,8 +523,8 @@ exec_compressed(Config) when is_list(Config) -> end. %%-------------------------------------------------------------------- -%%% Idle timeout test -idle_time(Config) -> +%%% Idle timeout test, client +idle_time_client(Config) -> SystemDir = filename:join(proplists:get_value(priv_dir, Config), system), UserDir = proplists:get_value(priv_dir, Config), @@ -544,6 +545,28 @@ idle_time(Config) -> ssh:stop_daemon(Pid). %%-------------------------------------------------------------------- +%%% Idle timeout test, server +idle_time_server(Config) -> + SystemDir = filename:join(proplists:get_value(priv_dir, Config), system), + UserDir = proplists:get_value(priv_dir, Config), + + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, + {user_dir, UserDir}, + {idle_time, 2000}, + {failfun, fun ssh_test_lib:failfun/2}]), + ConnectionRef = + ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user_dir, UserDir}, + {user_interaction, false}]), + {ok, Id} = ssh_connection:session_channel(ConnectionRef, 1000), + ssh_connection:close(ConnectionRef, Id), + receive + after 10000 -> + {error, closed} = ssh_connection:session_channel(ConnectionRef, 1000) + end, + ssh:stop_daemon(Pid). + +%%-------------------------------------------------------------------- %%% Test that ssh:shell/2 works shell(Config) when is_list(Config) -> process_flag(trap_exit, true), @@ -719,7 +742,8 @@ known_hosts(Config) when is_list(Config) -> Lines = string:tokens(binary_to_list(Binary), "\n"), [Line] = Lines, [HostAndIp, Alg, _KeyData] = string:tokens(Line, " "), - [Host, _Ip] = string:tokens(HostAndIp, ","), + [StoredHost, _Ip] = string:tokens(HostAndIp, ","), + true = ssh_test_lib:match_ip(StoredHost, Host), "ssh-" ++ _ = Alg, ssh:stop_daemon(Pid). %%-------------------------------------------------------------------- diff --git a/lib/ssh/test/ssh_bench_SUITE.erl b/lib/ssh/test/ssh_bench_SUITE.erl index ac52bb7e28..317e50ed1d 100644 --- a/lib/ssh/test/ssh_bench_SUITE.erl +++ b/lib/ssh/test/ssh_bench_SUITE.erl @@ -98,7 +98,7 @@ end_per_testcase(_Func, _Conf) -> connect(Config) -> KexAlgs = proplists:get_value(kex, ssh:default_algorithms()), - ct:pal("KexAlgs = ~p",[KexAlgs]), + ct:log("KexAlgs = ~p",[KexAlgs]), lists:foreach( fun(KexAlg) -> PrefAlgs = preferred_algorithms(KexAlg), @@ -242,11 +242,11 @@ median(Data) when is_list(Data) -> 1 -> lists:nth(N div 2 + 1, SortedData) end, - ct:pal("median(~p) = ~p",[SortedData,Median]), + ct:log("median(~p) = ~p",[SortedData,Median]), Median. report(Data) -> - ct:pal("EventData = ~p",[Data]), + ct:log("EventData = ~p",[Data]), ct_event:notify(#event{name = benchmark_data, data = Data}). diff --git a/lib/ssh/test/ssh_connection_SUITE.erl b/lib/ssh/test/ssh_connection_SUITE.erl index 2819a4dbd9..b911cf0e9e 100644 --- a/lib/ssh/test/ssh_connection_SUITE.erl +++ b/lib/ssh/test/ssh_connection_SUITE.erl @@ -89,7 +89,7 @@ end_per_suite(Config) -> %%-------------------------------------------------------------------- init_per_group(openssh, Config) -> - case gen_tcp:connect("localhost", 22, []) of + case ssh_test_lib:gen_tcp_connect("localhost", 22, []) of {error,econnrefused} -> {skip,"No openssh deamon"}; {ok, Socket} -> @@ -126,7 +126,7 @@ simple_exec(Config) when is_list(Config) -> simple_exec_sock(_Config) -> - {ok, Sock} = gen_tcp:connect("localhost", ?SSH_DEFAULT_PORT, [{active,false}]), + {ok, Sock} = ssh_test_lib:gen_tcp_connect("localhost", ?SSH_DEFAULT_PORT, [{active,false}]), {ok, ConnectionRef} = ssh:connect(Sock, [{silently_accept_hosts, true}, {user_interaction, false}]), do_simple_exec(ConnectionRef). @@ -179,13 +179,13 @@ daemon_sock_not_tcp(_Config) -> %%-------------------------------------------------------------------- connect_sock_not_passive(_Config) -> - {ok,Sock} = gen_tcp:connect("localhost", ?SSH_DEFAULT_PORT, []), + {ok,Sock} = ssh_test_lib:gen_tcp_connect("localhost", ?SSH_DEFAULT_PORT, []), {error, not_passive_mode} = ssh:connect(Sock, []), gen_tcp:close(Sock). %%-------------------------------------------------------------------- daemon_sock_not_passive(_Config) -> - {ok,Sock} = gen_tcp:connect("localhost", ?SSH_DEFAULT_PORT, []), + {ok,Sock} = ssh_test_lib:gen_tcp_connect("localhost", ?SSH_DEFAULT_PORT, []), {error, not_passive_mode} = ssh:daemon(Sock), gen_tcp:close(Sock). @@ -585,12 +585,13 @@ start_shell_sock_exec_fun(Config) when is_list(Config) -> UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth file:make_dir(UserDir), SysDir = proplists:get_value(data_dir, Config), - {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, - {user_dir, UserDir}, - {password, "morot"}, - {exec, fun ssh_exec/1}]), + {Pid, HostD, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"}, + {exec, fun ssh_exec/1}]), + Host = ssh_test_lib:ntoa(ssh_test_lib:mangle_connect_address(HostD)), - {ok, Sock} = gen_tcp:connect(Host, Port, [{active,false}]), + {ok, Sock} = ssh_test_lib:gen_tcp_connect(Host, Port, [{active,false}]), {ok,ConnectionRef} = ssh:connect(Sock, [{silently_accept_hosts, true}, {user, "foo"}, {password, "morot"}, @@ -623,7 +624,7 @@ start_shell_sock_daemon_exec(Config) -> {ok,{_IP,Port}} = inet:sockname(Sl), % _IP is likely to be {0,0,0,0}. Win don't like... spawn_link(fun() -> - {ok,Ss} = gen_tcp:connect("localhost", Port, [{active,false}]), + {ok,Ss} = ssh_test_lib:gen_tcp_connect("localhost", Port, [{active,false}]), {ok, _Pid} = ssh:daemon(Ss, [{system_dir, SysDir}, {user_dir, UserDir}, {password, "morot"}, @@ -658,10 +659,10 @@ gracefull_invalid_version(Config) when is_list(Config) -> SysDir = proplists:get_value(data_dir, Config), {_Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, - {user_dir, UserDir}, - {password, "morot"}]), + {user_dir, UserDir}, + {password, "morot"}]), - {ok, S} = gen_tcp:connect(Host, Port, []), + {ok, S} = ssh_test_lib:gen_tcp_connect(Host, Port, []), ok = gen_tcp:send(S, ["SSH-8.-1","\r\n"]), receive Verstring -> @@ -680,10 +681,10 @@ gracefull_invalid_start(Config) when is_list(Config) -> file:make_dir(UserDir), SysDir = proplists:get_value(data_dir, Config), {_Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, - {user_dir, UserDir}, - {password, "morot"}]), + {user_dir, UserDir}, + {password, "morot"}]), - {ok, S} = gen_tcp:connect(Host, Port, []), + {ok, S} = ssh_test_lib:gen_tcp_connect(Host, Port, []), ok = gen_tcp:send(S, ["foobar","\r\n"]), receive Verstring -> @@ -702,10 +703,10 @@ gracefull_invalid_long_start(Config) when is_list(Config) -> file:make_dir(UserDir), SysDir = proplists:get_value(data_dir, Config), {_Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, - {user_dir, UserDir}, - {password, "morot"}]), + {user_dir, UserDir}, + {password, "morot"}]), - {ok, S} = gen_tcp:connect(Host, Port, []), + {ok, S} = ssh_test_lib:gen_tcp_connect(Host, Port, []), ok = gen_tcp:send(S, [lists:duplicate(257, $a), "\r\n"]), receive Verstring -> @@ -725,10 +726,10 @@ gracefull_invalid_long_start_no_nl(Config) when is_list(Config) -> file:make_dir(UserDir), SysDir = proplists:get_value(data_dir, Config), {_Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, - {user_dir, UserDir}, - {password, "morot"}]), + {user_dir, UserDir}, + {password, "morot"}]), - {ok, S} = gen_tcp:connect(Host, Port, []), + {ok, S} = ssh_test_lib:gen_tcp_connect(Host, Port, []), ok = gen_tcp:send(S, [lists:duplicate(257, $a), "\r\n"]), receive Verstring -> @@ -779,22 +780,21 @@ stop_listener(Config) when is_list(Config) -> ct:fail("Exec Timeout") end, - {ok, HostAddr} = inet:getaddr(Host, inet), - case ssh_test_lib:daemon(HostAddr, Port, [{system_dir, SysDir}, - {user_dir, UserDir}, - {password, "potatis"}, - {exec, fun ssh_exec/1}]) of - {Pid1, HostAddr, Port} -> + case ssh_test_lib:daemon(Port, [{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "potatis"}, + {exec, fun ssh_exec/1}]) of + {Pid1, Host, Port} -> ConnectionRef1 = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, {user, "foo"}, {password, "potatis"}, {user_interaction, true}, {user_dir, UserDir}]), {error, _} = ssh:connect(Host, Port, [{silently_accept_hosts, true}, - {user, "foo"}, - {password, "morot"}, - {user_interaction, true}, - {user_dir, UserDir}]), + {user, "foo"}, + {password, "morot"}, + {user_interaction, true}, + {user_dir, UserDir}]), ssh:close(ConnectionRef0), ssh:close(ConnectionRef1), ssh:stop_daemon(Pid0), diff --git a/lib/ssh/test/ssh_options_SUITE.erl b/lib/ssh/test/ssh_options_SUITE.erl index 758c20e2b8..344a042d79 100644 --- a/lib/ssh/test/ssh_options_SUITE.erl +++ b/lib/ssh/test/ssh_options_SUITE.erl @@ -868,13 +868,13 @@ really_do_hostkey_fingerprint_check(Config, HashAlg) -> ct:log("Fingerprints(~p) = ~p",[HashAlg,FPs]), %% Start daemon with the public keys that we got fingerprints from - {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {Pid, Host0, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, {user_dir, UserDirServer}, {password, "morot"}]), - + Host = ssh_test_lib:ntoa(Host0), FP_check_fun = fun(PeerName, FP) -> - ct:pal("PeerName = ~p, FP = ~p",[PeerName,FP]), - HostCheck = (Host == PeerName), + ct:log("PeerName = ~p, FP = ~p",[PeerName,FP]), + HostCheck = ssh_test_lib:match_ip(Host, PeerName), FPCheck = if is_atom(HashAlg) -> lists:member(FP, FPs); is_list(HashAlg) -> lists:all(fun(FP1) -> lists:member(FP1,FPs) end, @@ -1052,20 +1052,20 @@ id_string_random_client(Config) -> %%-------------------------------------------------------------------- id_string_no_opt_server(Config) -> {_Server, Host, Port} = ssh_test_lib:std_daemon(Config, []), - {ok,S1}=gen_tcp:connect(Host,Port,[{active,false},{packet,line}]), + {ok,S1}=ssh_test_lib:gen_tcp_connect(Host,Port,[{active,false},{packet,line}]), {ok,"SSH-2.0-Erlang/"++Vsn} = gen_tcp:recv(S1, 0, 2000), true = expected_ssh_vsn(Vsn). %%-------------------------------------------------------------------- id_string_own_string_server(Config) -> {_Server, Host, Port} = ssh_test_lib:std_daemon(Config, [{id_string,"Olle"}]), - {ok,S1}=gen_tcp:connect(Host,Port,[{active,false},{packet,line}]), + {ok,S1}=ssh_test_lib:gen_tcp_connect(Host,Port,[{active,false},{packet,line}]), {ok,"SSH-2.0-Olle\r\n"} = gen_tcp:recv(S1, 0, 2000). %%-------------------------------------------------------------------- id_string_random_server(Config) -> {_Server, Host, Port} = ssh_test_lib:std_daemon(Config, [{id_string,random}]), - {ok,S1}=gen_tcp:connect(Host,Port,[{active,false},{packet,line}]), + {ok,S1}=ssh_test_lib:gen_tcp_connect(Host,Port,[{active,false},{packet,line}]), {ok,"SSH-2.0-"++Rnd} = gen_tcp:recv(S1, 0, 2000), case Rnd of "Erlang"++_ -> ct:log("Id=~p",[Rnd]), @@ -1086,11 +1086,11 @@ ssh_connect_negtimeout(Config, Parallel) -> ct:log("Parallel: ~p",[Parallel]), {_Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir},{user_dir, UserDir}, - {parallel_login, Parallel}, - {negotiation_timeout, NegTimeOut}, - {failfun, fun ssh_test_lib:failfun/2}]), - - {ok,Socket} = gen_tcp:connect(Host, Port, []), + {parallel_login, Parallel}, + {negotiation_timeout, NegTimeOut}, + {failfun, fun ssh_test_lib:failfun/2}]), + + {ok,Socket} = ssh_test_lib:gen_tcp_connect(Host, Port, []), Factor = 2, ct:log("And now sleeping ~p*NegTimeOut (~p ms)...", [Factor, round(Factor * NegTimeOut)]), diff --git a/lib/ssh/test/ssh_relay.erl b/lib/ssh/test/ssh_relay.erl index 28000fbb97..1e3810e9d4 100644 --- a/lib/ssh/test/ssh_relay.erl +++ b/lib/ssh/test/ssh_relay.erl @@ -131,7 +131,8 @@ init([ListenAddr, ListenPort, PeerAddr, PeerPort | _Options]) -> S = #state{local_addr = ListenAddr, local_port = ListenPort, lpid = LPid, - peer_addr = PeerAddr, + peer_addr = ssh_test_lib:ntoa( + ssh_test_lib:mangle_connect_address(PeerAddr)), peer_port = PeerPort }, {ok, S}; diff --git a/lib/ssh/test/ssh_sftp_SUITE.erl b/lib/ssh/test/ssh_sftp_SUITE.erl index acf76157a2..7efeb3a0ad 100644 --- a/lib/ssh/test/ssh_sftp_SUITE.erl +++ b/lib/ssh/test/ssh_sftp_SUITE.erl @@ -660,7 +660,7 @@ start_channel_sock(Config) -> {Host,Port} = proplists:get_value(peer, Config), %% Get a tcp socket - {ok, Sock} = gen_tcp:connect(Host, Port, [{active,false}]), + {ok, Sock} = ssh_test_lib:gen_tcp_connect(Host, Port, [{active,false}]), %% and open one channel on one new Connection {ok, ChPid1, Conn} = ssh_sftp:start_channel(Sock, Opts), diff --git a/lib/ssh/test/ssh_sftpd_SUITE.erl b/lib/ssh/test/ssh_sftpd_SUITE.erl index b167f98ac8..379c0bcb0a 100644 --- a/lib/ssh/test/ssh_sftpd_SUITE.erl +++ b/lib/ssh/test/ssh_sftpd_SUITE.erl @@ -151,8 +151,8 @@ init_per_testcase(TestCase, Config) -> SubSystems = [ssh_sftpd:subsystem_spec([])], ssh:daemon(0, [{subsystems, SubSystems}|Options]) end, - {ok,Dinf} = ssh:daemon_info(Sftpd), - Port = proplists:get_value(port, Dinf), + + Port = ssh_test_lib:daemon_port(Sftpd), Cm = ssh_test_lib:connect(Port, [{user_dir, ClientUserDir}, @@ -187,7 +187,7 @@ init_per_testcase(TestCase, Config) -> [{sftp, {Cm, Channel}}, {sftpd, Sftpd }| Config]. end_per_testcase(_TestCase, Config) -> - ssh_sftpd:stop(proplists:get_value(sftpd, Config)), + catch ssh:stop_daemon(proplists:get_value(sftpd, Config)), {Cm, Channel} = proplists:get_value(sftp, Config), ssh_connection:close(Cm, Channel), ssh:close(Cm), @@ -705,10 +705,10 @@ try_access(Path, Cm, Channel, ReqId) -> {ok, <<?SSH_FXP_STATUS, ?UINT32(ReqId), ?UINT32(Code), Rest/binary>>, <<>>} -> case Code of ?SSH_FX_FILE_IS_A_DIRECTORY -> - ct:pal("Got the expected SSH_FX_FILE_IS_A_DIRECTORY status",[]), + ct:log("Got the expected SSH_FX_FILE_IS_A_DIRECTORY status",[]), ok; ?SSH_FX_FAILURE -> - ct:pal("Got the expected SSH_FX_FAILURE status",[]), + ct:log("Got the expected SSH_FX_FAILURE status",[]), ok; _ -> case Rest of diff --git a/lib/ssh/test/ssh_sftpd_erlclient_SUITE.erl b/lib/ssh/test/ssh_sftpd_erlclient_SUITE.erl index b4d7eadfa4..9b5d6b5fae 100644 --- a/lib/ssh/test/ssh_sftpd_erlclient_SUITE.erl +++ b/lib/ssh/test/ssh_sftpd_erlclient_SUITE.erl @@ -138,7 +138,7 @@ init_per_testcase(TestCase, Config) -> [{port, Port}, {sftp, {ChannelPid, Connection}}, {sftpd, Sftpd} | NewConfig]. end_per_testcase(_TestCase, Config) -> - catch ssh_sftpd:stop(proplists:get_value(sftpd, Config)), + catch ssh:stop_daemon(proplists:get_value(sftpd, Config)), {Sftp, Connection} = proplists:get_value(sftp, Config), catch ssh_sftp:stop_channel(Sftp), catch ssh:close(Connection), diff --git a/lib/ssh/test/ssh_sup_SUITE.erl b/lib/ssh/test/ssh_sup_SUITE.erl index ff53e1c4c6..dd7c4b1473 100644 --- a/lib/ssh/test/ssh_sup_SUITE.erl +++ b/lib/ssh/test/ssh_sup_SUITE.erl @@ -41,7 +41,8 @@ suite() -> {timetrap,{seconds,100}}]. all() -> - [default_tree, sshc_subtree, sshd_subtree, sshd_subtree_profile]. + [default_tree, sshc_subtree, sshd_subtree, sshd_subtree_profile, + killed_acceptor_restarts]. groups() -> []. @@ -100,6 +101,7 @@ default_tree(Config) when is_list(Config) -> ?wait_match([], supervisor:which_children(sshc_sup)), ?wait_match([], supervisor:which_children(sshd_sup)). +%%------------------------------------------------------------------------- sshc_subtree() -> [{doc, "Make sure the sshc subtree is correct"}]. sshc_subtree(Config) when is_list(Config) -> @@ -128,27 +130,31 @@ sshc_subtree(Config) when is_list(Config) -> ssh:close(Pid2), ?wait_match([], supervisor:which_children(sshc_sup)). +%%------------------------------------------------------------------------- sshd_subtree() -> [{doc, "Make sure the sshd subtree is correct"}]. sshd_subtree(Config) when is_list(Config) -> HostIP = proplists:get_value(host_ip, Config), Port = proplists:get_value(port, Config), SystemDir = proplists:get_value(data_dir, Config), - ssh:daemon(HostIP, Port, [{system_dir, SystemDir}, - {failfun, fun ssh_test_lib:failfun/2}, - {user_passwords, - [{?USER, ?PASSWD}]}]), + {ok,Daemon} = ssh:daemon(HostIP, Port, [{system_dir, SystemDir}, + {failfun, fun ssh_test_lib:failfun/2}, + {user_passwords, + [{?USER, ?PASSWD}]}]), - ?wait_match([{{server,ssh_system_sup, HostIP, Port, ?DEFAULT_PROFILE}, + ct:log("Expect HostIP=~p, Port=~p, Daemon=~p",[HostIP,Port,Daemon]), + ?wait_match([{{server,ssh_system_sup, ListenIP, Port, ?DEFAULT_PROFILE}, Daemon, supervisor, [ssh_system_sup]}], supervisor:which_children(sshd_sup), - Daemon), + [ListenIP,Daemon]), + true = ssh_test_lib:match_ip(HostIP, ListenIP), check_sshd_system_tree(Daemon, Config), ssh:stop_daemon(HostIP, Port), ct:sleep(?WAIT_FOR_SHUTDOWN), ?wait_match([], supervisor:which_children(sshd_sup)). +%%------------------------------------------------------------------------- sshd_subtree_profile() -> [{doc, "Make sure the sshd subtree using profile option is correct"}]. sshd_subtree_profile(Config) when is_list(Config) -> @@ -157,34 +163,105 @@ sshd_subtree_profile(Config) when is_list(Config) -> Profile = proplists:get_value(profile, Config), SystemDir = proplists:get_value(data_dir, Config), - {ok, _} = ssh:daemon(HostIP, Port, [{system_dir, SystemDir}, - {failfun, fun ssh_test_lib:failfun/2}, - {user_passwords, - [{?USER, ?PASSWD}]}, - {profile, Profile}]), - ?wait_match([{{server,ssh_system_sup, HostIP,Port,Profile}, + {ok, Daemon} = ssh:daemon(HostIP, Port, [{system_dir, SystemDir}, + {failfun, fun ssh_test_lib:failfun/2}, + {user_passwords, + [{?USER, ?PASSWD}]}, + {profile, Profile}]), + ct:log("Expect HostIP=~p, Port=~p, Profile=~p, Daemon=~p",[HostIP,Port,Profile,Daemon]), + ?wait_match([{{server,ssh_system_sup, ListenIP,Port,Profile}, Daemon, supervisor, [ssh_system_sup]}], supervisor:which_children(sshd_sup), - Daemon), + [ListenIP,Daemon]), + true = ssh_test_lib:match_ip(HostIP, ListenIP), check_sshd_system_tree(Daemon, Config), ssh:stop_daemon(HostIP, Port, Profile), ct:sleep(?WAIT_FOR_SHUTDOWN), ?wait_match([], supervisor:which_children(sshd_sup)). +%%------------------------------------------------------------------------- +killed_acceptor_restarts(Config) -> + Profile = proplists:get_value(profile, Config), + SystemDir = proplists:get_value(data_dir, Config), + UserDir = proplists:get_value(userdir, Config), + {ok, DaemonPid} = ssh:daemon(0, [{system_dir, SystemDir}, + {failfun, fun ssh_test_lib:failfun/2}, + {user_passwords, [{?USER, ?PASSWD}]}, + {profile, Profile}]), + + {ok, DaemonPid2} = ssh:daemon(0, [{system_dir, SystemDir}, + {failfun, fun ssh_test_lib:failfun/2}, + {user_passwords, [{?USER, ?PASSWD}]}, + {profile, Profile}]), + + Port = ssh_test_lib:daemon_port(DaemonPid), + Port2 = ssh_test_lib:daemon_port(DaemonPid2), + true = (Port /= Port2), + + ct:pal("~s",[lists:flatten(ssh_info:string())]), + + {ok,[{AccPid,ListenAddr,Port}]} = acceptor_pid(DaemonPid), + {ok,[{AccPid2,ListenAddr,Port2}]} = acceptor_pid(DaemonPid2), + + true = (AccPid /= AccPid2), + + %% Connect first client and check it is alive: + {ok,C1} = ssh:connect("localhost", Port, [{silently_accept_hosts, true}, + {user_interaction, false}, + {user, ?USER}, + {password, ?PASSWD}, + {user_dir, UserDir}]), + [{client_version,_}] = ssh:connection_info(C1,[client_version]), + + %% Make acceptor restart: + exit(AccPid, kill), + %% Check it is a new acceptor: + {ok,[{AccPid1,ListenAddr,Port}]} = acceptor_pid(DaemonPid), + true = (AccPid /= AccPid1), + true = (AccPid2 /= AccPid1), + + %% Connect second client and check it is alive: + {ok,C2} = ssh:connect("localhost", Port, [{silently_accept_hosts, true}, + {user_interaction, false}, + {user, ?USER}, + {password, ?PASSWD}, + {user_dir, UserDir}]), + [{client_version,_}] = ssh:connection_info(C2,[client_version]), + + ct:pal("~s",[lists:flatten(ssh_info:string())]), + + %% Check first client is still alive: + [{client_version,_}] = ssh:connection_info(C1,[client_version]), + + ok = ssh:stop_daemon(DaemonPid2), + timer:sleep(15000), + [{client_version,_}] = ssh:connection_info(C1,[client_version]), + [{client_version,_}] = ssh:connection_info(C2,[client_version]), + + ok = ssh:stop_daemon(DaemonPid), + timer:sleep(15000), + {error,closed} = ssh:connection_info(C1,[client_version]), + {error,closed} = ssh:connection_info(C2,[client_version]). + +%%------------------------------------------------------------------------- +%% Help functions +%%------------------------------------------------------------------------- check_sshd_system_tree(Daemon, Config) -> Host = proplists:get_value(host, Config), Port = proplists:get_value(port, Config), UserDir = proplists:get_value(userdir, Config), {ok, Client} = ssh:connect(Host, Port, [{silently_accept_hosts, true}, - {user_interaction, false}, - {user, ?USER}, {password, ?PASSWD},{user_dir, UserDir}]), + {user_interaction, false}, + {user, ?USER}, + {password, ?PASSWD}, + {user_dir, UserDir}]), ?wait_match([{_,SubSysSup, supervisor,[ssh_subsystem_sup]}, {{ssh_acceptor_sup,_,_,_}, AccSup, supervisor,[ssh_acceptor_sup]}], supervisor:which_children(Daemon), - [SubSysSup,AccSup]), + [SubSysSup,AccSup]), ?wait_match([{{server,ssh_connection_sup, _,_}, ConnectionSup, supervisor, @@ -208,4 +285,33 @@ check_sshd_system_tree(Daemon, Config) -> ?wait_match([{_, _,worker,[ssh_channel]}], supervisor:which_children(ChannelSup)), ssh:close(Client). - + +acceptor_pid(DaemonPid) -> + Parent = self(), + Pid = spawn(fun() -> + Parent ! {self(), supsearch, + [{AccPid,ListenAddr,Port} + + || {{server,ssh_system_sup,ListenAddr,Port,NS}, + DPid,supervisor, + [ssh_system_sup]} <- supervisor:which_children(sshd_sup), + DPid == DaemonPid, + + {{ssh_acceptor_sup,L1,P1,NS1}, + AccSupPid,supervisor, + [ssh_acceptor_sup]} <- supervisor:which_children(DaemonPid), + L1 == ListenAddr, + P1 == Port, + NS1 == NS1, + + {{ssh_acceptor_sup,L2,P2,NS2}, + AccPid,worker, + [ssh_acceptor]} <- supervisor:which_children(AccSupPid), + L2 == ListenAddr, + P2 == Port, + NS2 == NS]} + end), + receive {Pid, supsearch, L} -> {ok,L} + after 2000 -> timeout + end. + diff --git a/lib/ssh/test/ssh_test_lib.erl b/lib/ssh/test/ssh_test_lib.erl index 1673f52821..6186d44890 100644 --- a/lib/ssh/test/ssh_test_lib.erl +++ b/lib/ssh/test/ssh_test_lib.erl @@ -32,15 +32,18 @@ -define(TIMEOUT, 50000). +%%%---------------------------------------------------------------- connect(Port, Options) when is_integer(Port) -> connect(hostname(), Port, Options). connect(any, Port, Options) -> connect(hostname(), Port, Options); connect(Host, Port, Options) -> + ct:log("~p:~p Calling ssh:connect(~p, ~p, ~p)",[?MODULE,?LINE,Host, Port, Options]), {ok, ConnectionRef} = ssh:connect(Host, Port, Options), ConnectionRef. +%%%---------------------------------------------------------------- daemon(Options) -> daemon(any, 0, Options). @@ -53,23 +56,57 @@ daemon(Host, Options) -> daemon(Host, Port, Options) -> ct:log("~p:~p Calling ssh:daemon(~p, ~p, ~p)",[?MODULE,?LINE,Host,Port,Options]), case ssh:daemon(Host, Port, Options) of - {ok, Pid} when Host == any -> - ct:log("ssh:daemon ok (1)",[]), - {Pid, hostname(), daemon_port(Port,Pid)}; {ok, Pid} -> - ct:log("ssh:daemon ok (2)",[]), - {Pid, Host, daemon_port(Port,Pid)}; + {ok,L} = ssh:daemon_info(Pid), + ListenPort = proplists:get_value(port, L), + ListenIP = proplists:get_value(ip, L), + {Pid, ListenIP, ListenPort}; Error -> ct:log("ssh:daemon error ~p",[Error]), Error end. +%%%---------------------------------------------------------------- +daemon_port(Pid) -> daemon_port(0, Pid). + + daemon_port(0, Pid) -> {ok,Dinf} = ssh:daemon_info(Pid), proplists:get_value(port, Dinf); daemon_port(Port, _) -> Port. - +%%%---------------------------------------------------------------- +gen_tcp_connect(Host0, Port, Options) -> + Host = ssh_test_lib:ntoa(ssh_test_lib:mangle_connect_address(Host0)), + ct:log("~p:~p gen_tcp:connect(~p, ~p, ~p)~nHost0 = ~p", + [?MODULE,?LINE, Host, Port, Options, Host0]), + Result = gen_tcp:connect(Host, Port, Options), + ct:log("~p:~p Result = ~p", [?MODULE,?LINE, Result]), + Result. + +%%%---------------------------------------------------------------- +open_sshc(Host0, Port, OptStr) -> + open_sshc(Host0, Port, OptStr, ""). +open_sshc(Host0, Port, OptStr, ExecStr) -> + Cmd = open_sshc_cmd(Host0, Port, OptStr, ExecStr), + Result = os:cmd(Cmd), + ct:log("~p:~p Result = ~p", [?MODULE,?LINE, Result]), + Result. + + +open_sshc_cmd(Host, Port, OptStr) -> + open_sshc_cmd(Host, Port, OptStr, ""). + +open_sshc_cmd(Host0, Port, OptStr, ExecStr) -> + Host = ssh_test_lib:ntoa(ssh_test_lib:mangle_connect_address(Host0)), + Cmd = lists:flatten(["ssh -p ", integer_to_list(Port), + " ", OptStr, + " ", Host, + " ", ExecStr]), + ct:log("~p:~p OpenSSH Cmd = ~p", [?MODULE,?LINE, Cmd]), + Cmd. + +%%%---------------------------------------------------------------- std_daemon(Config, ExtraOpts) -> PrivDir = proplists:get_value(priv_dir, Config), UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth @@ -85,6 +122,7 @@ std_daemon1(Config, ExtraOpts) -> {failfun, fun ssh_test_lib:failfun/2} | ExtraOpts]). +%%%---------------------------------------------------------------- std_connect(Config, Host, Port, ExtraOpts) -> UserDir = proplists:get_value(priv_dir, Config), _ConnectionRef = @@ -95,6 +133,7 @@ std_connect(Config, Host, Port, ExtraOpts) -> {user_interaction, false} | ExtraOpts]). +%%%---------------------------------------------------------------- std_simple_sftp(Host, Port, Config) -> std_simple_sftp(Host, Port, Config, []). @@ -109,6 +148,7 @@ std_simple_sftp(Host, Port, Config, Opts) -> ok = ssh:close(ConnectionRef), Data == ReadData. +%%%---------------------------------------------------------------- std_simple_exec(Host, Port, Config) -> std_simple_exec(Host, Port, Config, []). @@ -135,6 +175,7 @@ std_simple_exec(Host, Port, Config, Opts) -> ct:fail(ExecResult) end. +%%%---------------------------------------------------------------- start_shell(Port, IOServer) -> start_shell(Port, IOServer, []). @@ -149,6 +190,7 @@ start_shell(Port, IOServer, ExtraOptions) -> end). +%%%---------------------------------------------------------------- start_io_server() -> spawn_link(?MODULE, init_io_server, [self()]). @@ -207,8 +249,7 @@ reply(TestCase, Result) -> %%ct:log("reply ~p sending ~p ! ~p",[self(), TestCase, Result]), TestCase ! Result. - - +%%%---------------------------------------------------------------- rcv_expected(Expect, SshPort, Timeout) -> receive {SshPort, Recvd} when is_function(Expect) -> @@ -862,3 +903,73 @@ create_random_dir(Config) -> %% The likelyhood of always generating an existing file name is low create_random_dir(Config) end. + +%%%---------------------------------------------------------------- +match_ip(A, B) -> + R = match_ip0(A,B) orelse match_ip0(B,A), + ct:log("match_ip(~p, ~p) -> ~p",[A, B, R]), + R. + +match_ip0(A, A) -> + true; +match_ip0(any, _) -> + true; +match_ip0(A, B) -> + case match_ip1(A, B) of + true -> + true; + false when is_list(A) -> + case inet:parse_address(A) of + {ok,IPa} -> match_ip0(IPa, B); + _ -> false + end; + false when is_list(B) -> + case inet:parse_address(B) of + {ok,IPb} -> match_ip0(A, IPb); + _ -> false + end; + false -> + false + end. + +match_ip1(any, _) -> true; +match_ip1(loopback, {127,_,_,_}) -> true; +match_ip1({0,0,0,0}, {127,_,_,_}) -> true; +match_ip1(loopback, {0,0,0,0,0,0,0,1}) -> true; +match_ip1({0,0,0,0,0,0,0,0}, {0,0,0,0,0,0,0,1}) -> true; +match_ip1(_, _) -> false. + +%%%---------------------------------------------------------------- +mangle_connect_address(A) -> + mangle_connect_address(A, []). + +mangle_connect_address(A, SockOpts) -> + mangle_connect_address1(A, proplists:get_value(inet6,SockOpts,false)). + +loopback(true) -> {0,0,0,0,0,0,0,1}; +loopback(false) -> {127,0,0,1}. + +mangle_connect_address1( loopback, V6flg) -> loopback(V6flg); +mangle_connect_address1( any, V6flg) -> loopback(V6flg); +mangle_connect_address1({0,0,0,0}, _) -> loopback(false); +mangle_connect_address1({0,0,0,0,0,0,0,0}, _) -> loopback(true); +mangle_connect_address1( IP, _) when is_tuple(IP) -> IP; +mangle_connect_address1(A, _) -> + case catch inet:parse_address(A) of + {ok, {0,0,0,0}} -> loopback(false); + {ok, {0,0,0,0,0,0,0,0}} -> loopback(true); + _ -> A + end. + +%%%---------------------------------------------------------------- +ntoa(A) -> + try inet:ntoa(A) + of + {error,_} when is_atom(A) -> atom_to_list(A); + {error,_} when is_list(A) -> A; + S when is_list(S) -> S + catch + _:_ when is_atom(A) -> atom_to_list(A); + _:_ when is_list(A) -> A + end. + diff --git a/lib/ssh/test/ssh_to_openssh_SUITE.erl b/lib/ssh/test/ssh_to_openssh_SUITE.erl index 7eda009552..35e3ee3edf 100644 --- a/lib/ssh/test/ssh_to_openssh_SUITE.erl +++ b/lib/ssh/test/ssh_to_openssh_SUITE.erl @@ -376,18 +376,18 @@ erlang_server_openssh_client_public_key_rsa(Config) when is_list(Config) -> erlang_server_openssh_client_public_key_X(Config, ssh_rsa). -erlang_server_openssh_client_public_key_X(Config, PubKeyAlg) -> +erlang_server_openssh_client_public_key_X(Config, _PubKeyAlg) -> SystemDir = proplists:get_value(data_dir, Config), PrivDir = proplists:get_value(priv_dir, Config), KnownHosts = filename:join(PrivDir, "known_hosts"), {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, {failfun, fun ssh_test_lib:failfun/2}]), - ct:sleep(500), - Cmd = "ssh -p " ++ integer_to_list(Port) ++ - " -o UserKnownHostsFile=" ++ KnownHosts ++ - " " ++ Host ++ " 1+1.", + Cmd = ssh_test_lib:open_sshc_cmd(Host, Port, + [" -o UserKnownHostsFile=", KnownHosts, + " -o StrictHostKeyChecking=no"], + "1+1."), OpenSsh = ssh_test_lib:open_port({spawn, Cmd}), ssh_test_lib:rcv_expected({data,<<"2\n">>}, OpenSsh, ?TIMEOUT), ssh:stop_daemon(Pid). @@ -395,13 +395,13 @@ erlang_server_openssh_client_public_key_X(Config, PubKeyAlg) -> %%-------------------------------------------------------------------- %% Test that the Erlang/OTP server can renegotiate with openSSH erlang_server_openssh_client_renegotiate(Config) -> - PubKeyAlg = ssh_rsa, + _PubKeyAlg = ssh_rsa, SystemDir = proplists:get_value(data_dir, Config), PrivDir = proplists:get_value(priv_dir, Config), KnownHosts = filename:join(PrivDir, "known_hosts"), {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, - {failfun, fun ssh_test_lib:failfun/2}]), + {failfun, fun ssh_test_lib:failfun/2}]), ct:sleep(500), RenegLimitK = 3, @@ -409,11 +409,13 @@ erlang_server_openssh_client_renegotiate(Config) -> Data = lists:duplicate(trunc(1.1*RenegLimitK*1024), $a), ok = file:write_file(DataFile, Data), - Cmd = "ssh -p " ++ integer_to_list(Port) ++ - " -o UserKnownHostsFile=" ++ KnownHosts ++ - " -o RekeyLimit=" ++ integer_to_list(RenegLimitK) ++"K" ++ - " " ++ Host ++ " < " ++ DataFile, - OpenSsh = ssh_test_lib:open_port({spawn, Cmd}), + Cmd = ssh_test_lib:open_sshc_cmd(Host, Port, + [" -o UserKnownHostsFile=", KnownHosts, + " -o StrictHostKeyChecking=no", + " -o RekeyLimit=",integer_to_list(RenegLimitK),"K"]), + + + OpenSsh = ssh_test_lib:open_port({spawn, Cmd++" < "++DataFile}), Expect = fun({data,R}) -> try @@ -462,7 +464,7 @@ erlang_client_openssh_server_renegotiate(_Config) -> {silently_accept_hosts,true}], group_leader(IO, self()), {ok, ConnRef} = ssh:connect(Host, ?SSH_DEFAULT_PORT, Options), - ct:pal("Parent = ~p, IO = ~p, Shell = ~p, ConnRef = ~p~n",[Parent, IO, self(), ConnRef]), + ct:log("Parent = ~p, IO = ~p, Shell = ~p, ConnRef = ~p~n",[Parent, IO, self(), ConnRef]), case ssh_connection:session_channel(ConnRef, infinity) of {ok,ChannelId} -> success = ssh_connection:ptty_alloc(ConnRef, ChannelId, []), diff --git a/lib/ssh/test/ssh_trpt_test_lib.erl b/lib/ssh/test/ssh_trpt_test_lib.erl index 261239c152..e1f4c65300 100644 --- a/lib/ssh/test/ssh_trpt_test_lib.erl +++ b/lib/ssh/test/ssh_trpt_test_lib.erl @@ -314,8 +314,7 @@ mangle_opts(Options) -> lists:keydelete(K,1,Opts) end, Options, SysOpts). -host({0,0,0,0}) -> "localhost"; -host(H) -> H. +host(H) -> ssh_test_lib:ntoa(ssh_test_lib:mangle_connect_address(H)). %%%---------------------------------------------------------------- send(S=#s{ssh=C}, hello) -> diff --git a/lib/ssh/vsn.mk b/lib/ssh/vsn.mk index 96c83cb0f7..48332d2e5a 100644 --- a/lib/ssh/vsn.mk +++ b/lib/ssh/vsn.mk @@ -1,5 +1,5 @@ #-*-makefile-*- ; force emacs to enter makefile-mode -SSH_VSN = 4.4.1 +SSH_VSN = 4.4.2 APP_VSN = "ssh-$(SSH_VSN)" diff --git a/lib/ssl/doc/src/notes.xml b/lib/ssl/doc/src/notes.xml index d3ab3e9216..7ffb9c0e88 100644 --- a/lib/ssl/doc/src/notes.xml +++ b/lib/ssl/doc/src/notes.xml @@ -28,6 +28,24 @@ <p>This document describes the changes made to the SSL application.</p> +<section><title>SSL 8.1.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Correct active once emulation, for TLS. Now all data + received by the connection process will be delivered + through active once, even when the active once arrives + after that the gen_tcp socket is closed by the peer.</p> + <p> + Own Id: OTP-14300</p> + </item> + </list> + </section> + +</section> + <section><title>SSL 8.1.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl index f607c86ae3..16e0df2101 100644 --- a/lib/ssl/src/dtls_connection.erl +++ b/lib/ssl/src/dtls_connection.erl @@ -263,9 +263,14 @@ init({call, From}, {start, Timeout}, {Record, State} = next_record(State3), next_event(hello, Record, State, Actions); init({call, _} = Type, Event, #state{role = server, transport_cb = gen_udp} = State) -> - ssl_connection:init(Type, Event, - State#state{flight_state = {retransmit, ?INITIAL_RETRANSMIT_TIMEOUT}}, - ?MODULE); + Result = ssl_connection:init(Type, Event, + State#state{flight_state = {retransmit, ?INITIAL_RETRANSMIT_TIMEOUT}, + protocol_specific = #{current_cookie_secret => dtls_v1:cookie_secret(), + previous_cookie_secret => <<>>}}, + ?MODULE), + erlang:send_after(dtls_v1:cookie_timeout(), self(), new_cookie_secret), + Result; + init({call, _} = Type, Event, #state{role = server} = State) -> %% I.E. DTLS over sctp ssl_connection:init(Type, Event, State#state{flight_state = reliable}, ?MODULE); @@ -288,10 +293,10 @@ error(_, _, _) -> hello(internal, #client_hello{cookie = <<>>, client_version = Version} = Hello, #state{role = server, transport_cb = Transport, - socket = Socket} = State0) -> - %% TODO: not hard code key + socket = Socket, + protocol_specific = #{current_cookie_secret := Secret}} = State0) -> {ok, {IP, Port}} = dtls_socket:peername(Transport, Socket), - Cookie = dtls_handshake:cookie(<<"secret">>, IP, Port, Hello), + Cookie = dtls_handshake:cookie(Secret, IP, Port, Hello), VerifyRequest = dtls_handshake:hello_verify_request(Cookie, Version), State1 = prepare_flight(State0#state{negotiated_version = Version}), {State2, Actions} = send_handshake(VerifyRequest, State1), @@ -299,15 +304,21 @@ hello(internal, #client_hello{cookie = <<>>, next_event(hello, Record, State#state{tls_handshake_history = ssl_handshake:init_handshake_history()}, Actions); hello(internal, #client_hello{cookie = Cookie} = Hello, #state{role = server, transport_cb = Transport, - socket = Socket} = State0) -> + socket = Socket, + protocol_specific = #{current_cookie_secret := Secret, + previous_cookie_secret := PSecret}} = State0) -> {ok, {IP, Port}} = dtls_socket:peername(Transport, Socket), - %% TODO: not hard code key - case dtls_handshake:cookie(<<"secret">>, IP, Port, Hello) of + case dtls_handshake:cookie(Secret, IP, Port, Hello) of Cookie -> handle_client_hello(Hello, State0); _ -> - %% Handle bad cookie as new cookie request RFC 6347 4.1.2 - hello(internal, Hello#client_hello{cookie = <<>>}, State0) + case dtls_handshake:cookie(PSecret, IP, Port, Hello) of + Cookie -> + handle_client_hello(Hello, State0); + _ -> + %% Handle bad cookie as new cookie request RFC 6347 4.1.2 + hello(internal, Hello#client_hello{cookie = <<>>}, State0) + end end; hello(internal, #hello_verify_request{cookie = Cookie}, #state{role = client, host = Host, port = Port, @@ -471,9 +482,14 @@ handle_info({CloseTag, Socket}, StateName, end, ssl_connection:handle_normal_shutdown(?ALERT_REC(?FATAL, ?CLOSE_NOTIFY), StateName, State), {stop, {shutdown, transport_closed}}; +handle_info(new_cookie_secret, StateName, #state{protocol_specific = #{cookie_secret := Secret} = CookieInfo} = State) -> + erlang:send_after(dtls_v1:cookie_timeout(), self(), new_cookie_secret), + {next_state, StateName, State#state{protocol_specific = CookieInfo#{cookie_secret => dtls_v1:cookie_secret(), + previous_cookie_secret => Secret}}}; handle_info(Msg, StateName, State) -> ssl_connection:handle_info(Msg, StateName, State). + handle_call(Event, From, StateName, State) -> ssl_connection:handle_call(Event, From, StateName, State, ?MODULE). diff --git a/lib/ssl/src/dtls_handshake.erl b/lib/ssl/src/dtls_handshake.erl index 4c525fae1b..d3ba90a226 100644 --- a/lib/ssl/src/dtls_handshake.erl +++ b/lib/ssl/src/dtls_handshake.erl @@ -474,14 +474,12 @@ merge_fragments(#handshake_fragment{ fragment = <<PreviousData/binary, Data/binary>>}; %% already fully contained fragment merge_fragments(#handshake_fragment{ - fragment_offset = PreviousOffSet, - fragment_length = PreviousLen, - fragment = PreviousData - } = Previous, - #handshake_fragment{ - fragment_offset = CurrentOffSet, - fragment_length = CurrentLen, - fragment = CurrentData}) + fragment_offset = PreviousOffSet, + fragment_length = PreviousLen + } = Previous, + #handshake_fragment{ + fragment_offset = CurrentOffSet, + fragment_length = CurrentLen}) when PreviousOffSet + PreviousLen >= CurrentOffSet andalso PreviousOffSet + PreviousLen >= CurrentOffSet + CurrentLen -> Previous; diff --git a/lib/ssl/src/dtls_v1.erl b/lib/ssl/src/dtls_v1.erl index dd0d35d404..4aaf8baa6c 100644 --- a/lib/ssl/src/dtls_v1.erl +++ b/lib/ssl/src/dtls_v1.erl @@ -22,7 +22,10 @@ -include("ssl_cipher.hrl"). -export([suites/1, all_suites/1, mac_hash/7, ecc_curves/1, - corresponding_tls_version/1, corresponding_dtls_version/1]). + corresponding_tls_version/1, corresponding_dtls_version/1, + cookie_secret/0, cookie_timeout/0]). + +-define(COOKIE_BASE_TIMEOUT, 30000). -spec suites(Minor:: 253|255) -> [ssl_cipher:cipher_suite()]. @@ -47,6 +50,13 @@ ecc_curves({_Major, Minor}) -> corresponding_tls_version({254, Minor}) -> {3, corresponding_minor_tls_version(Minor)}. +cookie_secret() -> + crypto:strong_rand_bytes(32). + +cookie_timeout() -> + %% Cookie will live for two timeouts periods + round(rand:uniform() * ?COOKIE_BASE_TIMEOUT/2). + corresponding_minor_tls_version(255) -> 2; corresponding_minor_tls_version(253) -> diff --git a/lib/ssl/src/ssl.appup.src b/lib/ssl/src/ssl.appup.src index bfdd0c205b..2eda9d9491 100644 --- a/lib/ssl/src/ssl.appup.src +++ b/lib/ssl/src/ssl.appup.src @@ -1,6 +1,7 @@ %% -*- erlang -*- {"%VSN%", [ + {<<"8.1.1">>, [{load_module, tls_connection, soft_purge, soft_purge, []}]}, {<<"8\\..*">>, [{restart_application, ssl}]}, {<<"7\\..*">>, [{restart_application, ssl}]}, {<<"6\\..*">>, [{restart_application, ssl}]}, @@ -9,6 +10,7 @@ {<<"3\\..*">>, [{restart_application, ssl}]} ], [ + {<<"8.1.1">>, [{load_module, tls_connection, soft_purge, soft_purge, []}]}, {<<"8\\..*">>, [{restart_application, ssl}]}, {<<"7\\..*">>, [{restart_application, ssl}]}, {<<"6\\..*">>, [{restart_application, ssl}]}, diff --git a/lib/ssl/src/ssl_connection.hrl b/lib/ssl/src/ssl_connection.hrl index b597c059af..368eaf6090 100644 --- a/lib/ssl/src/ssl_connection.hrl +++ b/lib/ssl/src/ssl_connection.hrl @@ -91,7 +91,8 @@ %% underlaying packet format. Introduced by DTLS - RFC 4347. %% The mecahnism is also usefull in TLS although we do not %% need to worry about packet loss in TLS. In DTLS we need to track DTLS handshake seqnr - flight_state = reliable %% reliable | {retransmit, integer()}| {waiting, ref(), integer()} - last two is used in DTLS over udp. + flight_state = reliable, %% reliable | {retransmit, integer()}| {waiting, ref(), integer()} - last two is used in DTLS over udp. + protocol_specific = #{} :: map() }). -define(DEFAULT_DIFFIE_HELLMAN_PARAMS, #'DHParameter'{prime = ?DEFAULT_DIFFIE_HELLMAN_PRIME, diff --git a/lib/ssl/src/ssl_internal.hrl b/lib/ssl/src/ssl_internal.hrl index c10ec3a2d6..0fbaa82b6a 100644 --- a/lib/ssl/src/ssl_internal.hrl +++ b/lib/ssl/src/ssl_internal.hrl @@ -144,7 +144,7 @@ honor_ecc_order :: boolean(), v2_hello_compatible :: boolean(), max_handshake_size :: integer() - }). + }). -record(socket_options, { diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index c6e530e164..bda6bf0349 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -397,23 +397,36 @@ handle_info({Protocol, _, Data}, StateName, end; handle_info({CloseTag, Socket}, StateName, #state{socket = Socket, close_tag = CloseTag, + socket_options = #socket_options{active = Active}, + protocol_buffers = #protocol_buffers{tls_cipher_texts = CTs}, negotiated_version = Version} = State) -> + %% Note that as of TLS 1.1, %% failure to properly close a connection no longer requires that a %% session not be resumed. This is a change from TLS 1.0 to conform %% with widespread implementation practice. - case Version of - {1, N} when N >= 1 -> - ok; - _ -> - %% As invalidate_sessions here causes performance issues, - %% we will conform to the widespread implementation - %% practice and go aginst the spec - %%invalidate_session(Role, Host, Port, Session) - ok - end, - ssl_connection:handle_normal_shutdown(?ALERT_REC(?FATAL, ?CLOSE_NOTIFY), StateName, State), - {stop, {shutdown, transport_closed}}; + + case (Active == false) andalso (CTs =/= []) of + false -> + case Version of + {1, N} when N >= 1 -> + ok; + _ -> + %% As invalidate_sessions here causes performance issues, + %% we will conform to the widespread implementation + %% practice and go aginst the spec + %%invalidate_session(Role, Host, Port, Session) + ok + end, + + ssl_connection:handle_normal_shutdown(?ALERT_REC(?FATAL, ?CLOSE_NOTIFY), StateName, State), + {stop, {shutdown, transport_closed}}; + true -> + %% Fixes non-delivery of final TLS record in {active, once}. + %% Basically allows the application the opportunity to set {active, once} again + %% and then receive the final message. + next_event(StateName, no_record, State) + end; handle_info(Msg, StateName, State) -> ssl_connection:handle_info(Msg, StateName, State). diff --git a/lib/ssl/vsn.mk b/lib/ssl/vsn.mk index 415a47949d..82184f5c74 100644 --- a/lib/ssl/vsn.mk +++ b/lib/ssl/vsn.mk @@ -1 +1 @@ -SSL_VSN = 8.1.1 +SSL_VSN = 8.1.2 diff --git a/lib/stdlib/doc/src/io_lib.xml b/lib/stdlib/doc/src/io_lib.xml index 931e50f6f2..5ae400da62 100644 --- a/lib/stdlib/doc/src/io_lib.xml +++ b/lib/stdlib/doc/src/io_lib.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2016</year> + <year>1996</year><year>2017</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -147,7 +147,7 @@ format string (that is, <c>~ts</c> or <c>~tc</c>), the resulting list can contain characters beyond the ISO Latin-1 character range (that is, numbers > 255). If so, the - result is not an ordinary Erlang <c>string()</c>, but can well be + result is still an ordinary Erlang <c>string()</c>, and can well be used in any context where Unicode data is allowed.</p> </desc> </func> @@ -384,6 +384,16 @@ </func> <func> + <name name="write_atom_as_latin1" arity="1"/> + <fsummary>Write an atom.</fsummary> + <desc> + <p>Returns the list of characters needed to print atom + <c><anno>Atom</anno></c>. Non-Latin-1 characters + are escaped.</p> + </desc> + </func> + + <func> <name name="write_char" arity="1"/> <fsummary>Write a character.</fsummary> <desc> diff --git a/lib/stdlib/doc/src/rand.xml b/lib/stdlib/doc/src/rand.xml index 8745e16908..2ddf3021ac 100644 --- a/lib/stdlib/doc/src/rand.xml +++ b/lib/stdlib/doc/src/rand.xml @@ -120,27 +120,50 @@ S0 = rand:seed_s(exsplus), {SND0, S2} = rand:normal_s(S1),</pre> <note> - <p>This random number generator is not cryptographically - strong. If a strong cryptographic random number generator is - needed, use one of functions in the - <seealso marker="crypto:crypto"><c>crypto</c></seealso> - module, for example, <seealso marker="crypto:crypto"> - <c>crypto:strong_rand_bytes/1</c></seealso>.</p> + <p>The builtin random number generator algorithms are not + cryptographically strong. If a cryptographically strong + random number generator is needed, use something like + <seealso marker="crypto:crypto#rand_seed-0"><c>crypto:rand_seed/0</c></seealso>. + </p> </note> </description> <datatypes> <datatype> + <name name="builtin_alg"/> + </datatype> + <datatype> <name name="alg"/> </datatype> <datatype> + <name name="alg_handler"/> + </datatype> + <datatype> + <name name="alg_state"/> + </datatype> + <datatype> + <name name="exs64_state"/> + <desc><p>Algorithm specific internal state</p></desc> + </datatype> + <datatype> + <name name="exsplus_state"/> + <desc><p>Algorithm specific internal state</p></desc> + </datatype> + <datatype> + <name name="exs1024_state"/> + <desc><p>Algorithm specific internal state</p></desc> + </datatype> + <datatype> <name name="state"/> <desc><p>Algorithm-dependent state.</p></desc> </datatype> <datatype> <name name="export_state"/> - <desc><p>Algorithm-dependent state that can be printed or saved to - file.</p></desc> + <desc> + <p> + Algorithm-dependent state that can be printed or saved to file. + </p> + </desc> </datatype> </datatypes> @@ -215,8 +238,11 @@ S0 = rand:seed_s(exsplus), <fsummary>Seed random number generator.</fsummary> <desc> <marker id="seed-1"/> - <p>Seeds random number generation with the specifed algorithm and - time-dependent data if <anno>AlgOrExpState</anno> is an algorithm.</p> + <p> + Seeds random number generation with the specifed algorithm and + time-dependent data if <c><anno>AlgOrStateOrExpState</anno></c> + is an algorithm. + </p> <p>Otherwise recreates the exported seed in the process dictionary, and returns the state. See also <seealso marker="#export_seed-0"><c>export_seed/0</c></seealso>.</p> @@ -236,8 +262,11 @@ S0 = rand:seed_s(exsplus), <name name="seed_s" arity="1"/> <fsummary>Seed random number generator.</fsummary> <desc> - <p>Seeds random number generation with the specifed algorithm and - time-dependent data if <anno>AlgOrExpState</anno> is an algorithm.</p> + <p> + Seeds random number generation with the specifed algorithm and + time-dependent data if <c><anno>AlgOrStateOrExpState</anno></c> + is an algorithm. + </p> <p>Otherwise recreates the exported seed and returns the state. See also <seealso marker="#export_seed-0"> <c>export_seed/0</c></seealso>.</p> @@ -258,7 +287,7 @@ S0 = rand:seed_s(exsplus), <fsummary>Return a random float.</fsummary> <desc><marker id="uniform-0"/> <p>Returns a random float uniformly distributed in the value - range <c>0.0 < <anno>X</anno> < 1.0</c> and + range <c>0.0 =< <anno>X</anno> < 1.0</c> and updates the state in the process dictionary.</p> </desc> </func> @@ -269,7 +298,7 @@ S0 = rand:seed_s(exsplus), <desc><marker id="uniform-1"/> <p>Returns, for a specified integer <c><anno>N</anno> >= 1</c>, a random integer uniformly distributed in the value range - <c>1 <= <anno>X</anno> <= <anno>N</anno></c> and + <c>1 =< <anno>X</anno> =< <anno>N</anno></c> and updates the state in the process dictionary.</p> </desc> </func> @@ -279,7 +308,7 @@ S0 = rand:seed_s(exsplus), <fsummary>Return a random float.</fsummary> <desc> <p>Returns, for a specified state, random float - uniformly distributed in the value range <c>0.0 < + uniformly distributed in the value range <c>0.0 =< <anno>X</anno> < 1.0</c> and a new state.</p> </desc> </func> @@ -290,7 +319,7 @@ S0 = rand:seed_s(exsplus), <desc> <p>Returns, for a specified integer <c><anno>N</anno> >= 1</c> and a state, a random integer uniformly distributed in the value - range <c>1 <= <anno>X</anno> <= <anno>N</anno></c> and a + range <c>1 =< <anno>X</anno> =< <anno>N</anno></c> and a new state.</p> </desc> </func> diff --git a/lib/stdlib/doc/src/shell.xml b/lib/stdlib/doc/src/shell.xml index f52bc39deb..ab62c2fcdd 100644 --- a/lib/stdlib/doc/src/shell.xml +++ b/lib/stdlib/doc/src/shell.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2016</year> + <year>1996</year><year>2017</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -854,7 +854,7 @@ q - quit erlang <c>{history, N}</c>, where <c>N</c> is the current command number. The function is to return a list of characters or an atom. This constraint is because of the Erlang I/O protocol. Unicode characters - beyond code point 255 are allowed in the list. Notice + beyond code point 255 are allowed in the list and the atom. Notice that in restricted mode the call <c>Mod:Func(L)</c> must be allowed or the default shell prompt function is called.</p> </section> diff --git a/lib/stdlib/src/erl_pp.erl b/lib/stdlib/src/erl_pp.erl index 6068afb293..ee5e7a11bf 100644 --- a/lib/stdlib/src/erl_pp.erl +++ b/lib/stdlib/src/erl_pp.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -44,7 +44,7 @@ | {encoding, latin1 | unicode | utf8}). -type(options() :: hook_function() | [option()]). --record(pp, {string_fun, char_fun}). +-record(pp, {value_fun, string_fun, char_fun}). -record(options, {hook, encoding, opts}). @@ -214,11 +214,15 @@ state(_Hook) -> state(). state() -> - #pp{string_fun = fun io_lib:write_string_as_latin1/1, + Options = [{encoding,latin1}], + #pp{value_fun = fun(V) -> io_lib_pretty:print(V, Options) end, + string_fun = fun io_lib:write_string_as_latin1/1, char_fun = fun io_lib:write_char_as_latin1/1}. unicode_state() -> - #pp{string_fun = fun io_lib:write_string/1, + Options = [{encoding,unicode}], + #pp{value_fun = fun(V) -> io_lib_pretty:print(V, Options) end, + string_fun = fun io_lib:write_string/1, char_fun = fun io_lib:write_char/1}. encoding(Options) -> @@ -253,31 +257,30 @@ lattribute({attribute,_Line,Name,Arg}, Opts) -> lattribute(module, {M,Vs}, _Opts) -> A = a0(), - attr("module",[{var,A,pname(M)}, - foldr(fun(V, C) -> {cons,A,{var,A,V},C} - end, {nil,A}, Vs)]); + attr(module,[{var,A,pname(M)}, + foldr(fun(V, C) -> {cons,A,{var,A,V},C} + end, {nil,A}, Vs)]); lattribute(module, M, _Opts) -> - attr("module", [{var,a0(),pname(M)}]); + attr(module, [{var,a0(),pname(M)}]); lattribute(export, Falist, _Opts) -> - call({var,a0(),"-export"}, [falist(Falist)], 0, options(none)); + attrib(export, falist(Falist)); lattribute(import, Name, _Opts) when is_list(Name) -> - attr("import", [{var,a0(),pname(Name)}]); + attr(import, [{var,a0(),pname(Name)}]); lattribute(import, {From,Falist}, _Opts) -> - attr("import",[{var,a0(),pname(From)},falist(Falist)]); + attrib(import, [leaf(pname(From)),falist(Falist)]); lattribute(export_type, Talist, _Opts) -> - call({var,a0(),"-export_type"}, [falist(Talist)], 0, options(none)); + attrib(export_type, falist(Talist)); lattribute(optional_callbacks, Falist, Opts) -> - ArgL = try falist(Falist) - catch _:_ -> abstract(Falist, Opts) - end, - call({var,a0(),"-optional_callbacks"}, [ArgL], 0, options(none)); + try attrib(optional_callbacks, falist(Falist)) + catch _:_ -> attr(optional_callbacks, [abstract(Falist, Opts)]) + end; lattribute(file, {Name,Line}, _Opts) -> - attr("file", [{string,a0(),Name},{integer,a0(),Line}]); + attr(file, [{string,a0(),Name},{integer,a0(),Line}]); lattribute(record, {Name,Is}, Opts) -> - Nl = leaf(format("-record(~w,", [Name])), + Nl = [leaf("-record("),{atom,Name},$,], [{first,Nl,record_fields(Is, Opts)},$)]; lattribute(Name, Arg, Options) -> - attr(write(Name), [abstract(Arg, Options)]). + attr(Name, [abstract(Arg, Options)]). abstract(Arg, #options{encoding = Encoding}) -> erl_parse:abstract(Arg, [{encoding,Encoding}]). @@ -340,7 +343,7 @@ ltype({user_type,Line,T,Ts}, _) -> ltype({remote_type,Line,[M,F,Ts]}, _) -> simple_type({remote,Line,M,F}, Ts); ltype({atom,_,T}, _) -> - leaf(write(T)); + {atom,T}; ltype(E, P) -> lexpr(E, P, options(none)). @@ -382,12 +385,12 @@ tuple_type(Ts, F) -> specattr(SpecKind, {FuncSpec,TypeSpecs}) -> Func = case FuncSpec of {F,_A} -> - format("~w", [F]); + {atom,F}; {M,F,_A} -> - format("~w:~w", [M, F]) + [{atom,M},$:,{atom,F}] end, {first,leaf(lists:concat(["-", SpecKind, " "])), - {list,[{first,leaf(Func),spec_clauses(TypeSpecs)}]}}. + {list,[{first,Func,spec_clauses(TypeSpecs)}]}}. spec_clauses(TypeSpecs) -> {prefer_nl,[$;],[sig_type(T) || T <- TypeSpecs]}. @@ -429,7 +432,10 @@ ltypes(Ts, F, Prec) -> [F(T, Prec) || T <- Ts]. attr(Name, Args) -> - call({var,a0(),format("-~s", [Name])}, Args, 0, options(none)). + {first,[$-,{atom,Name}],args(Args, options(none))}. + +attrib(Name, Args) -> + {first,[$-,{atom,Name}],[{seq,$(,$),[$,],Args}]}. pname(['' | As]) -> [$. | pname(As)]; @@ -441,10 +447,13 @@ pname(A) when is_atom(A) -> write(A). falist([]) -> - {nil,a0()}; -falist([{Name,Arity}|Falist]) -> - A = a0(), - {cons,A,{var,A,format("~w/~w", [Name,Arity])},falist(Falist)}. + [leaf("[]")]; +falist(Falist) -> + L = [begin + {Name,Arity} = Fa, + [{atom,Name},leaf(format("/~w", [Arity]))] + end || Fa <- Falist], + [{seq,$[,$],$,,L}]. lfunction({function,_Line,Name,_Arity,Cs}, Opts) -> Cll = nl_clauses(fun (C, H) -> func_clause(Name, C, H) end, $;, Opts, Cs), @@ -489,7 +498,7 @@ lexpr({var,_,V}, _, _) -> leaf(format("~ts", [V])); lexpr({char,_,C}, _, _) -> {char,C}; lexpr({integer,_,N}, _, _) -> leaf(write(N)); lexpr({float,_,F}, _, _) -> leaf(write(F)); -lexpr({atom,_,A}, _, _) -> leaf(write(A)); +lexpr({atom,_,A}, _, _) -> {atom,A}; lexpr({string,_,S}, _, _) -> {string,S}; lexpr({nil,_}, _, _) -> '[]'; lexpr({cons,_,H,T}, _, Opts) -> @@ -519,7 +528,7 @@ lexpr({record, _, Name, Fs}, Prec, Opts) -> lexpr({record_field, _, Rec, Name, F}, Prec, Opts) -> {L,P,R} = inop_prec('#'), Rl = lexpr(Rec, L, Opts), - Nl = leaf(format("#~w.", [Name])), + Nl = [$#,{atom,Name},$.], El = [Rl,Nl,lexpr(F, R, Opts)], maybe_paren(P, Prec, El); lexpr({record, _, Rec, Name, Fs}, Prec, Opts) -> @@ -538,12 +547,12 @@ lexpr({record_field, _, Rec, F}, Prec, Opts) -> maybe_paren(P, Prec, El); lexpr({map, _, Fs}, Prec, Opts) -> {P,_R} = preop_prec('#'), - El = {first,leaf("#"),map_fields(Fs, Opts)}, + El = {first,$#,map_fields(Fs, Opts)}, maybe_paren(P, Prec, El); lexpr({map, _, Map, Fs}, Prec, Opts) -> {L,P,_R} = inop_prec('#'), Rl = lexpr(Map, L, Opts), - El = {first,[Rl,leaf("#")],map_fields(Fs, Opts)}, + El = {first,[Rl,$#],map_fields(Fs, Opts)}, maybe_paren(P, Prec, El); lexpr({block,_,Es}, _, Opts) -> {list,[{step,'begin',body(Es, Opts)},'end']}; @@ -563,13 +572,16 @@ lexpr({'receive',_,Cs,To,ToOpt}, _, Opts) -> {step,'after',Al}, 'end']}; lexpr({'fun',_,{function,F,A}}, _Prec, _Opts) -> - leaf(format("fun ~w/~w", [F,A])); -lexpr({'fun',_,{function,F,A},Extra}, _Prec, _Opts) -> - {force_nl,fun_info(Extra),leaf(format("fun ~w/~w", [F,A]))}; -lexpr({'fun',_,{function,M,F,A}}, _Prec, _Opts) + [leaf("fun "),{atom,F},leaf(format("/~w", [A]))]; +lexpr({'fun',L,{function,_,_}=Func,Extra}, Prec, Opts) -> + {force_nl,fun_info(Extra),lexpr({'fun',L,Func}, Prec, Opts)}; +lexpr({'fun',L,{function,M,F,A}}, Prec, Opts) when is_atom(M), is_atom(F), is_integer(A) -> %% For backward compatibility with pre-R15 abstract format. - leaf(format("fun ~w:~w/~w", [M,F,A])); + Mod = erl_parse:abstract(M), + Fun = erl_parse:abstract(F), + Arity = erl_parse:abstract(A), + lexpr({'fun',L,{function,Mod,Fun,Arity}}, Prec, Opts); lexpr({'fun',_,{function,M,F,A}}, _Prec, Opts) -> %% New format in R15. NameItem = lexpr(M, Opts), @@ -660,7 +672,7 @@ lexpr({bin,_,Fs}, _, Opts) -> bit_grp(Fs, Opts); %% Special case for straight values. lexpr({value,_,Val}, _,_) -> - leaf(write(Val)); + {value,Val}; %% Now do the hook. lexpr(Other, _Precedence, #options{hook = none}) -> leaf(format("INVALID-FORM:~w:",[Other])); @@ -676,7 +688,7 @@ call(Name, Args, Prec, Opts) -> maybe_paren(P, Prec, Item). fun_info(Extra) -> - leaf(format("% fun-info: ~w", [Extra])). + [leaf("% fun-info: "),{value,Extra}]. %% BITS: @@ -717,7 +729,7 @@ bit_elem_type(T) -> %% end of BITS record_name(Name) -> - leaf(format("#~w", [Name])). + [$#,{atom,Name}]. record_fields(Fs, Opts) -> tuple(Fs, fun record_field/2, Opts). @@ -919,8 +931,10 @@ frmt(Item, I, PP) -> %%% - {force_nl,ExtraInfo,I}: fun-info (a comment) forces linebreak before I. %%% - {prefer_nl,Sep,IPs}: forces linebreak between Is unlesss negative %%% indentation. +%%% - {atom,A}: an atom %%% - {char,C}: a character %%% - {string,S}: a string. +%%% - {value,T}: a term. %%% - {hook,...}, {ehook,...}: hook expressions. %%% %%% list, first, seq, force_nl, and prefer_nl all accept IPs, where each @@ -981,6 +995,10 @@ f({prefer_nl,Sep,LItems}, I0, ST, WT, PP) -> true -> {insert_newlines(CharsSize2L, I0, ST),nsz(lists:last(Sizes), I0)} end; +f({value,V}, I, ST, WT, PP) -> + f(write_a_value(V, PP), I, ST, WT, PP); +f({atom,A}, I, ST, WT, PP) -> + f(write_an_atom(A, PP), I, ST, WT, PP); f({char,C}, I, ST, WT, PP) -> f(write_a_char(C, PP), I, ST, WT, PP); f({string,S}, I, ST, WT, PP) -> @@ -1119,6 +1137,12 @@ has_nl([C|Cs]) -> has_nl([]) -> false. +write_a_value(V, PP) -> + flat_leaf(write_value(V, PP)). + +write_an_atom(A, PP) -> + flat_leaf(write_atom(A, PP)). + write_a_char(C, PP) -> flat_leaf(write_char(C, PP)). @@ -1135,7 +1159,7 @@ write_a_string([], _N, _Len, _PP) -> write_a_string(S, N, Len, PP) -> SS = string:sub_string(S, 1, N), Sl = write_string(SS, PP), - case (length(Sl) > Len) and (N > ?MIN_SUBSTRING) of + case (chars_size(Sl) > Len) and (N > ?MIN_SUBSTRING) of true -> write_a_string(S, N-1, Len, PP); false -> @@ -1147,11 +1171,17 @@ flat_leaf(S) -> L = lists:flatten(S), {leaf,length(L),L}. +write_value(V, PP) -> + (PP#pp.value_fun)(V). + +write_atom(A, PP) -> + (PP#pp.value_fun)(A). + write_string(S, PP) -> - lists:flatten((PP#pp.string_fun)(S)). + (PP#pp.string_fun)(S). write_char(C, PP) -> - lists:flatten((PP#pp.char_fun)(C)). + (PP#pp.char_fun)(C). %% %% Utilities diff --git a/lib/stdlib/src/io_lib.erl b/lib/stdlib/src/io_lib.erl index a91143a764..28e5007e5a 100644 --- a/lib/stdlib/src/io_lib.erl +++ b/lib/stdlib/src/io_lib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -68,8 +68,8 @@ -export([write_atom/1,write_string/1,write_string/2,write_latin1_string/1, write_latin1_string/2, write_char/1, write_latin1_char/1]). --export([write_string_as_latin1/1, write_string_as_latin1/2, - write_char_as_latin1/1]). +-export([write_atom_as_latin1/1, write_string_as_latin1/1, + write_string_as_latin1/2, write_char_as_latin1/1]). -export([quote_atom/2, char_list/1, latin1_char_list/1, deep_char_list/1, deep_latin1_char_list/1, @@ -344,6 +344,11 @@ write_binary_body(B, _D) -> <<X:L>> = B, [integer_to_list(X),$:,integer_to_list(L)]. +%%% There are two functions to write Unicode atoms: +%%% - they both escape control characters < 160; +%%% - write_atom() never escapes characters >= 160; +%%% - write_atom_as_latin1() also escapes characters >= 255. + %% write_atom(Atom) -> [Char] %% Generate the list of characters needed to print an atom. @@ -351,17 +356,26 @@ write_binary_body(B, _D) -> Atom :: atom(). write_atom(Atom) -> + write_possibly_quoted_atom(Atom, fun write_string/2). + +-spec write_atom_as_latin1(Atom) -> latin1_string() when + Atom :: atom(). + +write_atom_as_latin1(Atom) -> + write_possibly_quoted_atom(Atom, fun write_string_as_latin1/2). + +write_possibly_quoted_atom(Atom, PFun) -> Chars = atom_to_list(Atom), case quote_atom(Atom, Chars) of true -> - write_string(Chars, $'); %' + PFun(Chars, $'); %' false -> Chars end. %% quote_atom(Atom, CharList) %% Return 'true' if atom with chars in CharList needs to be quoted, else -%% return 'false'. +%% return 'false'. Notice that characters >= 160 are always quoted. -spec quote_atom(atom(), chars()) -> boolean(). diff --git a/lib/stdlib/src/io_lib_pretty.erl b/lib/stdlib/src/io_lib_pretty.erl index aabccfc5d9..ff368d02da 100644 --- a/lib/stdlib/src/io_lib_pretty.erl +++ b/lib/stdlib/src/io_lib_pretty.erl @@ -105,6 +105,8 @@ print(_, _, _, 0, _M, _RF, _Enc, _Str) -> "..."; print(Term, Col, Ll, D, M, RecDefFun, Enc, Str) when Col =< 0 -> %% ensure Col is at least 1 print(Term, 1, Ll, D, M, RecDefFun, Enc, Str); +print(Atom, _Col, _Ll, _D, _M, _RF, Enc, _Str) when is_atom(Atom) -> + write_atom(Atom, Enc); print(Term, Col, Ll, D, M0, RecDefFun, Enc, Str) when is_tuple(Term); is_list(Term); is_map(Term); @@ -407,6 +409,9 @@ print_length({}, _D, _RF, _Enc, _Str) -> {"{}", 2}; print_length(#{}=M, _D, _RF, _Enc, _Str) when map_size(M) =:= 0 -> {"#{}", 3}; +print_length(Atom, _D, _RF, Enc, _Str) when is_atom(Atom) -> + S = write_atom(Atom, Enc), + {S, lists:flatlength(S)}; print_length(List, D, RF, Enc, Str) when is_list(List) -> %% only flat lists are "printable" case Str andalso printable_list(List, D, Enc) of @@ -500,7 +505,7 @@ print_length_tuple(Tuple, D, RF, Enc, Str) -> print_length_record(_Tuple, 1, _RF, _RDefs, _Enc, _Str) -> {"{...}", 5}; print_length_record(Tuple, D, RF, RDefs, Enc, Str) -> - Name = [$# | io_lib:write_atom(element(1, Tuple))], + Name = [$# | write_atom(element(1, Tuple), Enc)], NameL = length(Name), Elements = tl(tuple_to_list(Tuple)), L = print_length_fields(RDefs, D - 1, Elements, RF, Enc, Str), @@ -515,7 +520,7 @@ print_length_fields([Def | Defs], D, [E | Es], RF, Enc, Str) -> print_length_fields(Defs, D - 1, Es, RF, Enc, Str)]. print_length_field(Def, D, E, RF, Enc, Str) -> - Name = io_lib:write_atom(Def), + Name = write_atom(Def, Enc), {S, L} = print_length(E, D, RF, Enc, Str), NameL = length(Name) + 3, {{field, Name, NameL, {S, L}}, NameL + L}. @@ -664,6 +669,11 @@ printable_char(C,unicode) -> C > 16#DFFF andalso C < 16#FFFE orelse C > 16#FFFF andalso C =< 16#10FFFF. +write_atom(A, latin1) -> + io_lib:write_atom_as_latin1(A); +write_atom(A, _Uni) -> + io_lib:write_atom(A). + write_string(S, latin1) -> io_lib:write_latin1_string(S, $"); %" write_string(S, _Uni) -> diff --git a/lib/stdlib/src/rand.erl b/lib/stdlib/src/rand.erl index 1f457b9e0e..dfd102f9ef 100644 --- a/lib/stdlib/src/rand.erl +++ b/lib/stdlib/src/rand.erl @@ -45,20 +45,31 @@ %% ===================================================================== %% This depends on the algorithm handler function --type alg_seed() :: exs64_state() | exsplus_state() | exs1024_state(). +-type alg_state() :: + exs64_state() | exsplus_state() | exs1024_state() | term(). + %% This is the algorithm handler function within this module --type alg_handler() :: #{type := alg(), - max := integer(), - next := fun(), - uniform := fun(), - uniform_n := fun(), - jump := fun()}. - -%% Internal state --opaque state() :: {alg_handler(), alg_seed()}. --type alg() :: exs64 | exsplus | exs1024. --opaque export_state() :: {alg(), alg_seed()}. --export_type([alg/0, state/0, export_state/0]). +-type alg_handler() :: + #{type := alg(), + max := integer() | infinity, + next := + fun((alg_state()) -> {non_neg_integer(), alg_state()}), + uniform := + fun((state()) -> {float(), state()}), + uniform_n := + fun((pos_integer(), state()) -> {pos_integer(), state()}), + jump := + fun((state()) -> state())}. + +%% Algorithm state +-type state() :: {alg_handler(), alg_state()}. +-type builtin_alg() :: exs64 | exsplus | exs1024. +-type alg() :: builtin_alg() | atom(). +-type export_state() :: {alg(), alg_state()}. +-export_type( + [builtin_alg/0, alg/0, alg_handler/0, alg_state/0, + state/0, export_state/0]). +-export_type([exs64_state/0, exsplus_state/0, exs1024_state/0]). %% ===================================================================== %% API @@ -72,7 +83,7 @@ export_seed() -> _ -> undefined end. --spec export_seed_s(state()) -> export_state(). +-spec export_seed_s(State :: state()) -> export_state(). export_seed_s({#{type:=Alg}, Seed}) -> {Alg, Seed}. %% seed(Alg) seeds RNG with runtime dependent values @@ -81,27 +92,37 @@ export_seed_s({#{type:=Alg}, Seed}) -> {Alg, Seed}. %% seed({Alg,Seed}) setup RNG with a previously exported seed %% and return the NEW state --spec seed(AlgOrExpState::alg() | export_state()) -> state(). +-spec seed( + AlgOrStateOrExpState :: builtin_alg() | state() | export_state()) -> + state(). seed(Alg) -> seed_put(seed_s(Alg)). --spec seed_s(AlgOrExpState::alg() | export_state()) -> state(). -seed_s(Alg) when is_atom(Alg) -> - seed_s(Alg, {erlang:phash2([{node(),self()}]), - erlang:system_time(), - erlang:unique_integer()}); +-spec seed_s( + AlgOrStateOrExpState :: builtin_alg() | state() | export_state()) -> + state(). +seed_s({AlgHandler, _Seed} = State) when is_map(AlgHandler) -> + State; seed_s({Alg0, Seed}) -> {Alg,_SeedFun} = mk_alg(Alg0), - {Alg, Seed}. + {Alg, Seed}; +seed_s(Alg) -> + seed_s(Alg, {erlang:phash2([{node(),self()}]), + erlang:system_time(), + erlang:unique_integer()}). %% seed/2: seeds RNG with the algorithm and given values %% and returns the NEW state. --spec seed(Alg :: alg(), {integer(), integer(), integer()}) -> state(). +-spec seed( + Alg :: builtin_alg(), Seed :: {integer(), integer(), integer()}) -> + state(). seed(Alg0, S0) -> seed_put(seed_s(Alg0, S0)). --spec seed_s(Alg :: alg(), {integer(), integer(), integer()}) -> state(). +-spec seed_s( + Alg :: builtin_alg(), Seed :: {integer(), integer(), integer()}) -> + state(). seed_s(Alg0, S0 = {_, _, _}) -> {Alg, Seed} = mk_alg(Alg0), AS = Seed(S0), @@ -113,7 +134,7 @@ seed_s(Alg0, S0 = {_, _, _}) -> %% uniform/0: returns a random float X where 0.0 < X < 1.0, %% updating the state in the process dictionary. --spec uniform() -> X::float(). +-spec uniform() -> X :: float(). uniform() -> {X, Seed} = uniform_s(seed_get()), _ = seed_put(Seed), @@ -123,7 +144,7 @@ uniform() -> %% uniform/1 returns a random integer X where 1 =< X =< N, %% updating the state in the process dictionary. --spec uniform(N :: pos_integer()) -> X::pos_integer(). +-spec uniform(N :: pos_integer()) -> X :: pos_integer(). uniform(N) -> {X, Seed} = uniform_s(N, seed_get()), _ = seed_put(Seed), @@ -133,7 +154,7 @@ uniform(N) -> %% returns a random float X where 0.0 < X < 1.0, %% and a new state. --spec uniform_s(state()) -> {X::float(), NewS :: state()}. +-spec uniform_s(State :: state()) -> {X :: float(), NewState :: state()}. uniform_s(State = {#{uniform:=Uniform}, _}) -> Uniform(State). @@ -141,7 +162,8 @@ uniform_s(State = {#{uniform:=Uniform}, _}) -> %% uniform_s/2 returns a random integer X where 1 =< X =< N, %% and a new state. --spec uniform_s(N::pos_integer(), state()) -> {X::pos_integer(), NewS::state()}. +-spec uniform_s(N :: pos_integer(), State :: state()) -> + {X :: pos_integer(), NewState :: state()}. uniform_s(N, State = {#{uniform_n:=Uniform, max:=Max}, _}) when 0 < N, N =< Max -> Uniform(N, State); @@ -155,7 +177,7 @@ uniform_s(N, State0 = {#{uniform:=Uniform}, _}) %% after a large number of call defined for each algorithm. %% The large number is algorithm dependent. --spec jump(state()) -> NewS :: state(). +-spec jump(state()) -> NewState :: state(). jump(State = {#{jump:=Jump}, _}) -> Jump(State). @@ -164,7 +186,7 @@ jump(State = {#{jump:=Jump}, _}) -> %% and write back the new value to the internal state, %% then returns the new value. --spec jump() -> NewS :: state(). +-spec jump() -> NewState :: state(). jump() -> seed_put(jump(seed_get())). @@ -182,7 +204,7 @@ normal() -> %% The Ziggurat Method for generating random variables - Marsaglia and Tsang %% Paper and reference code: http://www.jstatsoft.org/v05/i08/ --spec normal_s(state()) -> {float(), NewS :: state()}. +-spec normal_s(State :: state()) -> {float(), NewState :: state()}. normal_s(State0) -> {Sign, R, State} = get_52(State0), Idx = R band 16#FF, @@ -245,7 +267,7 @@ mk_alg(exs1024) -> %% Reference URL: http://xorshift.di.unimi.it/ %% ===================================================================== --type exs64_state() :: uint64(). +-opaque exs64_state() :: uint64(). exs64_seed({A1, A2, A3}) -> {V1, _} = exs64_next(((A1 band ?UINT32MASK) * 4294967197 + 1)), @@ -280,7 +302,7 @@ exs64_jump(_) -> %% Modification of the original Xorshift128+ algorithm to 116 %% by Sebastiano Vigna, a lot of thanks for his help and work. %% ===================================================================== --type exsplus_state() :: nonempty_improper_list(uint58(), uint58()). +-opaque exsplus_state() :: nonempty_improper_list(uint58(), uint58()). -dialyzer({no_improper_lists, exsplus_seed/1}). @@ -349,7 +371,7 @@ exsplus_jump(S, [AS0|AS1], J, N) -> %% Reference URL: http://xorshift.di.unimi.it/ %% ===================================================================== --type exs1024_state() :: {list(uint64()), list(uint64())}. +-opaque exs1024_state() :: {list(uint64()), list(uint64())}. exs1024_seed({A1, A2, A3}) -> B1 = (((A1 band ?UINT21MASK) + 1) * 2097131) band ?UINT21MASK, diff --git a/lib/stdlib/src/shell.erl b/lib/stdlib/src/shell.erl index 28f37ef8bf..394f4f2fa4 100644 --- a/lib/stdlib/src/shell.erl +++ b/lib/stdlib/src/shell.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -349,10 +349,16 @@ default_prompt(N) -> %% Don't bother flattening the list irrespective of what the %% I/O-protocol states. case is_alive() of - true -> io_lib:format(<<"(~s)~w> ">>, [node(), N]); + true -> io_lib:format(<<"(~ts)~w> ">>, [node_string(), N]); false -> io_lib:format(<<"~w> ">>, [N]) end. +node_string() -> + case encoding() of + latin1 -> io_lib:write_atom_as_latin1(node()); + _ -> io_lib:write_atom(node()) + end. + %% expand_hist(Expressions, CommandNumber) %% Preprocess the expression list replacing all history list commands %% with their expansions. @@ -967,10 +973,11 @@ local_func(f, [{var,_,Name}], Bs, _Shell, _RT, _Lf, _Ef) -> {value,ok,erl_eval:del_binding(Name, Bs)}; local_func(f, [_Other], _Bs, _Shell, _RT, _Lf, _Ef) -> erlang:raise(error, function_clause, [{shell,f,1}]); -local_func(rd, [{atom,_,RecName},RecDef0], Bs, _Shell, RT, _Lf, _Ef) -> +local_func(rd, [{atom,_,RecName0},RecDef0], Bs, _Shell, RT, _Lf, _Ef) -> RecDef = expand_value(RecDef0), RDs = lists:flatten(erl_pp:expr(RecDef)), - Attr = lists:concat(["-record('", RecName, "',", RDs, ")."]), + RecName = io_lib:write_atom_as_latin1(RecName0), + Attr = lists:concat(["-record(", RecName, ",", RDs, ")."]), {ok, Tokens, _} = erl_scan:string(Attr), case erl_parse:parse_form(Tokens) of {ok,AttrForm} -> @@ -1417,9 +1424,11 @@ columns() -> {ok,N} -> N; _ -> 80 end. + encoding() -> [{encoding, Encoding}] = enc(), Encoding. + enc() -> case lists:keyfind(encoding, 1, io:getopts()) of false -> [{encoding,latin1}]; % should never happen diff --git a/lib/stdlib/test/erl_pp_SUITE.erl b/lib/stdlib/test/erl_pp_SUITE.erl index 1a028204b4..808ba9b4c1 100644 --- a/lib/stdlib/test/erl_pp_SUITE.erl +++ b/lib/stdlib/test/erl_pp_SUITE.erl @@ -51,7 +51,7 @@ otp_6321/1, otp_6911/1, otp_6914/1, otp_8150/1, otp_8238/1, otp_8473/1, otp_8522/1, otp_8567/1, otp_8664/1, otp_9147/1, otp_10302/1, otp_10820/1, otp_11100/1, otp_11861/1, pr_1014/1, - otp_13662/1]). + otp_13662/1, otp_14285/1]). %% Internal export. -export([ehook/6]). @@ -80,7 +80,8 @@ groups() -> {tickets, [], [otp_6321, otp_6911, otp_6914, otp_8150, otp_8238, otp_8473, otp_8522, otp_8567, otp_8664, otp_9147, - otp_10302, otp_10820, otp_11100, otp_11861, pr_1014, otp_13662]}]. + otp_10302, otp_10820, otp_11100, otp_11861, pr_1014, otp_13662, + otp_14285]}]. init_per_suite(Config) -> Config. @@ -627,11 +628,6 @@ do_hook(HookFun) -> true = "{some,value}" =:= lists:flatten(erl_pp:expr({value,A0,{some,value}})), - %% Silly... - true = - "if true -> 0 end" =:= - flat_expr({'if',0,[{clause,0,[],[],[{atom,0,0}]}]}), - %% More compatibility: before R6 OldIf = {'if',A0,[{clause,A0,[],[{atom,A0,true}],[{atom,A0,b}]}]}, NewIf = {'if',A0,[{clause,A0,[],[[{atom,A0,true}]],[{atom,A0,b}]}]}, @@ -1069,9 +1065,6 @@ otp_11100(Config) when is_list(Config) -> %% doesn't make a difference (pp:bit_elem_type/1 is an example). A1 = erl_anno:new(1), - %% Cannot trigger the use of the hook function with export/import. - "-export([{fy,a}/b]).\n" = - pf({attribute,A1,export,[{{fy,a},b}]}), "-type foo() :: integer(INVALID-FORM:{foo,bar}:).\n" = pf({attribute,A1,type,{foo,{type,A1,integer,[{foo,bar}]},[]}}), pf({attribute,A1,type, @@ -1146,6 +1139,34 @@ otp_13662(Config) -> ], compile(Config, Ts). +otp_14285(_Config) -> + pp_forms(<<"-export([t/0, '\\x{400}\\''/0]).">>), + pp_forms(<<"-import(lists, [append/2]).">>), + pp_forms(<<"-optional_callbacks([]).">>), + pp_forms(<<"-optional_callbacks(['\\x{400}\\''/1]).">>), + pp_forms(<<"-'\\x{400}\\''('\\x{400}\\'').">>), + pp_forms(<<"-type '\\x{400}\\''() :: '\\x{400}\\''.">>), + pp_forms(<<"-record('\\x{400}\\'', {'\\x{400}\\''}).">>), + pp_forms(<<"-callback '\\x{400}\\''(_) -> '\\x{400}\\''.">>), + pp_forms(<<"t() -> '\\x{400}\\''('\\x{400}\\'').">>), + pp_forms(<<"'\\x{400}\\''(_) -> '\\x{400}\\''.">>), + pp_forms(<<"-spec '\\x{400}'() -> " + "#'\\x{400}'{'\\x{400}' :: '\\x{400}'}.">>), + pp_forms(<<"'\\x{400}\\''() ->" + "R = #'\\x{400}\\''{}," + "#'\\x{400}\\''{'\\x{400}\\'' =" + "{'\\x{400}\\''," + "fun '\\x{400}\\''/0," + "R#'\\x{400}\\''.'\\x{400}\\''," + "#'\\x{400}\\''.'\\x{400}\\''}}.">>), + + %% Special... + true = + "{some,'\\x{400}\\''}" =:= + lists:flatten(erl_pp:expr({value,erl_anno:new(0),{some,'\x{400}\''}}, + [{encoding,latin1}])), + ok. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% compile(Config, Tests) -> diff --git a/lib/stdlib/test/filelib_SUITE.erl b/lib/stdlib/test/filelib_SUITE.erl index 87fba815d2..6133a3ded4 100644 --- a/lib/stdlib/test/filelib_SUITE.erl +++ b/lib/stdlib/test/filelib_SUITE.erl @@ -507,7 +507,12 @@ file_props_symlink(Config) -> end. find_source(Config) when is_list(Config) -> - BeamFile = code:which(lists), + %% filename:find_{file,source}() does not work if the files are + %% cover-compiled. To make sure that the test does not fail + %% when the STDLIB is cover-compiled, search for modules in + %% the compiler application. + + BeamFile = code:which(compile), BeamName = filename:basename(BeamFile), BeamDir = filename:dirname(BeamFile), SrcName = filename:basename(BeamFile, ".beam") ++ ".erl", @@ -530,7 +535,7 @@ find_source(Config) when is_list(Config) -> {error, not_found} = filelib:find_source(BeamName, BeamDir, [{".erl",".yrl",[{"",""}]}]), - {ok, ParserErl} = filelib:find_source(code:which(erl_parse)), + {ok, ParserErl} = filelib:find_source(code:which(core_parse)), {ok, ParserYrl} = filelib:find_source(ParserErl), "lry." ++ _ = lists:reverse(ParserYrl), {ok, ParserYrl} = filelib:find_source(ParserErl, diff --git a/lib/stdlib/test/io_SUITE.erl b/lib/stdlib/test/io_SUITE.erl index d546e8fad2..b2754e47ba 100644 --- a/lib/stdlib/test/io_SUITE.erl +++ b/lib/stdlib/test/io_SUITE.erl @@ -30,7 +30,8 @@ io_lib_print_binary_depth_one/1, otp_10302/1, otp_10755/1, otp_10836/1, io_lib_width_too_small/1, io_with_huge_message_queue/1, format_string/1, - maps/1, coverage/1, otp_14178_unicode_atoms/1, otp_14175/1]). + maps/1, coverage/1, otp_14178_unicode_atoms/1, otp_14175/1, + otp_14285/1]). -export([pretty/2]). @@ -61,7 +62,8 @@ all() -> printable_range, bad_printable_range, io_lib_print_binary_depth_one, otp_10302, otp_10755, otp_10836, io_lib_width_too_small, io_with_huge_message_queue, - format_string, maps, coverage, otp_14178_unicode_atoms, otp_14175]. + format_string, maps, coverage, otp_14178_unicode_atoms, otp_14175, + otp_14285]. %% Error cases for output. error_1(Config) when is_list(Config) -> @@ -755,6 +757,8 @@ rfd(rrrrr, 3) -> [f1, f2, f3]; rfd(aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, 0) -> []; +rfd('\x{400}', 1) -> + ['\x{400}']; rfd(_, _) -> no. @@ -1881,6 +1885,7 @@ otp_10302(Suite) when is_list(Suite) -> pretty(Term, Depth) when is_integer(Depth) -> Opts = [{column, 1}, {line_length, 20}, {depth, Depth}, {max_chars, 60}, + {record_print_fun, fun rfd/2}, {encoding, unicode}], pretty(Term, Opts); pretty(Term, Opts) when is_list(Opts) -> @@ -2324,3 +2329,23 @@ text1([T|Ts]) -> [erl_anno:text(Anno) | text1(Ts)]. -endif. % EXACT + +otp_14285(_Config) -> + UOpts = [{record_print_fun, fun rfd/2}, + {encoding, unicode}], + LOpts = [{record_print_fun, fun rfd/2}, + {encoding, latin1}], + + RT = {'\x{400}','\x{400}'}, + "#'\x{400}'{'\x{400}' = '\x{400}'}" = pretty(RT, UOpts), + "#'\\x{400}'{'\\x{400}' = '\\x{400}'}" = pretty(RT, LOpts), + + Chars = lists:seq(0, 512), + [] = [C || + C <- Chars, + S <- io_lib:write_atom_as_latin1(list_to_atom([C])), + not is_latin1(S)], + L1 = [S || C <- Chars, S <- io_lib:write_atom(list_to_atom([C])), + not is_latin1(S)], + L1 = lists:seq(256, 512), + ok. diff --git a/lib/stdlib/test/rand_SUITE.erl b/lib/stdlib/test/rand_SUITE.erl index fe5eaccda5..098eefeb61 100644 --- a/lib/stdlib/test/rand_SUITE.erl +++ b/lib/stdlib/test/rand_SUITE.erl @@ -356,14 +356,23 @@ basic_normal_1(0, {#{type:=Alg}, _}, Sum, SumSq) -> %% Test that the user can write algorithms. plugin(Config) when is_list(Config) -> - _ = lists:foldl(fun(_, S0) -> - {V1, S1} = rand:uniform_s(10000, S0), - true = is_integer(V1), - {V2, S2} = rand:uniform_s(S1), - true = is_float(V2), - S2 - end, crypto_seed(), lists:seq(1, 200)), - ok. + try crypto:strong_rand_bytes(1) of + <<_>> -> + _ = lists:foldl( + fun(_, S0) -> + {V1, S1} = rand:uniform_s(10000, S0), + true = is_integer(V1), + {V2, S2} = rand:uniform_s(S1), + true = is_float(V2), + S2 + end, crypto_seed(), lists:seq(1, 200)), + ok + catch + error:low_entropy -> + {skip,low_entropy}; + error:undef -> + {skip,no_crypto} + end. %% Test implementation crypto_seed() -> @@ -397,7 +406,13 @@ crypto_uniform_n(N, State0) -> measure(Suite) when is_atom(Suite) -> []; measure(_Config) -> ct:timetrap({minutes,15}), %% valgrind needs a lot of time - Algos = [crypto64|algs()], + Algos = + try crypto:strong_rand_bytes(1) of + <<_>> -> [crypto64] + catch + error:low_entropy -> []; + error:undef -> [] + end ++ algs(), io:format("RNG uniform integer performance~n",[]), _ = measure_1(random, fun(State) -> {int, random:uniform_s(10000, State)} end), _ = [measure_1(Algo, fun(State) -> {int, rand:uniform_s(10000, State)} end) || Algo <- Algos], diff --git a/lib/stdlib/test/shell_SUITE.erl b/lib/stdlib/test/shell_SUITE.erl index 4864bc3d72..56002dda25 100644 --- a/lib/stdlib/test/shell_SUITE.erl +++ b/lib/stdlib/test/shell_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2016. All Rights Reserved. +%% Copyright Ericsson AB 2004-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -30,7 +30,8 @@ progex_bit_syntax/1, progex_records/1, progex_lc/1, progex_funs/1, otp_5990/1, otp_6166/1, otp_6554/1, - otp_7184/1, otp_7232/1, otp_8393/1, otp_10302/1, otp_13719/1]). + otp_7184/1, otp_7232/1, otp_8393/1, otp_10302/1, otp_13719/1, + otp_14285/1]). -export([ start_restricted_from_shell/1, start_restricted_on_command_line/1,restricted_local/1]). @@ -91,7 +92,7 @@ groups() -> progex_funs]}, {tickets, [], [otp_5990, otp_6166, otp_6554, otp_7184, - otp_7232, otp_8393, otp_10302, otp_13719]}]. + otp_7232, otp_8393, otp_10302, otp_13719, otp_14285]}]. init_per_suite(Config) -> Config. @@ -2824,6 +2825,22 @@ otp_13719(Config) when is_list(Config) -> file:delete(File), ok. +otp_14285(Config) -> + {ok,Node} = start_node(shell_suite_helper_4, + "-pa "++proplists:get_value(priv_dir,Config)++ + " +pc unicode"), + Test1 = + <<"begin + io:setopts([{encoding,utf8}]), + [1024] = atom_to_list('\\x{400}'), + rd('\\x{400}', {'\\x{400}' = '\\x{400}'}), + ok = rl('\\x{400}') + end.">>, + "-record('\x{400}',{'\x{400}' = '\x{400}'}).\nok.\n" = + t({Node,Test1}), + test_server:stop_node(Node), + ok. + scan(B) -> F = fun(Ts) -> case erl_parse:parse_term(Ts) of diff --git a/lib/syntax_tools/src/erl_prettypr.erl b/lib/syntax_tools/src/erl_prettypr.erl index 378d69095d..40ddd2b22a 100644 --- a/lib/syntax_tools/src/erl_prettypr.erl +++ b/lib/syntax_tools/src/erl_prettypr.erl @@ -452,7 +452,7 @@ lay_2(Node, Ctxt) -> text(erl_syntax:variable_literal(Node)); atom -> - text(erl_syntax:atom_literal(Node)); + text(erl_syntax:atom_literal(Node, Ctxt#ctxt.encoding)); integer -> text(erl_syntax:integer_literal(Node)); diff --git a/lib/syntax_tools/src/erl_syntax.erl b/lib/syntax_tools/src/erl_syntax.erl index 4347cc46c1..9b2b503762 100644 --- a/lib/syntax_tools/src/erl_syntax.erl +++ b/lib/syntax_tools/src/erl_syntax.erl @@ -139,6 +139,7 @@ is_atom/2, atom_value/1, atom_literal/1, + atom_literal/2, atom_name/1, attribute/1, attribute/2, @@ -1841,7 +1842,7 @@ char_literal(Node) -> %% @doc Returns the literal string represented by a `char' %% node. This includes the leading "`$'" character. %% Depending on the encoding a character beyond 255 will be escaped -%% ('latin1') or copied as is ('utf8'). +%% (`latin1') or copied as is (`utf8'). %% %% @see char/1 @@ -1944,7 +1945,7 @@ string_literal(Node) -> %% @doc Returns the literal string represented by a `string' %% node. This includes surrounding double-quote characters. %% Depending on the encoding characters beyond 255 will be escaped -%% ('latin1') or copied as is ('utf8'). +%% (`latin1') or copied as is (`utf8'). %% %% @see string/1 @@ -1965,6 +1966,7 @@ string_literal(Node, latin1) -> %% @see atom_value/1 %% @see atom_name/1 %% @see atom_literal/1 +%% @see atom_literal/2 %% @see is_atom/2 %% type(Node) = atom @@ -2037,6 +2039,7 @@ atom_name(Node) -> %% ===================================================================== %% @doc Returns the literal string represented by an `atom' %% node. This includes surrounding single-quote characters if necessary. +%% Characters beyond 255 will be escaped. %% %% Note that e.g. the result of `atom("x\ny")' represents %% any and all of `'x\ny'', `'x\12y'', @@ -2048,8 +2051,24 @@ atom_name(Node) -> -spec atom_literal(syntaxTree()) -> string(). atom_literal(Node) -> - io_lib:write_atom(atom_value(Node)). + atom_literal(Node, latin1). + +%% ===================================================================== +%% @doc Returns the literal string represented by an `atom' +%% node. This includes surrounding single-quote characters if necessary. +%% Depending on the encoding a character beyond 255 will be escaped +%% (`latin1') or copied as is (`utf8'). +%% +%% @see atom/1 +%% @see atom_literal/1 +%% @see string/1 +atom_literal(Node, utf8) -> + io_lib:write_atom(atom_value(Node)); +atom_literal(Node, unicode) -> + io_lib:write_atom(atom_value(Node)); +atom_literal(Node, latin1) -> + io_lib:write_atom_as_latin1(atom_value(Node)). %% ===================================================================== %% @equiv map_expr(none, Fields) diff --git a/lib/wx/c_src/wxe_impl.cpp b/lib/wx/c_src/wxe_impl.cpp index 0d2da5d4a7..05d56667ab 100644 --- a/lib/wx/c_src/wxe_impl.cpp +++ b/lib/wx/c_src/wxe_impl.cpp @@ -128,7 +128,7 @@ bool WxeApp::OnInit() delayed_cleanup = new wxList; wxe_ps_init2(); - // wxIdleEvent::SetMode(wxIDLE_PROCESS_SPECIFIED); // Hmm printpreview doesn't work in 2.9 with this + wxIdleEvent::SetMode(wxIDLE_PROCESS_SPECIFIED); Connect(wxID_ANY, wxEVT_IDLE, (wxObjectEventFunction) (wxEventFunction) &WxeApp::idle); Connect(CREATE_PORT, wxeEVT_META_COMMAND,(wxObjectEventFunction) (wxEventFunction) &WxeApp::newMemEnv); @@ -200,7 +200,8 @@ void WxeApp::OnAssertFailure(const wxChar *file, int line, const wxChar *cfunc, // Called by wx thread void WxeApp::idle(wxIdleEvent& event) { event.Skip(true); - dispatch_cmds(); + if(dispatch_cmds()) + event.RequestMore(); } /* ************************************************************ @@ -233,14 +234,15 @@ void handle_event_callback(ErlDrvPort port, ErlDrvTermData process) } } -void WxeApp::dispatch_cmds() +int WxeApp::dispatch_cmds() { + int more = 0; if(wxe_status != WXE_INITIATED) - return; + return more; recurse_level++; // fprintf(stderr, "\r\ndispatch_normal %d\r\n", recurse_level);fflush(stderr); wxe_queue->cb_start = 0; - dispatch(wxe_queue); + more = dispatch(wxe_queue); // fprintf(stderr, "\r\ndispatch_done %d\r\n", recurse_level);fflush(stderr); recurse_level--; @@ -262,12 +264,14 @@ void WxeApp::dispatch_cmds() delete event; } } + return more; } int WxeApp::dispatch(wxeFifo * batch) { int ping = 0; int blevel = 0; + int wait = 0; // Let event handling generate events sometime wxeCommand *event; erl_drv_mutex_lock(wxe_batch_locker_m); while(true) { @@ -275,10 +279,10 @@ int WxeApp::dispatch(wxeFifo * batch) erl_drv_mutex_unlock(wxe_batch_locker_m); switch(event->op) { case WXE_BATCH_END: - {--blevel; } + if(blevel>0) blevel--; break; case WXE_BATCH_BEGIN: - {blevel++; } + blevel++; break; case WXE_DEBUG_PING: // When in debugger we don't want to hang waiting for a BATCH_END @@ -293,7 +297,7 @@ int WxeApp::dispatch(wxeFifo * batch) memcpy(cb_buff, event->buffer, event->len); } event->Delete(); - return blevel; + return 1; default: if(event->op < OPENGL_START) { // fprintf(stderr, " c %d (%d) \r\n", event->op, blevel); @@ -307,13 +311,15 @@ int WxeApp::dispatch(wxeFifo * batch) erl_drv_mutex_lock(wxe_batch_locker_m); batch->Cleanup(); } - if(blevel <= 0) { + if(blevel <= 0 || wait > 3) { erl_drv_mutex_unlock(wxe_batch_locker_m); - return blevel; + if(blevel > 0) return 1; // We are still in a batch but we can let wx check for events + else return 0; } // sleep until something happens - //fprintf(stderr, "%s:%d sleep %d %d\r\n", __FILE__, __LINE__, batch->m_n, blevel);fflush(stderr); + // fprintf(stderr, "%s:%d sleep %d %d %d\r\n", __FILE__, __LINE__, batch->m_n, blevel, wait);fflush(stderr); wxe_needs_signal = 1; + wait += 1; while(batch->m_n == 0) { erl_drv_cond_wait(wxe_batch_locker_c, wxe_batch_locker_m); } diff --git a/lib/wx/c_src/wxe_impl.h b/lib/wx/c_src/wxe_impl.h index 57dac997ab..68f5deb336 100644 --- a/lib/wx/c_src/wxe_impl.h +++ b/lib/wx/c_src/wxe_impl.h @@ -73,7 +73,7 @@ public: void wxe_dispatch(wxeCommand& event); void idle(wxIdleEvent& event); - void dispatch_cmds(); + int dispatch_cmds(); void dummy_close(wxEvent& Ev); bool sendevent(wxEvent *event); diff --git a/lib/wx/examples/demo/demo.erl b/lib/wx/examples/demo/demo.erl index 8b7412017a..0258202a67 100644 --- a/lib/wx/examples/demo/demo.erl +++ b/lib/wx/examples/demo/demo.erl @@ -243,6 +243,9 @@ handle_event(#wx{id = Id, %% If you are going to printout mainly text it is easier if %% you generate HTML code and use a wxHtmlEasyPrint %% instead of using DCs + + %% Printpreview doesn't work in >2.9 without this + wxIdleEvent:setMode(?wxIDLE_PROCESS_ALL), Module = "ex_" ++ wxListBox:getStringSelection(State#state.selector) ++ ".erl", HEP = wxHtmlEasyPrinting:new([{name, "Print"}, {parentWindow, State#state.win}]), diff --git a/lib/wx/src/wxe_master.erl b/lib/wx/src/wxe_master.erl index e17a3327ac..913bf4d41b 100644 --- a/lib/wx/src/wxe_master.erl +++ b/lib/wx/src/wxe_master.erl @@ -82,8 +82,14 @@ init_port(SilentStart) -> %% Initalizes the opengl library %%-------------------------------------------------------------------- init_opengl() -> - GLLib = wxe_util:wxgl_dl(), - wxe_util:call(?WXE_INIT_OPENGL, <<(list_to_binary(GLLib))/binary, 0:8>>). + case get(wx_init_opengl) of + true -> {ok, "already initialized"}; + _ -> + GLLib = wxe_util:wxgl_dl(), + Res = wxe_util:call(?WXE_INIT_OPENGL, <<(list_to_binary(GLLib))/binary, 0:8>>), + element(1, Res) =:= ok andalso put(wx_init_opengl, true), + Res + end. %%-------------------------------------------------------------------- %% Fetch early messages, hack to get start up args on mac diff --git a/otp_versions.table b/otp_versions.table index 40a01f6899..8fb437bb96 100644 --- a/otp_versions.table +++ b/otp_versions.table @@ -1,3 +1,4 @@ +OTP-19.3.1 : crypto-3.7.4 erts-8.3.1 inets-6.3.7 ssh-4.4.2 ssl-8.1.2 # asn1-4.0.4 common_test-1.14 compiler-7.0.4 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 debugger-4.2.1 dialyzer-3.1 diameter-1.12.2 edoc-0.8.1 eldap-1.2.2 erl_docgen-0.6.1 erl_interface-3.9.3 et-1.6 eunit-2.3.2 gs-1.6.2 hipe-3.15.4 ic-4.4.2 jinterface-1.7.1 kernel-5.2 megaco-3.18.1 mnesia-4.14.3 observer-2.3.1 odbc-2.12 orber-3.8.2 os_mon-2.4.2 otp_mibs-1.1.1 parsetools-2.1.4 percept-0.9 public_key-1.4 reltool-0.7.3 runtime_tools-1.11.1 sasl-3.0.3 snmp-5.2.5 stdlib-3.3 syntax_tools-2.1.1 tools-2.9.1 typer-0.9.12 wx-1.8 xmerl-1.3.13 : OTP-19.3 : common_test-1.14 compiler-7.0.4 crypto-3.7.3 dialyzer-3.1 diameter-1.12.2 erl_interface-3.9.3 erts-8.3 hipe-3.15.4 inets-6.3.6 kernel-5.2 observer-2.3.1 os_mon-2.4.2 public_key-1.4 reltool-0.7.3 runtime_tools-1.11.1 sasl-3.0.3 snmp-5.2.5 ssh-4.4.1 ssl-8.1.1 stdlib-3.3 tools-2.9.1 typer-0.9.12 xmerl-1.3.13 # asn1-4.0.4 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 debugger-4.2.1 edoc-0.8.1 eldap-1.2.2 erl_docgen-0.6.1 et-1.6 eunit-2.3.2 gs-1.6.2 ic-4.4.2 jinterface-1.7.1 megaco-3.18.1 mnesia-4.14.3 odbc-2.12 orber-3.8.2 otp_mibs-1.1.1 parsetools-2.1.4 percept-0.9 syntax_tools-2.1.1 wx-1.8 : OTP-19.2.3 : erts-8.2.2 inets-6.3.5 # asn1-4.0.4 common_test-1.13 compiler-7.0.3 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7.2 debugger-4.2.1 dialyzer-3.0.3 diameter-1.12.1 edoc-0.8.1 eldap-1.2.2 erl_docgen-0.6.1 erl_interface-3.9.2 et-1.6 eunit-2.3.2 gs-1.6.2 hipe-3.15.3 ic-4.4.2 jinterface-1.7.1 kernel-5.1.1 megaco-3.18.1 mnesia-4.14.3 observer-2.3 odbc-2.12 orber-3.8.2 os_mon-2.4.1 otp_mibs-1.1.1 parsetools-2.1.4 percept-0.9 public_key-1.3 reltool-0.7.2 runtime_tools-1.11 sasl-3.0.2 snmp-5.2.4 ssh-4.4 ssl-8.1 stdlib-3.2 syntax_tools-2.1.1 tools-2.9 typer-0.9.11 wx-1.8 xmerl-1.3.12 : OTP-19.2.2 : mnesia-4.14.3 # asn1-4.0.4 common_test-1.13 compiler-7.0.3 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7.2 debugger-4.2.1 dialyzer-3.0.3 diameter-1.12.1 edoc-0.8.1 eldap-1.2.2 erl_docgen-0.6.1 erl_interface-3.9.2 erts-8.2.1 et-1.6 eunit-2.3.2 gs-1.6.2 hipe-3.15.3 ic-4.4.2 inets-6.3.4 jinterface-1.7.1 kernel-5.1.1 megaco-3.18.1 observer-2.3 odbc-2.12 orber-3.8.2 os_mon-2.4.1 otp_mibs-1.1.1 parsetools-2.1.4 percept-0.9 public_key-1.3 reltool-0.7.2 runtime_tools-1.11 sasl-3.0.2 snmp-5.2.4 ssh-4.4 ssl-8.1 stdlib-3.2 syntax_tools-2.1.1 tools-2.9 typer-0.9.11 wx-1.8 xmerl-1.3.12 : |