diff options
Diffstat (limited to 'lib')
73 files changed, 2732 insertions, 577 deletions
diff --git a/lib/common_test/test_server/ts_erl_config.erl b/lib/common_test/test_server/ts_erl_config.erl index c2852131d6..f3972bea4e 100644 --- a/lib/common_test/test_server/ts_erl_config.erl +++ b/lib/common_test/test_server/ts_erl_config.erl @@ -197,28 +197,24 @@ system_include(Root, Vars) -> " -I" ++ quote(filename:nativename(filename:join([Root, "erts", "emulator", SysDir]))). erl_interface(Vars,OsType) -> - {Incl, TargetIncl, {LibPath, MkIncl}} = + {Incl, {LibPath, MkIncl}} = case lib_dir(Vars, erl_interface) of {error, bad_name} -> throw({cannot_find_app, erl_interface}); Dir -> - BaseIncl = filename:join(Dir, "include"), - case erl_root(Vars) of - {installed, _Root} -> - {BaseIncl, - [], - {filename:join(Dir, "lib"), - filename:join([Dir, "src", "eidefs.mk"])}}; - {srctree, _Root, Target} -> - Obj = case is_debug_build() of - true -> "obj.debug"; - false -> "obj" - end, - {BaseIncl, - filename:join(BaseIncl, Target), - {filename:join([Dir, Obj, Target]), - filename:join([Dir, "src", Target, "eidefs.mk"])}} - end + {filename:join(Dir, "include"), + case erl_root(Vars) of + {installed, _Root} -> + {filename:join(Dir, "lib"), + filename:join([Dir, "src", "eidefs.mk"])}; + {srctree, _Root, Target} -> + Obj = case is_debug_build() of + true -> "obj.debug"; + false -> "obj" + end, + {filename:join([Dir, Obj, Target]), + filename:join([Dir, "src", Target, "eidefs.mk"])} + end} end, Lib = link_library("erl_interface",OsType), Lib1 = link_library("ei",OsType), @@ -264,10 +260,6 @@ erl_interface(Vars,OsType) -> {erl_interface_eilib_drv, quote(filename:join(LibPath, Lib1Drv))}, {erl_interface_threadlib, ThreadLib}, {erl_interface_include, quote(filename:nativename(Incl))}, - {erl_interface_target_include, case TargetIncl of - [] -> []; - _ -> "-I" ++ quote(filename:nativename(TargetIncl)) - end}, {erl_interface_mk_include, quote(filename:nativename(MkIncl))} | Vars]. diff --git a/lib/crypto/c_src/atoms.c b/lib/crypto/c_src/atoms.c index 2e417da7f4..798c26c9bb 100644 --- a/lib/crypto/c_src/atoms.c +++ b/lib/crypto/c_src/atoms.c @@ -41,6 +41,18 @@ ERL_NIF_TERM atom_not_enabled; ERL_NIF_TERM atom_not_supported; #endif +ERL_NIF_TERM atom_type; +ERL_NIF_TERM atom_size; +ERL_NIF_TERM atom_block_size; +ERL_NIF_TERM atom_key_length; +ERL_NIF_TERM atom_iv_length; +ERL_NIF_TERM atom_mode; +ERL_NIF_TERM atom_ecb_mode; +ERL_NIF_TERM atom_cbc_mode; +ERL_NIF_TERM atom_cfb_mode; +ERL_NIF_TERM atom_ofb_mode; +ERL_NIF_TERM atom_stream_cipher; + #if defined(HAVE_EC) ERL_NIF_TERM atom_prime_field; ERL_NIF_TERM atom_characteristic_two_field; @@ -140,6 +152,18 @@ int init_atoms(ErlNifEnv *env, const ERL_NIF_TERM fips_mode, const ERL_NIF_TERM atom_notsup = enif_make_atom(env,"notsup"); atom_digest = enif_make_atom(env,"digest"); + atom_type = enif_make_atom(env,"type"); + atom_size = enif_make_atom(env,"size"); + atom_block_size = enif_make_atom(env,"block_size"); + atom_key_length = enif_make_atom(env,"key_length"); + atom_iv_length = enif_make_atom(env,"iv_length"); + atom_mode = enif_make_atom(env,"mode"); + atom_ecb_mode = enif_make_atom(env,"ecb_mode"); + atom_cbc_mode = enif_make_atom(env,"cbc_mode"); + atom_cfb_mode = enif_make_atom(env,"cfb_mode"); + atom_ofb_mode = enif_make_atom(env,"ofb_mode"); + atom_stream_cipher = enif_make_atom(env,"stream_cipher"); + #if defined(HAVE_EC) atom_prime_field = enif_make_atom(env,"prime_field"); atom_characteristic_two_field = enif_make_atom(env,"characteristic_two_field"); diff --git a/lib/crypto/c_src/atoms.h b/lib/crypto/c_src/atoms.h index f15523d865..f8e9211459 100644 --- a/lib/crypto/c_src/atoms.h +++ b/lib/crypto/c_src/atoms.h @@ -45,6 +45,18 @@ extern ERL_NIF_TERM atom_not_enabled; extern ERL_NIF_TERM atom_not_supported; #endif +extern ERL_NIF_TERM atom_type; +extern ERL_NIF_TERM atom_size; +extern ERL_NIF_TERM atom_block_size; +extern ERL_NIF_TERM atom_key_length; +extern ERL_NIF_TERM atom_iv_length; +extern ERL_NIF_TERM atom_mode; +extern ERL_NIF_TERM atom_ecb_mode; +extern ERL_NIF_TERM atom_cbc_mode; +extern ERL_NIF_TERM atom_cfb_mode; +extern ERL_NIF_TERM atom_ofb_mode; +extern ERL_NIF_TERM atom_stream_cipher; + #if defined(HAVE_EC) extern ERL_NIF_TERM atom_prime_field; extern ERL_NIF_TERM atom_characteristic_two_field; diff --git a/lib/crypto/c_src/cipher.c b/lib/crypto/c_src/cipher.c index f8e44b228a..c055a62654 100644 --- a/lib/crypto/c_src/cipher.c +++ b/lib/crypto/c_src/cipher.c @@ -67,14 +67,26 @@ static struct cipher_type_t cipher_types[] = {{"aes_cfb8"}, {&EVP_aes_192_cfb8}, 24, NO_FIPS_CIPHER | AES_CFBx}, {{"aes_cfb8"}, {&EVP_aes_256_cfb8}, 32, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_128_cfb8"}, {&EVP_aes_128_cfb8}, 16, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_192_cfb8"}, {&EVP_aes_192_cfb8}, 24, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_256_cfb8"}, {&EVP_aes_256_cfb8}, 32, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_cfb128"}, {&EVP_aes_128_cfb128}, 16, NO_FIPS_CIPHER | AES_CFBx}, {{"aes_cfb128"}, {&EVP_aes_192_cfb128}, 24, NO_FIPS_CIPHER | AES_CFBx}, {{"aes_cfb128"}, {&EVP_aes_256_cfb128}, 32, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_128_cfb128"}, {&EVP_aes_128_cfb128}, 16, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_192_cfb128"}, {&EVP_aes_192_cfb128}, 24, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_256_cfb128"}, {&EVP_aes_256_cfb128}, 32, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_ecb"}, {&EVP_aes_128_ecb}, 16, ECB_BUG_0_9_8L}, {{"aes_ecb"}, {&EVP_aes_192_ecb}, 24, ECB_BUG_0_9_8L}, {{"aes_ecb"}, {&EVP_aes_256_ecb}, 32, ECB_BUG_0_9_8L}, + {{"aes_128_ecb"}, {&EVP_aes_128_ecb}, 16, ECB_BUG_0_9_8L}, + {{"aes_192_ecb"}, {&EVP_aes_192_ecb}, 24, ECB_BUG_0_9_8L}, + {{"aes_256_ecb"}, {&EVP_aes_256_ecb}, 32, ECB_BUG_0_9_8L}, + #if defined(HAVE_EVP_AES_CTR) {{"aes_128_ctr"}, {&EVP_aes_128_ctr}, 16, 0}, {{"aes_192_ctr"}, {&EVP_aes_192_ctr}, 24, 0}, @@ -207,6 +219,87 @@ int cmp_cipher_types(const void *keyp, const void *elemp) { } +ERL_NIF_TERM cipher_info_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Type) */ + const struct cipher_type_t *cipherp; + const EVP_CIPHER *cipher; + ERL_NIF_TERM ret, ret_mode; + unsigned type; + unsigned long mode; + + if ((cipherp = get_cipher_type_no_key(argv[0])) == NULL) + return enif_make_badarg(env); + + if (FORBIDDEN_IN_FIPS(cipherp)) + return enif_raise_exception(env, atom_notsup); + if ((cipher = cipherp->cipher.p) == NULL) + return enif_raise_exception(env, atom_notsup); + + ret = enif_make_new_map(env); + + type = EVP_CIPHER_type(cipher); + enif_make_map_put(env, ret, atom_type, + type == NID_undef ? atom_undefined : enif_make_int(env, type), + &ret); + + enif_make_map_put(env, ret, atom_key_length, + enif_make_int(env, EVP_CIPHER_key_length(cipher)), &ret); + enif_make_map_put(env, ret, atom_iv_length, + enif_make_int(env, EVP_CIPHER_iv_length(cipher)), &ret); + enif_make_map_put(env, ret, atom_block_size, + enif_make_int(env, EVP_CIPHER_block_size(cipher)), &ret); + + mode = EVP_CIPHER_mode(cipher); + switch (mode) { + case EVP_CIPH_ECB_MODE: + ret_mode = atom_ecb_mode; + break; + + case EVP_CIPH_CBC_MODE: + ret_mode = atom_cbc_mode; + break; + + case EVP_CIPH_CFB_MODE: + ret_mode = atom_cfb_mode; + break; + + case EVP_CIPH_OFB_MODE: + ret_mode = atom_ofb_mode; + break; + + case EVP_CIPH_STREAM_CIPHER: + ret_mode = atom_stream_cipher; + break; + + default: + ret_mode = atom_undefined; + break; + } + + enif_make_map_put(env, ret, atom_mode, ret_mode, &ret); + + return ret; +} + +const struct cipher_type_t* get_cipher_type_no_key(ERL_NIF_TERM type) +{ + struct cipher_type_t key; + + key.type.atom = type; + + return bsearch(&key, cipher_types, num_cipher_types, sizeof(cipher_types[0]), cmp_cipher_types_no_key); +} + +int cmp_cipher_types_no_key(const void *keyp, const void *elemp) { + const struct cipher_type_t *key = keyp; + const struct cipher_type_t *elem = elemp; + + if (key->type.atom < elem->type.atom) return -1; + else if (key->type.atom > elem->type.atom) return 1; + else /* key->type.atom == elem->type.atom */ return 0; +} + + ERL_NIF_TERM cipher_types_as_list(ErlNifEnv* env) { struct cipher_type_t* p; diff --git a/lib/crypto/c_src/cipher.h b/lib/crypto/c_src/cipher.h index 6b43afea99..b0d9d324e1 100644 --- a/lib/crypto/c_src/cipher.h +++ b/lib/crypto/c_src/cipher.h @@ -61,12 +61,16 @@ struct evp_cipher_ctx { EVP_CIPHER_CTX* ctx; }; +ERL_NIF_TERM cipher_info_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + int init_cipher_ctx(ErlNifEnv *env); void init_cipher_types(ErlNifEnv* env); +const struct cipher_type_t* get_cipher_type_no_key(ERL_NIF_TERM type); const struct cipher_type_t* get_cipher_type(ERL_NIF_TERM type, size_t key_len); int cmp_cipher_types(const void *keyp, const void *elemp); +int cmp_cipher_types_no_key(const void *keyp, const void *elemp); ERL_NIF_TERM cipher_types_as_list(ErlNifEnv* env); diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c index 06439c34b2..261590d9a5 100644 --- a/lib/crypto/c_src/crypto.c +++ b/lib/crypto/c_src/crypto.c @@ -67,6 +67,7 @@ static ErlNifFunc nif_funcs[] = { {"info_fips", 0, info_fips, 0}, {"enable_fips_mode", 1, enable_fips_mode, 0}, {"algorithms", 0, algorithms, 0}, + {"hash_info", 1, hash_info_nif, 0}, {"hash_nif", 2, hash_nif, 0}, {"hash_init_nif", 1, hash_init_nif, 0}, {"hash_update_nif", 2, hash_update_nif, 0}, @@ -78,6 +79,7 @@ static ErlNifFunc nif_funcs[] = { {"hmac_final_nif", 1, hmac_final_nif, 0}, {"hmac_final_nif", 2, hmac_final_nif, 0}, {"cmac_nif", 3, cmac_nif, 0}, + {"cipher_info_nif", 1, cipher_info_nif, 0}, {"block_crypt_nif", 5, block_crypt_nif, 0}, {"block_crypt_nif", 4, block_crypt_nif, 0}, {"aes_ige_crypt_nif", 4, aes_ige_crypt_nif, 0}, diff --git a/lib/crypto/c_src/hash.c b/lib/crypto/c_src/hash.c index 457e9d071a..0a9f64acef 100644 --- a/lib/crypto/c_src/hash.c +++ b/lib/crypto/c_src/hash.c @@ -61,6 +61,32 @@ int init_hash_ctx(ErlNifEnv* env) { #endif } +ERL_NIF_TERM hash_info_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Type) */ + struct digest_type_t *digp = NULL; + const EVP_MD *md; + ERL_NIF_TERM ret; + + ASSERT(argc == 1); + + if ((digp = get_digest_type(argv[0])) == NULL) + return enif_make_badarg(env); + + if ((md = digp->md.p) == NULL) + return atom_notsup; + + ret = enif_make_new_map(env); + + enif_make_map_put(env, ret, atom_type, + enif_make_int(env, EVP_MD_type(md)), &ret); + enif_make_map_put(env, ret, atom_size, + enif_make_int(env, EVP_MD_size(md)), &ret); + enif_make_map_put(env, ret, atom_block_size, + enif_make_int(env, EVP_MD_block_size(md)), &ret); + + return ret; +} + ERL_NIF_TERM hash_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {/* (Type, Data) */ struct digest_type_t *digp = NULL; diff --git a/lib/crypto/c_src/hash.h b/lib/crypto/c_src/hash.h index 8bae07f39a..92a25cedb7 100644 --- a/lib/crypto/c_src/hash.h +++ b/lib/crypto/c_src/hash.h @@ -25,6 +25,7 @@ int init_hash_ctx(ErlNifEnv *env); +ERL_NIF_TERM hash_info_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM hash_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM hash_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM hash_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index fe8390c5b8..97a4a7a3f0 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -24,6 +24,7 @@ -export([start/0, stop/0, info_lib/0, info_fips/0, supports/0, enable_fips_mode/1, version/0, bytes_to_integer/1]). +-export([cipher_info/1, hash_info/1]). -export([hash/2, hash_init/1, hash_update/2, hash_final/1]). -export([sign/4, sign/5, verify/5, verify/6]). -export([generate_key/2, generate_key/3, compute_key/4]). @@ -403,6 +404,11 @@ enable_fips_mode(_) -> ?nif_stub. -define(HASH_HASH_ALGORITHM, sha1() | sha2() | sha3() | blake2() | ripemd160 | compatibility_only_hash() ). +-spec hash_info(Type) -> map() when Type :: ?HASH_HASH_ALGORITHM. + +hash_info(Type) -> + notsup_to_error(hash_info_nif(Type)). + -spec hash(Type, Data) -> Digest when Type :: ?HASH_HASH_ALGORITHM, Data :: iodata(), Digest :: binary(). @@ -531,6 +537,12 @@ poly1305(Key, Data) -> %%% %%%================================================================ +-spec cipher_info(Type) -> map() when Type :: block_cipher_with_iv() + | aead_cipher() + | block_cipher_without_iv(). +cipher_info(Type) -> + cipher_info_nif(Type). + %%%---- Block ciphers %%%---------------------------------------------------------------- @@ -1726,6 +1738,7 @@ hash_update(State0, Data, _, MaxBytes) -> State = notsup_to_error(hash_update_nif(State0, Increment)), hash_update(State, Rest, erlang:byte_size(Rest), MaxBytes). +hash_info_nif(_Hash) -> ?nif_stub. hash_nif(_Hash, _Data) -> ?nif_stub. hash_init_nif(_Hash) -> ?nif_stub. hash_update_nif(_State, _Data) -> ?nif_stub. @@ -1770,6 +1783,8 @@ poly1305_nif(_Key, _Data) -> ?nif_stub. %% CIPHERS -------------------------------------------------------------------- +cipher_info_nif(_Type) -> ?nif_stub. + block_crypt_nif(_Type, _Key, _Ivec, _Text, _IsEncrypt) -> ?nif_stub. block_crypt_nif(_Type, _Key, _Text, _IsEncrypt) -> ?nif_stub. diff --git a/lib/crypto/test/crypto_SUITE.erl b/lib/crypto/test/crypto_SUITE.erl index ab6d88deb2..7257f4fb9f 100644 --- a/lib/crypto/test/crypto_SUITE.erl +++ b/lib/crypto/test/crypto_SUITE.erl @@ -40,7 +40,9 @@ all() -> rand_uniform, rand_threads, rand_plugin, - rand_plugin_s + rand_plugin_s, + cipher_info, + hash_info ]. groups() -> @@ -666,6 +668,23 @@ rand_plugin_s(Config) when is_list(Config) -> rand_plugin_aux(explicit_state). %%-------------------------------------------------------------------- +cipher_info() -> + [{doc, "crypto cipher_info testing"}]. +cipher_info(Config) when is_list(Config) -> + #{type := _,key_length := _,iv_length := _, + block_size := _,mode := _} = crypto:cipher_info(aes_128_cbc), + {'EXIT',_} = (catch crypto:cipher_info(not_a_cipher)), + ok. + +%%-------------------------------------------------------------------- +hash_info() -> + [{doc, "crypto hash_info testing"}]. +hash_info(Config) when is_list(Config) -> + #{type := _,size := _,block_size := _} = crypto:hash_info(sha256), + {'EXIT',_} = (catch crypto:hash_info(not_a_hash)), + ok. + +%%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- hash(_, [], []) -> diff --git a/lib/diameter/doc/src/diameter.xml b/lib/diameter/doc/src/diameter.xml index 0a0194af2d..85522c99b2 100644 --- a/lib/diameter/doc/src/diameter.xml +++ b/lib/diameter/doc/src/diameter.xml @@ -1,7 +1,9 @@ <?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE erlref SYSTEM "erlref.dtd" [ - <!ENTITY spawn_opt + <!ENTITY spawn_opt2 '<seealso marker="erts:erlang#spawn_opt-2">erlang:spawn_opt/2</seealso>'> + <!ENTITY spawn_opt5 + '<seealso marker="erts:erlang#spawn_opt-5">erlang:spawn_opt/5</seealso>'> <!ENTITY nodes '<seealso marker="erts:erlang#nodes-0">erlang:nodes/0</seealso>'> <!ENTITY make_ref @@ -21,7 +23,7 @@ <copyright> <year>2011</year> -<year>2017</year> +<year>2019</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -1384,12 +1386,22 @@ the same peer.</p> </item> <tag> -<marker id="spawn_opt"/><c>{spawn_opt, [term()]}</c></tag> +<marker id="spawn_opt"/><c>{spawn_opt, [term()] | {M,F,A}}</c></tag> <item> <p> -Options passed to &spawn_opt; when spawning a process for an -incoming Diameter request. -Options <c>monitor</c> and <c>link</c> are ignored.</p> +An options list passed to &spawn_opt2; to spawn a handler process for an +incoming Diameter request on the local node, or an MFA that returns +the pid of a handler process.</p> + +<p> +Options <c>monitor</c> and <c>link</c> are ignored in the list-valued +case. +An MFA is applied with an additional term prepended to its argument +list, and should return either the pid of the handler process that +invokes <c>diameter_traffic:request/1</c> on the term in order to +process the request, or the atom <c>discard</c>. +The handler process need not be local, but diameter must be started on +the remote node.</p> <p> Defaults to the empty list.</p> diff --git a/lib/diameter/doc/src/diameter_transport.xml b/lib/diameter/doc/src/diameter_transport.xml index 67fd54bc56..0a8ef321c6 100644 --- a/lib/diameter/doc/src/diameter_transport.xml +++ b/lib/diameter/doc/src/diameter_transport.xml @@ -14,7 +14,8 @@ <erlref> <header> <copyright> -<year>2011</year><year>2016</year> +<year>2011</year> +<year>2019</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -174,10 +175,13 @@ its parent.</p> <taglist> -<tag><c>{diameter, {send, &message;}}</c></tag> +<tag><c>{diameter, {send, &message; | false}}</c></tag> <item> <p> -An outbound Diameter message.</p> +An outbound Diameter message. +The atom <c>false</c> can only be received when request +acknowledgements have been requests: see the <c>ack</c> message +below.</p> </item> <tag><c>{diameter, {close, Pid}}</c></tag> @@ -246,6 +250,27 @@ A <c>LocalAddr</c> list has the same semantics as one returned from &start;.</p> </item> +<tag><c>{diameter, ack}</c></tag> +<item> +<p> +Request acknowledgements of unanswered requests. +A transport process should send this once before passing incoming +Diameter messages into diameter. +As a result, every Diameter request passed into diameter with a +<c>recv</c> message (below) will be answered with a +<c>send</c> message (above), either a &message; for the transport +process to send or the atom <c>false</c> if the request has been +discarded or otherwise not answered.</p> + +<p> +This is to allow a transport process to keep count of the number +of incoming request messages that have not yet been answered or +discarded, to allow it to regulate the amount of incoming traffic. +Both diameter_tcp and diameter_sctp request acknowledgements when a +<c>message_cb</c> is configured, turning send/recv message into +callbacks that can be used to regulate traffic.</p> +</item> + <tag><c>{diameter, {recv, &message;}}</c></tag> <item> <p> diff --git a/lib/diameter/src/base/diameter.erl b/lib/diameter/src/base/diameter.erl index b90b794611..7f172e1fa1 100644 --- a/lib/diameter/src/base/diameter.erl +++ b/lib/diameter/src/base/diameter.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% Copyright Ericsson AB 2010-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -365,7 +365,7 @@ call(SvcName, App, Message) -> | {connect_timer, 'Unsigned32'()} | {watchdog_timer, 'Unsigned32'() | {module(), atom(), list()}} | {watchdog_config, [{okay|suspect, non_neg_integer()}]} - | {spawn_opt, list()}. + | {spawn_opt, list() | mfa()}. %% Options passed to start_service/2 diff --git a/lib/diameter/src/base/diameter_callback.erl b/lib/diameter/src/base/diameter_callback.erl index d04a416bef..3bcf550cd8 100644 --- a/lib/diameter/src/base/diameter_callback.erl +++ b/lib/diameter/src/base/diameter_callback.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% Copyright Ericsson AB 2010-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -70,7 +70,7 @@ -module(diameter_callback). -%% Default callbacks when no aleternate is specified. +%% Default callbacks when no alternate is specified. -export([peer_up/3, peer_down/3, pick_peer/4, diff --git a/lib/diameter/src/base/diameter_dist.erl b/lib/diameter/src/base/diameter_dist.erl new file mode 100644 index 0000000000..5c29ea95a4 --- /dev/null +++ b/lib/diameter/src/base/diameter_dist.erl @@ -0,0 +1,525 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(diameter_dist). + +-behaviour(gen_server). + +%% +%% Implements callbacks that can be configured as a spawn_opt +%% transport configuration, to be able to distribute incoming Diameter +%% requests to handler processes (local or remote) in various ways. +%% + +%% spawn_opt callbacks +-export([spawn_local/2, + spawn_local/1, + route_session/2, + route_session/1]). + +%% signal availability for handling incoming requests to route_sesssion/2 +-export([attach/1, + detach/1]). + +%% consistent hashing +-export([hash/3, %% for use as default MFA in route_session/2 options map + hash/2]). %% arbitrary key/values + +-include_lib("diameter/include/diameter.hrl"). + +%% server start +-export([start_link/0]). + +%% gen_server callbacks +-export([init/1, + handle_info/2, + handle_cast/2, + handle_call/3, + code_change/3, + terminate/2]). + +-type request() :: tuple(). %% callback argument from diameter_traffic + +-define(SERVER, ?MODULE). %% server monitoring node connections + +%% Maps a node name binary to the corresponding atom. Used by +%% route_session/2 to map the optional value of a Session-Id to +%% node(). +-define(NODE_TABLE, diameter_dist_node). + +%% Maps a diameter:service_name() to a node() that has called attach/1 +%% to declare its willingness to handle incoming requests for the +%% service. Use by route_session/2 in case the optional value mapping +%% has failed. +-define(SERVICE_TABLE, diameter_dist_service). + +-define(B(A), atom_to_binary(A, utf8)). +-define(ORCOND(List), list_to_tuple(['orelse', false | List])). +-define(HASH(T), erlang:phash2(T, 16#100000000)). + +%% spawn_local/2 +%% +%% Callback that is equivalent to an options list. That is, the +%% following are equivalent when passed as options to +%% diameter:add_transport/2. +%% +%% {spawn_opt, Opts} +%% {spawn_opt, {diameter_dist, spawn_local, [Opts]}} + +-spec spawn_local(ReqT :: request(), Opts :: list()) + -> pid(). + +spawn_local(ReqT, Opts) -> + spawn_opt(diameter_traffic, request, [ReqT], Opts). + +%% spawn_local/1 + +spawn_local(ReqT) -> + spawn_local(ReqT, []). + +%% route_session/2 +%% +%% Callback that maps the Session-Id of an incoming request to a +%% handler node. +%% +%% With an options list, maps an id whose optional value is the name +%% of a connected node to the same node, to handle the case that the +%% session id has been returned from diameter:session_id/1; otherwise +%% to a node that has called diameter_dist:attach/1 using the +%% consistent hashing provided by hash/3, or to the local node() if a +%% session id could not be extracted or there are no attached nodes. A +%% handler process is spawned on the selected node using +%% erlang:spawn_opt/4. +%% +%% Different behaviour can be configured by supplying an options map +%% of the following form: +%% +%% #{search => non_neg_integer(), +%% id => [binary()], +%% default => discard | local | mfa(), +%% dispatch => list() | mfa()} +%% +%% The search member limits the number of AVPs that are examined in +%% the message (from the front), to avoid searching entire message in +%% case it's known that peers follow RFC 6733's recommendation that +%% Session-Id be placed at the head of a message. The default is to +%% search the entire message. +%% +%% The id member restricts the optional value mapping to session ids +%% whose DiamterIdentity is one of those specified. Set this to the +%% list of Diameter identities advertised by the service in question +%% (typically one) to ensure that only locally generated session ids +%% are mapped; or to the empty list to disable the mapping. +%% +%% The default member determines where to handle a message whose +%% Session-Id isn't found or whose optional value isn't mapped to the +%% name of a connected node. The atom local says the local node, an +%% MFA is invoked on Session-Id | false, the name of the diameter +%% service, and the message binary, and should return either a node() +%% or false to discard the message. Defaults to {diameter_dist, hash, []}. +%% +%% The dispatch member determines how the pid() of the request handler +%% process is retrieved. An MFA is applied to a previously selected +%% node(), and the module, function, and arguments list to apply in +%% the handler process to handle the request, the MFA being supplied +%% by diameter, and returns pid() | discard. A list is equivalent to +%% {erlang, spawn_opt, []}. Defaults to []. +%% +%% This can be used with search = 0 to route on something other than +%% Session-Id, but this is probably no simpler than just implementing +%% an own spawn_opt callback. (Except with the default dispatch possibly.) +%% +%% Note that if the peer is also implemented with OTP diameter and +%% generating session ids with diameter:session_id/1 then +%% route_session/2 can map an optional value to a local node that +%% happens to have the same name as one of the peer's nodes. This +%% could lead to an uneven distribution; for example, if the peer +%% nodes are a subset of the local nodes. In practice, it's typically +%% known if it's peers or the local node originating sessions; if the +%% former then setting id = [] disables the optional value mapping, if +%% the latter then setting default = local disables the hashing. +-spec route_session(ReqT :: request(), Opts) + -> discard + | pid() + when Opts :: pos_integer() %% aka #{search => N} + | list() %% aka #{dispatch => Opts} + | #{search => non_neg_integer(), %% limit number of examined AVPs + id => [binary()], %% restrict optional value map on DiamIdent + default => local %% handle locally + | discard + | mfa(), %% return node() | false + dispatch => list() %% spawn options + | mfa()}. %% (Node, M, F, A) -> pid() | discard + +route_session(ReqT, Opts) -> + {_, Bin} = Info = diameter_traffic:request_info(ReqT), + Sid = session_id(avps(Bin), search(Opts)), + Node = default(node_of_session_id(Sid, Opts), Sid, Opts, Info), + dispatch(Node, ReqT, dispatch(Opts)). + +%% avps/1 + +avps(<<_:20/binary, Bin/binary>>) -> + Bin; + +avps(_) -> + false. + +%% dispatch/3 + +dispatch(false, _, _) -> + discard; + +dispatch(Node, ReqT, {M,F,A}) -> + apply(M, F, [Node, diameter_traffic, request, [ReqT] | A]); + +dispatch(Node, ReqT, Opts) -> + spawn_opt(Node, diameter_traffic, request, [ReqT], Opts). + +%% route_session/1 + +route_session(ReqT) -> + route_session(ReqT, []). + +%% node_of_session_id/2 +%% +%% Return the node name encoded as optional value in a Session-Id, +%% assuming the id has been created with diameter:session_id/0. Lookup +%% the node name to ensure we don't convert arbitrary binaries to +%% atom. + +node_of_session_id([Id, _, _, Bin], #{id := Ids}) -> + lists:member(Id, Ids) andalso nodemap(Bin); + +node_of_session_id([_, _, _, Bin], _) -> + nodemap(Bin); + +node_of_session_id(_, _) -> + false. + +%% nodemap/1 + +nodemap(Bin) -> + try + ets:lookup_element(?NODE_TABLE, Bin, 2) + catch + error: badarg -> false + end. + +%% session_id/2 + +session_id(_, 0) -> %% give up + false; + +%% Session-Id = Command Code 263, V-bit = 0. +session_id(<<263:32, 0:1, _:7, Len:24, _/binary>> = Bin, _) -> + case Bin of + <<Avp:Len/binary, _/binary>> -> + <<_:8/binary, Sid/binary>> = Avp, + split(Sid); + _ -> + false + end; + +%% Jump to the next AVP. This is potentially costly for a message with +%% many AVPs and no Session-Id, which an attacker is prone to send. +%% 8.8 or RFC 6733 says that Session-Id SHOULD (but not MUST) appear +%% immediately following the Diameter Header, so there is no +%% guarantee. +session_id(<<_:40, Len:24, _/binary>> = Bin, N) -> + Pad = (4 - (Len rem 4)) rem 4, + case Bin of + <<_:Len/binary, _:Pad/binary, Rest/binary>> -> + session_id(Rest, if N == infinity -> N; true -> N-1 end); + _ -> + false + end; + +session_id(_, _) -> + false. + +%% split/1 +%% +%% Split a Session-Id at no more than three semicolons: the optional +%% value (if any) follows the third. binary:split/2 does better than +%% matching character by character, especially when the pattern is +%% compiled. + +split(Bin) -> + split(3, Bin, pattern()). + +%% split/3 + +split(0, Bin, _) -> + [Bin]; + +split(N, Bin, Pattern) -> + [H|T] = binary:split(Bin, Pattern), + [H | case T of + [] -> + T; + [Rest] -> + split(N-1, Rest, Pattern) + end]. + +%% pattern/0 +%% +%% Since this is being called in a watchdog process, compile the +%% pattern once and maintain it in the process dictionary. + +pattern() -> + case get(?MODULE) of + undefined -> + CP = binary:compile_pattern(<<$;>>), %% tuple + put(?MODULE, CP), + CP; + CP -> + CP + end. + +%% dispatch/1 + +dispatch(#{} = Opts) -> + maps:get(dispatch, Opts, []); + +dispatch(Opts) + when is_list(Opts) -> + Opts; + +dispatch(_) -> + []. + +%% search/1 +%% +%% Bound number of AVPs examined when looking for Session-Id. + +search(#{search := N}) + when is_integer(N), 0 =< N -> + N; + +search(N) + when is_integer(N), 0 =< N -> + N; + +search(_) -> + infinity. + +%% default/3 +%% +%% Choose a node when Session-Id lookup has failed. + +default(false, _, #{default := discard}, _) -> + false; + +default(false, _, #{default := local}, _) -> + node(); + +default(false, Sid, #{default := {M,F,A}}, Info) -> + {ServiceName, Bin} = Info, + apply(M, F, [Sid, ServiceName, Bin | A]); %% node() | false + +default(false, Sid, _, Info) -> %% aka {?MODULE, hash, []} + {ServiceName, Bin} = Info, + hash(Sid, ServiceName, Bin); + +default(Node, _, _, _) -> + Node. + +%% =========================================================================== + +%% hash/3 +%% +%% Consistent hashing of Session-Id to an attached node, or the local +%% node if Session-Id = false or no attached nodes. + +hash(Sid, ServiceName, _) -> + case false /= Sid andalso attached(ServiceName) of + [_|_] = Nodes -> + hash(Sid, Nodes); + _ -> + node() + end. + +%% hash/2 +%% +%% Consistent hashing on arbitrary key/values. Returns false if the +%% list is empty. + +%% No key or no values. +hash(_, []) -> + false; + +%% Not much choice. +hash(_, [Value]) -> + Value; + +%% Hash on a circle and choose the closest predecessor. +hash(Key, Values) -> + Hash = ?HASH(Key), + tl(lists:foldl(fun(V,A) -> + choose(Hash, [?HASH({Key, V}) | V], A) + end, + false, %% < list() + Values)). + +%% choose/3 + +choose(Hash, [Hash1 | _] = T, [Hash2 | _]) + when Hash1 =< Hash, Hash < Hash2 -> + T; + +choose(Hash, [Hash1 | _], [Hash2 | _] = T) + when Hash2 =< Hash, Hash < Hash1 -> + T; + +choose(_, T1, T2) -> + max(T1, T2). + +%% =========================================================================== + +%% attach/1 +%% +%% Register the local node as a handler of incoming requests for the +%% specified services when using the route_session/2 spawn_opt +%% callback. + +attach(ServiceNames) -> + abcast({attach, node(), ServiceNames}). + +%% detach/1 +%% +%% Deregister the local node as a handler of incoming requests. + +detach(ServiceNames) -> + abcast({detach, node(), ServiceNames}). + +%% abcast/1 + +abcast(T) -> + gen_server:abcast([node() | nodes()], ?SERVER, T), + ok. + +%% attached/1 + +attached(ServiceName) -> + try + ets:lookup_element(?SERVICE_TABLE, ServiceName, 2) + catch + error: badarg -> [] + end. + +%% cast/2 + +cast(Node, T) -> + gen_server:cast({?SERVER, Node}, T). + +%% attach/2 + +attach(Node, S) -> + case sets:to_list(S) of + [] -> + ok; + Services -> + cast(Node, {attach, node(), Services}) + end. + +%% =========================================================================== + +start_link() -> + gen_server:start_link({local, ?SERVER}, ?MODULE, _Args = [], _Opts = []). + +%% init/1 +%% +%% Maintain [node() | nodes()] in a table that maps from binary-valued +%% names, so we can lookup the corresponding atoms rather than convert +%% binaries that aren't necessarily node names. + +init([]) -> + ets:new(?NODE_TABLE, [set, named_table]), + ets:new(?SERVICE_TABLE, [bag, named_table]), + ok = net_kernel:monitor_nodes(true, [{node_type, all}, nodedown_reason]), + ets:insert(?NODE_TABLE, [{?B(N), N} || N <- [node() | nodes()]]), + abcast({attach, node()}), + {ok, sets:new()}. + +%% handle_call/3 + +handle_call(_, _From, S) -> + {reply, nok, S}. + +%% handle_cast/2 + +%% Remote node is asking which services the local node wants to handle. +handle_cast({attach, Node}, S) + when Node /= node() -> + attach(Node, S), + {noreply, S}; + +%% Node wants to handle incoming requests ... +handle_cast({attach, Node, ServiceNames}, S) -> + ets:insert(?SERVICE_TABLE, [{N, Node} || N <- ServiceNames]), + {noreply, case node() of + Node -> + sets:union(S, sets:from_list(ServiceNames)); + _ -> + S + end}; + +%% ... or not. +handle_cast({detach, Node, ServiceNames}, S) -> + ets:select_delete(?SERVICE_TABLE, [{{'$1', Node}, + [?ORCOND([{'==', '$1', {const, N}} + || N <- ServiceNames])], + [true]}]), + {noreply, case node() of + Node -> + sets:subtract(S, sets:from_list(ServiceNames)); + _ -> + S + end}; + +handle_cast(_, S) -> + {noreply, S}. + +%% handle_info/2 + +handle_info({nodeup, Node, _}, S) -> + ets:insert(?NODE_TABLE, {?B(Node), Node}), + cast(Node, {attach, node()}), %% ask which services remote node handles + attach(Node, S), %% say which service local node handles + {noreply, S}; + +handle_info({nodedown, Node, _}, S) -> + ets:delete(?NODE_TABLE, ?B(Node)), + ets:select_delete(?SERVICE_TABLE, [{{'_', Node}, [], [true]}]), + {noreply, S}; + +handle_info(_, S) -> + {noreply, S}. + +%% terminate/2 + +terminate(_, _) -> + ok. + +%% code_change/3 + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. diff --git a/lib/diameter/src/base/diameter_misc_sup.erl b/lib/diameter/src/base/diameter_misc_sup.erl index 343688be23..fec5a41b5c 100644 --- a/lib/diameter/src/base/diameter_misc_sup.erl +++ b/lib/diameter/src/base/diameter_misc_sup.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -35,6 +35,7 @@ diameter_stats, %% statistics counter management diameter_reg, %% service/property publishing diameter_peer, %% remote peer manager + diameter_dist, %% request distribution diameter_config]). %% configuration/restart %% start_link/0 diff --git a/lib/diameter/src/base/diameter_traffic.erl b/lib/diameter/src/base/diameter_traffic.erl index 2d3e4a2ac9..8423e30269 100644 --- a/lib/diameter/src/base/diameter_traffic.erl +++ b/lib/diameter/src/base/diameter_traffic.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2017. All Rights Reserved. +%% Copyright Ericsson AB 2013-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -42,8 +42,12 @@ peer_up/1, peer_down/1]). +%% towards diameter_dist +-export([request_info/1]). + %% internal -export([send/1, %% send from remote node + request/1, %% process request in handler process init/1]). %% monitor process start -include_lib("diameter/include/diameter.hrl"). @@ -232,7 +236,7 @@ incr_rc(Dir, Pkt, TPid, MsgDict, AppDict, Dict0) -> -spec receive_message(pid(), Route, #diameter_packet{}, module(), RecvData) -> pid() %% request handler | boolean() %% answer, known request or not - | discard %% request discarded by MFA + | discard %% request discarded when Route :: {Handler, RequestRef, TPid} | Ack, RecvData :: {[SpawnOpt], #recvdata{}}, @@ -252,7 +256,8 @@ receive_message(TPid, Route, Pkt, Dict0, RecvData) -> recv(true, Ack, TPid, Pkt, Dict0, T) when is_boolean(Ack) -> {Opts, RecvData} = T, - spawn_request(Ack, TPid, Pkt, Dict0, RecvData, Opts); + AppT = find_app(TPid, Pkt, RecvData), + ack(Ack, TPid, spawn_request(AppT, Opts, Ack, TPid, Pkt, Dict0, RecvData)); %% ... answer to known request ... recv(false, {Pid, Ref, TPid}, _, Pkt, Dict0, _) -> @@ -274,58 +279,73 @@ recv(false, false, TPid, Pkt, _, _) -> incr(TPid, {{unknown, 0}, recv, discarded}), false. -%% spawn_request/6 +%% spawn_request/7 + +spawn_request(false, _, _, _, _, _, _) -> %% no transport + discard; -%% An MFA should return a pid() or the atom 'discard'. The latter -%% results in an acknowledgment back to the transport process when -%% appropriate, to ensure that send/recv callbacks can count -%% outstanding requests. Acknowledgement is implicit if the +%% An MFA should return the pid() of a process in which the argument +%% fun in applied, or the atom 'discard' if the fun is not applied. +%% The latter results in an acknowledgment back to the transport +%% process when appropriate, to ensure that send/recv callbacks can +%% count outstanding requests. Acknowledgement is implicit if the %% handler process dies (in a handle_request callback for example). -spawn_request(Ack, TPid, Pkt, Dict0, RecvData, {M,F,A}) -> - ReqF = fun() -> - ack(Ack, TPid, recv_request(Ack, TPid, Pkt, Dict0, RecvData)) - end, - ack(Ack, TPid, apply(M, F, [ReqF | A])); +spawn_request(AppT, {M,F,A}, Ack, TPid, Pkt, Dict0, RecvData) -> + %% Term to pass to request/1 in an appropriate process. Module + %% diameter_dist implements callbacks. + ReqT = {Pkt, AppT, Ack, TPid, Dict0, RecvData}, + apply(M, F, [ReqT | A]); %% A spawned process acks implicitly when it dies, so there's no need %% to handle 'discard'. -spawn_request(Ack, TPid, Pkt, Dict0, RecvData, Opts) -> +spawn_request(AppT, Opts, Ack, TPid, Pkt, Dict0, RecvData) -> spawn_opt(fun() -> - recv_request(Ack, TPid, Pkt, Dict0, RecvData) + recv_request(Ack, TPid, Pkt, Dict0, RecvData, AppT) end, Opts). +%% request_info/1 +%% +%% Limited request information for diameter_dist. + +request_info({Pkt, _AppT, _Ack, _TPid, _Dict0, RecvData} = _ReqT) -> + {RecvData#recvdata.service_name, Pkt#diameter_packet.bin}. + +%% request/1 +%% +%% Called from a handler process chosen by a transport spawn_opt MFA +%% to process an incoming request. + +request({Pkt, AppT, Ack, TPid, Dict0, RecvData} = _ReqT) -> + ack(Ack, TPid, recv_request(Ack, TPid, Pkt, Dict0, RecvData, AppT)). + %% ack/3 ack(Ack, TPid, RC) -> - RC == discard andalso Ack andalso (TPid ! {send, false}), + RC == discard + andalso Ack + andalso (TPid ! {send, false}), RC. %% --------------------------------------------------------------------------- -%% recv_request/5 +%% recv_request/6 %% --------------------------------------------------------------------------- -spec recv_request(Ack :: boolean(), TPid :: pid(), #diameter_packet{}, Dict0 :: module(), - #recvdata{}) + #recvdata{}, + AppT :: {#diameter_app{}, #diameter_caps{}} + | #diameter_caps{}) %% no suitable app -> ok %% answer was sent - | discard %% or not - | false. %% no transport - -recv_request(Ack, - TPid, - #diameter_packet{header = #diameter_header{application_id = Id}} - = Pkt, - Dict0, - #recvdata{peerT = PeerT, - apps = Apps, - counters = Count} - = RecvData) -> + | discard. %% or not + +recv_request(Ack, TPid, Pkt, Dict0, RecvData, AppT) -> Ack andalso (TPid ! {handler, self()}), - case diameter_service:find_incoming_app(PeerT, TPid, Id, Apps) of + case AppT of {#diameter_app{id = Aid, dictionary = AppDict} = App, Caps} -> + Count = RecvData#recvdata.counters, Count andalso incr(recv, Pkt, TPid, AppDict), DecPkt = decode(Aid, AppDict, RecvData, Pkt), Count andalso incr_error(recv, DecPkt, TPid, AppDict), @@ -349,11 +369,20 @@ recv_request(Ack, Dict0, RecvData, DecPkt, - [[]]); - false = No -> %% transport has gone down - No + [[]]) end. +%% find_app/3 +%% +%% Lookup the application of a received Diameter request on the node +%% on which it's received. + +find_app(TPid, + #diameter_packet{header = #diameter_header{application_id = Id}}, + #recvdata{peerT = PeerT, + apps = Apps}) -> + diameter_service:find_incoming_app(PeerT, TPid, Id, Apps). + %% decode/4 decode(Id, Dict, #recvdata{codec = Opts}, Pkt) -> diff --git a/lib/diameter/src/diameter.appup.src b/lib/diameter/src/diameter.appup.src index 4e6b983bac..52263633fb 100644 --- a/lib/diameter/src/diameter.appup.src +++ b/lib/diameter/src/diameter.appup.src @@ -60,7 +60,8 @@ {"2.1.3", [{restart_application, diameter}]}, %% 20.2 {"2.1.4", [{restart_application, diameter}]}, %% 20.3 {"2.1.4.1", [{restart_application, diameter}]}, %% 20.3.8.19 - {"2.1.5", [{update, diameter_peer_fsm}]} %% 21.0 + {"2.1.5", [{restart_application, diameter}]}, %% 21.0 + {"2.1.6", [{restart_application, diameter}]} %% 21.1 ], [ {"0.9", [{restart_application, diameter}]}, @@ -102,6 +103,7 @@ {"2.1.3", [{restart_application, diameter}]}, {"2.1.4", [{restart_application, diameter}]}, {"2.1.4.1", [{restart_application, diameter}]}, - {"2.1.5", [{update, diameter_peer_fsm}]} + {"2.1.5", [{restart_application, diameter}]}, + {"2.1.6", [{restart_application, diameter}]} ] }. diff --git a/lib/diameter/src/modules.mk b/lib/diameter/src/modules.mk index bb86de016a..d16292bb88 100644 --- a/lib/diameter/src/modules.mk +++ b/lib/diameter/src/modules.mk @@ -1,7 +1,7 @@ # %CopyrightBegin% # -# Copyright Ericsson AB 2010-2017. All Rights Reserved. +# Copyright Ericsson AB 2010-2019. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -40,6 +40,7 @@ RT_MODULES = \ base/diameter_config \ base/diameter_config_sup \ base/diameter_codec \ + base/diameter_dist \ base/diameter_gen \ base/diameter_lib \ base/diameter_misc_sup \ diff --git a/lib/diameter/test/diameter_dist_SUITE.erl b/lib/diameter/test/diameter_dist_SUITE.erl new file mode 100644 index 0000000000..b2e4c35b9a --- /dev/null +++ b/lib/diameter/test/diameter_dist_SUITE.erl @@ -0,0 +1,332 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +%% Tests of traffic between two Diameter nodes, the server being +%% spread across three Erlang nodes. +%% + +-module(diameter_dist_SUITE). + +-export([suite/0, + all/0]). + +%% testcases +-export([enslave/1, enslave/0, + ping/1, + start/1, + connect/1, + send/1, + stop/1, stop/0]). + +%% diameter callbacks +-export([peer_up/3, + peer_down/3, + pick_peer/4, + prepare_request/3, + prepare_retransmit/3, + handle_answer/4, + handle_error/4, + handle_request/3]). + +-export([call/1]). + +-include("diameter.hrl"). +-include("diameter_gen_base_rfc6733.hrl"). + +%% =========================================================================== + +-define(util, diameter_util). + +-define(CLIENT, 'CLIENT'). +-define(SERVER, 'SERVER'). +-define(REALM, "erlang.org"). +-define(DICT, diameter_gen_base_rfc6733). +-define(ADDR, {127,0,0,1}). + +%% Config for diameter:start_service/2. +-define(SERVICE(Host), + [{'Origin-Host', Host ++ [$.|?REALM]}, + {'Origin-Realm', ?REALM}, + {'Host-IP-Address', [?ADDR]}, + {'Vendor-Id', 12345}, + {'Product-Name', "OTP/diameter"}, + {'Auth-Application-Id', [?DICT:id()]}, + {'Origin-State-Id', origin()}, + {spawn_opt, {diameter_dist, route_session, [#{id => []}]}}, + {sequence, fun sequence/0}, + {string_decode, false}, + {application, [{dictionary, ?DICT}, + {module, ?MODULE}, + {request_errors, callback}, + {answer_errors, callback}]}]). + +-define(SUCCESS, 2001). +-define(BUSY, 3004). +-define(LOGOUT, ?'DIAMETER_BASE_TERMINATION-CAUSE_LOGOUT'). +-define(MOVED, ?'DIAMETER_BASE_TERMINATION-CAUSE_USER_MOVED'). +-define(TIMEOUT, ?'DIAMETER_BASE_TERMINATION-CAUSE_SESSION_TIMEOUT'). + +-define(L, atom_to_list). +-define(A, list_to_atom). + +%% The order here is significant and causes the server to listen +%% before the clients connect. The server listens on the first node, +%% and distributes requests to the other two. +-define(NODES, [{server0, ?SERVER}, + {server1, ?SERVER}, + {server2, ?SERVER}, + {client, ?CLIENT}]). + +%% Options to ct_slave:start/2. +-define(TIMEOUTS, [{T, 15000} || T <- [boot_timeout, + init_timeout, + start_timeout]]). + +%% =========================================================================== + +suite() -> + [{timetrap, {seconds, 60}}]. + +all() -> + [enslave, + ping, + start, + connect, + send, + stop]. + +%% =========================================================================== +%% start/stop testcases + +%% enslave/1 +%% +%% Start four slave nodes, three to implement a Diameter server, +%% one to implement a client. + +enslave() -> + [{timetrap, {seconds, 30*length(?NODES)}}]. + +enslave(Config) -> + Here = filename:dirname(code:which(?MODULE)), + Ebin = filename:join([Here, "..", "ebin"]), + Dirs = [Here, Ebin], + Nodes = [{N,S} || {M,S} <- ?NODES, N <- [slave(M, Dirs)]], + ?util:write_priv(Config, nodes, [{N,S} || {{N,ok},S} <- Nodes]), + [] = [{T,S} || {{_,E} = T, S} <- Nodes, E /= ok]. + +slave(Name, Dirs) -> + add_pathsa(Dirs, ct_slave:start(Name, ?TIMEOUTS)). + +add_pathsa(Dirs, {ok, Node}) -> + {Node, rpc:call(Node, code, add_pathsa, [Dirs])}; +add_pathsa(_, No) -> + {No, error}. + +%% ping/1 +%% +%% Ensure the server nodes are connected so that diameter_dist can attach. + +ping({S, Nodes}) -> + ?SERVER = S, + [N || {N,_} <- Nodes, + node() /= N, + pang <- [net_adm:ping(N)]]; + +ping(Config) -> + Nodes = lists:droplast(?util:read_priv(Config, nodes)), + [] = [{N,RC} || {N,S} <- Nodes, + RC <- [rpc:call(N, ?MODULE, ping, [{S,Nodes}])], + RC /= []]. + +%% start/1 +%% +%% Start diameter services. + +start(SvcName) + when is_atom(SvcName) -> + ok = diameter:start(), + ok = diameter:start_service(SvcName, ?SERVICE((?L(SvcName)))); + +start(Config) -> + Nodes = ?util:read_priv(Config, nodes), + [] = [{N,RC} || {N,S} <- Nodes, + RC <- [rpc:call(N, ?MODULE, start, [S])], + RC /= ok]. + +sequence() -> + sequence(sname()). + +sequence(client) -> + {0,32}; +sequence(Server) -> + "server" ++ N = ?L(Server), + {list_to_integer(N), 30}. + +origin() -> + origin(sname()). + +origin(client) -> + 99; +origin(Server) -> + "server" ++ N = ?L(Server), + list_to_integer(N). + +%% connect/1 +%% +%% Establish one connection from the client, terminated on the first +%% server node, the others handling requests. + +connect({?SERVER, Config, [{Node, _} | _]}) -> + if Node == node() -> %% server0 + ?util:write_priv(Config, lref, {Node, ?util:listen(?SERVER, tcp)}); + true -> + diameter_dist:attach([?SERVER]) + end, + ok; + +connect({?CLIENT, Config, _}) -> + ?util:connect(?CLIENT, tcp, ?util:read_priv(Config, lref)), + ok; + +connect(Config) -> + Nodes = ?util:read_priv(Config, nodes), + [] = [{N,RC} || {N,S} <- Nodes, + RC <- [rpc:call(N, ?MODULE, connect, [{S, Config, Nodes}])], + RC /= ok]. + +%% stop/1 +%% +%% Stop the slave nodes. + +stop() -> + [{timetrap, {seconds, 30*length(?NODES)}}]. + +stop(_Config) -> + [] = [{N,E} || {N,_} <- ?NODES, + {error, _, _} = E <- [ct_slave:stop(N)]]. + +%% =========================================================================== +%% traffic testcases + +%% send/1 +%% +%% Send 100 requests and ensure the node name sent as User-Name isn't +%% the node terminating transport. + +send(Config) -> + send(Config, 100, dict:new()). + +%% send/2 + +send(Config, 0, Dict) -> + [{Server0, _} | _] = ?util:read_priv(Config, nodes) , + Node = atom_to_binary(Server0, utf8), + {false, _} = {dict:is_key(Node, Dict), dict:to_list(Dict)}; + +send(Config, N, Dict) -> + #diameter_base_STA{'Result-Code' = ?SUCCESS, + 'User-Name' = [ServerNode]} + = send(Config, str(?LOGOUT)), + true = is_binary(ServerNode), + send(Config, N-1, dict:update_counter(ServerNode, 1, Dict)). + +%% =========================================================================== + +str(Cause) -> + #diameter_base_STR{'Destination-Realm' = ?REALM, + 'Auth-Application-Id' = ?DICT:id(), + 'Termination-Cause' = Cause}. + +%% send/2 + +send(Config, Req) -> + {Node, _} = lists:last(?util:read_priv(Config, nodes)), + rpc:call(Node, ?MODULE, call, [Req]). + +%% call/1 + +call(Req) -> + diameter:call(?CLIENT, ?DICT, Req, []). + +%% sname/0 + +sname() -> + ?A(hd(string:tokens(?L(node()), "@"))). + +%% =========================================================================== +%% diameter callbacks + +%% peer_up/3 + +peer_up(_SvcName, _Peer, State) -> + State. + +%% peer_down/3 + +peer_down(_SvcName, _Peer, State) -> + State. + +%% pick_peer/4 + +pick_peer([Peer], [], ?CLIENT, _State) -> + {ok, Peer}. + +%% prepare_request/3 + +prepare_request(Pkt, ?CLIENT, {_Ref, Caps}) -> + #diameter_packet{msg = Req} + = Pkt, + #diameter_caps{origin_host = {OH, _}, + origin_realm = {OR, _}} + = Caps, + {send, Req#diameter_base_STR{'Origin-Host' = OH, + 'Origin-Realm' = OR, + 'Session-Id' = diameter:session_id(OH)}}. + +%% prepare_retransmit/3 + +prepare_retransmit(_, ?CLIENT, _) -> + discard. + +%% handle_answer/5 + +handle_answer(Pkt, _Req, ?CLIENT, _Peer) -> + #diameter_packet{msg = Rec, errors = []} = Pkt, + Rec. + +%% handle_error/5 + +handle_error(Reason, _Req, ?CLIENT, _Peer) -> + {error, Reason}. + +%% handle_request/3 + +handle_request(Pkt, ?SERVER, {_, Caps}) -> + #diameter_packet{msg = #diameter_base_STR{'Session-Id' = SId}} + = Pkt, + #diameter_caps{origin_host = {OH, _}, + origin_realm = {OR, _}} + = Caps, + {reply, #diameter_base_STA{'Result-Code' = ?SUCCESS, + 'Session-Id' = SId, + 'Origin-Host' = OH, + 'Origin-Realm' = OR, + 'User-Name' = [atom_to_binary(node(), utf8)]}}. diff --git a/lib/diameter/test/diameter_distribution_SUITE.erl b/lib/diameter/test/diameter_distribution_SUITE.erl index 5146f68ff1..5fe02284ae 100644 --- a/lib/diameter/test/diameter_distribution_SUITE.erl +++ b/lib/diameter/test/diameter_distribution_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2016. All Rights Reserved. +%% Copyright Ericsson AB 2013-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -76,6 +76,7 @@ {share_peers, peers()}, {use_shared_peers, peers()}, {restrict_connections, false}, + {spawn_opt, {diameter_dist, spawn_local, []}}, {sequence, fun sequence/0}, {application, [{dictionary, ?DICT}, {module, ?MODULE}, @@ -125,7 +126,7 @@ all() -> %% enslave/1 %% %% Start four slave nodes, one to implement a Diameter server, -%% two three to implement a client. +%% three to implement a client. enslave() -> [{timetrap, {seconds, 30*length(?NODES)}}]. @@ -331,6 +332,8 @@ prepare_request(Pkt, ?CLIENT, {_Ref, Caps}, {_, client0}) -> 'Origin-Realm' = OR, 'Session-Id' = diameter:session_id(OH)}}. +%% prepare_retransmit/4 + prepare_retransmit(Pkt, ?CLIENT, _, {_, client0}) -> #diameter_packet{msg = #diameter_base_STR{'Termination-Cause' = ?MOVED}} = Pkt, %% assert diff --git a/lib/diameter/test/diameter_pool_SUITE.erl b/lib/diameter/test/diameter_pool_SUITE.erl index 97c16940ff..a36a4fa17a 100644 --- a/lib/diameter/test/diameter_pool_SUITE.erl +++ b/lib/diameter/test/diameter_pool_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2015-2017. All Rights Reserved. +%% Copyright Ericsson AB 2015-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -51,6 +51,7 @@ {'Auth-Application-Id', [0]}, %% common {'Acct-Application-Id', [3]}, %% accounting {restrict_connections, false}, + {spawn_opt, {diameter_dist, route_session, []}}, {application, [{alias, common}, {dictionary, diameter_gen_base_rfc6733}, {module, diameter_callback}]}, diff --git a/lib/diameter/test/diameter_traffic_SUITE.erl b/lib/diameter/test/diameter_traffic_SUITE.erl index 434aef01dd..47b00c25a2 100644 --- a/lib/diameter/test/diameter_traffic_SUITE.erl +++ b/lib/diameter/test/diameter_traffic_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2018. All Rights Reserved. +%% Copyright Ericsson AB 2010-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -539,8 +539,7 @@ add_transports(Config) -> ++ [{unordered, unordered()} || T == sctp], [{capabilities_cb, fun capx/2}, {pool_size, 8} - | server_apps()] - ++ [{spawn_opt, {erlang, spawn, []}} || CS]), + | server_apps()]), Cs = [?util:connect(CN, [T, {sender, CS} | client_opts(T)], LRef, diff --git a/lib/diameter/test/modules.mk b/lib/diameter/test/modules.mk index 0c73adca12..90b0a25d5f 100644 --- a/lib/diameter/test/modules.mk +++ b/lib/diameter/test/modules.mk @@ -1,7 +1,7 @@ # %CopyrightBegin% # -# Copyright Ericsson AB 2010-2017. All Rights Reserved. +# Copyright Ericsson AB 2010-2019. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -31,6 +31,7 @@ MODULES = \ diameter_codec_test \ diameter_config_SUITE \ diameter_compiler_SUITE \ + diameter_dist_SUITE \ diameter_distribution_SUITE \ diameter_dpr_SUITE \ diameter_event_SUITE \ diff --git a/lib/erl_interface/Makefile b/lib/erl_interface/Makefile index 5fff2cabdd..9471b0df18 100644 --- a/lib/erl_interface/Makefile +++ b/lib/erl_interface/Makefile @@ -23,11 +23,7 @@ include $(ERL_TOP)/make/$(TARGET)/otp.mk # ---------------------------------------------------- # Common Macros # ---------------------------------------------------- -ifdef TERTIARY_BOOTSTRAP -SUB_DIRECTORIES = -else SUB_DIRECTORIES = src doc/src -endif SPECIAL_TARGETS = diff --git a/lib/erl_interface/configure.in b/lib/erl_interface/configure.in index 67acb7c180..f0e9b2eb3f 100644 --- a/lib/erl_interface/configure.in +++ b/lib/erl_interface/configure.in @@ -61,7 +61,7 @@ fi TARGET=$host AC_SUBST(TARGET) -AC_CONFIG_HEADER([src/$host/config.h:config.h.in include/$host/ei_config.h:include/ei_config.h.in]) +AC_CONFIG_HEADER([src/$host/config.h:config.h.in]) dnl ---------------------------------------------------------------------- dnl Optional features @@ -184,14 +184,6 @@ AC_TRY_COMPILE([#include <sys/types.h> AC_DEFINE(HAVE_SOCKLEN_T, [], [Define if you have the `socklen_t' type])], [AC_MSG_RESULT(no)]) -AC_MSG_CHECKING([for deprecated attribute]) -AC_TRY_COMPILE([], -[void my_function(void) __attribute__((deprecated));], -[AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_DEPRECATED_ATTRIBUTE, [1], [Define to 1 if you have the `deprecated' attribute])], -[AC_MSG_RESULT(no)]) - - # Checks for library functions. AC_FUNC_ALLOCA dnl AC_FUNC_FORK diff --git a/lib/erl_interface/include/ei.h b/lib/erl_interface/include/ei.h index 10a6e5bf50..aa2a49098f 100644 --- a/lib/erl_interface/include/ei.h +++ b/lib/erl_interface/include/ei.h @@ -47,21 +47,26 @@ typedef LONG_PTR ssize_t; /* Sigh... */ # include <netdb.h> #endif -#ifndef EI_INCLUDED_CONFIG_H__ -#include "ei_config.h" +#ifdef __has_attribute +#if __has_attribute(deprecated) +#define EI_HAVE_DEPRECATED_ATTR__ 1 +#else +#undef EI_HAVE_DEPRECATED_ATTR__ +#endif #endif -#if defined(HAVE_DEPRECATED_ATTRIBUTE) && !defined(EI_NO_DEPR_WARN) -#define EI_HAVE_DEPRECATED_ATTR__ 1 +#ifdef EI_NO_DEPR_WARN +#undef EI_HAVE_DEPRECATED_ATTR__ +#endif + +#ifdef EI_HAVE_DEPRECATED_ATTR__ #define EI_DEPRECATED_ATTR_NAME deprecated #define EI_DEPRECATED_ATTR __attribute__((EI_DEPRECATED_ATTR_NAME)) #else -#undef EI_HAVE_DEPRECATED_ATTR__ #define EI_DEPRECATED_ATTR_NAME #define EI_DEPRECATED_ATTR #endif - /* -------------------------------------------------------------------- */ /* Defines part of API */ /* -------------------------------------------------------------------- */ diff --git a/lib/erl_interface/include/ei_config.h.in b/lib/erl_interface/include/ei_config.h.in deleted file mode 100644 index dfcf676bed..0000000000 --- a/lib/erl_interface/include/ei_config.h.in +++ /dev/null @@ -1,3 +0,0 @@ - -/* Define to 1 if you have the `deprecated' attribute */ -#undef HAVE_DEPRECATED_ATTRIBUTE diff --git a/lib/erl_interface/src/Makefile b/lib/erl_interface/src/Makefile index 800557fbfe..00c49f1622 100644 --- a/lib/erl_interface/src/Makefile +++ b/lib/erl_interface/src/Makefile @@ -27,7 +27,9 @@ include $(ERL_TOP)/make/output.mk include $(ERL_TOP)/make/target.mk debug opt shared purify quantify purecov gcov: +ifndef TERTIARY_BOOTSTRAP $(make_verbose)$(MAKE) -f $(TARGET)/Makefile TYPE=$@ +endif clean depend docs release release_docs tests release_tests check xmllint: $(make_verbose)$(MAKE) -f $(TARGET)/Makefile $@ diff --git a/lib/erl_interface/src/Makefile.in b/lib/erl_interface/src/Makefile.in index 1b19a85f1e..b2600f0fab 100644 --- a/lib/erl_interface/src/Makefile.in +++ b/lib/erl_interface/src/Makefile.in @@ -118,7 +118,7 @@ else AR_FLAGS=rcv endif -INCFLAGS = -I. -I../include -I../include/$(TARGET) -Iconnect -Iencode -Idecode -Imisc -Iepmd \ +INCFLAGS = -I. -I../include -Iconnect -Iencode -Idecode -Imisc -Iepmd \ -Iregistry -I$(TARGET) ifeq ($(USING_VC),yes) @@ -322,8 +322,7 @@ HEADERS = \ ../include/ei.h \ ../include/ei_connect.h \ ../include/eicode.h \ - ../include/erl_interface.h \ - ../include/$(TARGET)/ei_config.h + ../include/erl_interface.h EISOURCES = \ $(CONNECTSRC) \ diff --git a/lib/erl_interface/src/misc/eidef.h b/lib/erl_interface/src/misc/eidef.h index 083814c6e9..f38824d826 100644 --- a/lib/erl_interface/src/misc/eidef.h +++ b/lib/erl_interface/src/misc/eidef.h @@ -25,7 +25,6 @@ /* Common definitions used in ei user interface */ -#define EI_INCLUDED_CONFIG_H__ #include "config.h" /* Central include of config.h */ /* vxWorks.h needs to be before stddef.h */ diff --git a/lib/erl_interface/test/all_SUITE_data/Makefile.src b/lib/erl_interface/test/all_SUITE_data/Makefile.src index 57e522fd3e..4f27b097c8 100644 --- a/lib/erl_interface/test/all_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/all_SUITE_data/Makefile.src @@ -21,8 +21,8 @@ include @erl_interface_mk_include@ CC0 = @CC@ CC = .@DS@gccifier@exe@ -CC"$(CC0)" -CFLAGS0 = @CFLAGS@ -I@erl_interface_include@ @erl_interface_target_include@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ +CFLAGS0 = @CFLAGS@ -I@erl_interface_include@ +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ EI_COMMON_OBJS = runner@obj@ ei_runner@obj@ ALL_OBJS = gccifier@exe@ $(EI_COMMON_OBJS) diff --git a/lib/erl_interface/test/ei_accept_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_accept_SUITE_data/Makefile.src index 76dc84221f..10ef437f8b 100644 --- a/lib/erl_interface/test/ei_accept_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_accept_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_ACCEPT_OBJS = ei_accept_test@obj@ ei_accept_test_decl@obj@ EIACCNODE_OBJS = eiaccnode@obj@ diff --git a/lib/erl_interface/test/ei_connect_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_connect_SUITE_data/Makefile.src index d1694e607d..c2d8261dd8 100644 --- a/lib/erl_interface/test/ei_connect_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_connect_SUITE_data/Makefile.src @@ -28,7 +28,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBERL) $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_CONNECT_OBJS = ei_connect_test@obj@ ei_connect_test_decl@obj@ EINODE_OBJS = einode@obj@ diff --git a/lib/erl_interface/test/ei_decode_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_decode_SUITE_data/Makefile.src index 76591d893c..e678914a40 100644 --- a/lib/erl_interface/test/ei_decode_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_decode_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_DECODE_OBJS = ei_decode_test@obj@ ei_decode_test_decl@obj@ all: ei_decode_test@exe@ diff --git a/lib/erl_interface/test/ei_decode_encode_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_decode_encode_SUITE_data/Makefile.src index 3f5fa4f295..853fe9ddeb 100644 --- a/lib/erl_interface/test/ei_decode_encode_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_decode_encode_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_DECODE_ENCODE_OBJS = ei_decode_encode_test@obj@ ei_decode_encode_test_decl@obj@ all: ei_decode_encode_test@exe@ diff --git a/lib/erl_interface/test/ei_encode_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_encode_SUITE_data/Makefile.src index 489382d85e..3b2cab7af4 100644 --- a/lib/erl_interface/test/ei_encode_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_encode_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_ENCODE_OBJS = ei_encode_test@obj@ ei_encode_test_decl@obj@ all: ei_encode_test@exe@ diff --git a/lib/erl_interface/test/ei_format_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_format_SUITE_data/Makefile.src index 9e5a271db6..b89dcae45a 100644 --- a/lib/erl_interface/test/ei_format_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_format_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_FORMAT_OBJS = ei_format_test@obj@ ei_format_test_decl@obj@ all: ei_format_test@exe@ diff --git a/lib/erl_interface/test/ei_print_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_print_SUITE_data/Makefile.src index 354011f1a5..150c11b99c 100644 --- a/lib/erl_interface/test/ei_print_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_print_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_PRINT_OBJS = ei_print_test@obj@ ei_print_test_decl@obj@ all: ei_print_test@exe@ diff --git a/lib/erl_interface/test/ei_tmo_SUITE_data/Makefile.src b/lib/erl_interface/test/ei_tmo_SUITE_data/Makefile.src index 76a9c6a606..b4ee361939 100644 --- a/lib/erl_interface/test/ei_tmo_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/ei_tmo_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/ei_runner@obj@ \ $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EI_TMO_OBJS = ei_tmo_test@obj@ ei_tmo_test_decl@obj@ all: ei_tmo_test@exe@ diff --git a/lib/erl_interface/test/erl_connect_SUITE_data/Makefile.src b/lib/erl_interface/test/erl_connect_SUITE_data/Makefile.src index 19b076794a..ff4c382c97 100644 --- a/lib/erl_interface/test/erl_connect_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/erl_connect_SUITE_data/Makefile.src @@ -28,7 +28,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/runner@obj@ \ $(LIBERL) $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data OBJS = erl_connect_test@obj@ erl_connect_test_decl@obj@ all: erl_connect_test@exe@ diff --git a/lib/erl_interface/test/erl_eterm_SUITE_data/Makefile.src b/lib/erl_interface/test/erl_eterm_SUITE_data/Makefile.src index cd4de56d7c..4b1ddf77b6 100644 --- a/lib/erl_interface/test/erl_eterm_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/erl_eterm_SUITE_data/Makefile.src @@ -28,7 +28,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/runner@obj@ \ $(LIBERL) $(LIBEI) @erl_interface_sock_libs@ @LIBS@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data ETERM_OBJS = eterm_test@obj@ eterm_test_decl@obj@ CNODE_OBJS = cnode@obj@ PRINT_OBJS = print_term@obj@ diff --git a/lib/erl_interface/test/erl_ext_SUITE_data/Makefile.src b/lib/erl_interface/test/erl_ext_SUITE_data/Makefile.src index 50b60637cd..fe8caebbd6 100644 --- a/lib/erl_interface/test/erl_ext_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/erl_ext_SUITE_data/Makefile.src @@ -28,7 +28,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/runner@obj@ \ $(LIBERL) $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data EXT_OBJS = ext_test@obj@ ext_test_decl@obj@ all: ext_test@exe@ diff --git a/lib/erl_interface/test/erl_format_SUITE_data/Makefile.src b/lib/erl_interface/test/erl_format_SUITE_data/Makefile.src index 7d51cd6007..2ba59ab651 100644 --- a/lib/erl_interface/test/erl_format_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/erl_format_SUITE_data/Makefile.src @@ -28,7 +28,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/runner@obj@ \ $(LIBERL) $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data FORMAT_OBJS = format_test@obj@ format_test_decl@obj@ all: format_test@exe@ diff --git a/lib/erl_interface/test/erl_global_SUITE_data/Makefile.src b/lib/erl_interface/test/erl_global_SUITE_data/Makefile.src index 9f2a8619ac..1c1530d1b6 100644 --- a/lib/erl_interface/test/erl_global_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/erl_global_SUITE_data/Makefile.src @@ -28,7 +28,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/runner@obj@ \ $(LIBERL) $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data OBJS = erl_global_test@obj@ erl_global_test_decl@obj@ all: erl_global_test@exe@ diff --git a/lib/erl_interface/test/erl_match_SUITE_data/Makefile.src b/lib/erl_interface/test/erl_match_SUITE_data/Makefile.src index 56ed1df203..156214a269 100644 --- a/lib/erl_interface/test/erl_match_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/erl_match_SUITE_data/Makefile.src @@ -28,7 +28,7 @@ LIBEI = @erl_interface_eilib@ LIBFLAGS = ../all_SUITE_data/runner@obj@ \ $(LIBERL) $(LIBEI) @LIBS@ @erl_interface_sock_libs@ \ @erl_interface_threadlib@ -CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +CFLAGS = @EI_CFLAGS@ $(THR_DEFS) -I@erl_interface_include@ -I../all_SUITE_data MATCH_OBJS = match_test@obj@ match_test_decl@obj@ all: match_test@exe@ diff --git a/lib/erl_interface/test/port_call_SUITE_data/Makefile.src b/lib/erl_interface/test/port_call_SUITE_data/Makefile.src index e09e0fe175..0f97ce9f70 100644 --- a/lib/erl_interface/test/port_call_SUITE_data/Makefile.src +++ b/lib/erl_interface/test/port_call_SUITE_data/Makefile.src @@ -27,7 +27,7 @@ LIBERL = @erl_interface_lib_drv@ LIBEI = @erl_interface_eilib_drv@ SHLIB_EXTRA_LDLIBS = $(LIBERL) $(LIBEI) @erl_interface_threadlib@ -SHLIB_EXTRA_CFLAGS = -I@erl_interface_include@ @erl_interface_target_include@ -I../all_SUITE_data +SHLIB_EXTRA_CFLAGS = -I@erl_interface_include@ -I../all_SUITE_data all: port_call_drv@dll@ diff --git a/lib/ftp/src/ftp.erl b/lib/ftp/src/ftp.erl index 40f6b53fa3..18cd8c7524 100644 --- a/lib/ftp/src/ftp.erl +++ b/lib/ftp/src/ftp.erl @@ -1065,9 +1065,9 @@ handle_call({_, {open, ip_comm, Opts, {CtrlOpts, DataPassOpts, DataActOpts}}}, F end end; -handle_call({_, {open, tls_upgrade, TLSOptions}}, From, State) -> - _ = send_ctrl_message(State, mk_cmd("AUTH TLS", [])), - activate_ctrl_connection(State), +handle_call({_, {open, tls_upgrade, TLSOptions}}, From, State0) -> + _ = send_ctrl_message(State0, mk_cmd("AUTH TLS", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From, caller = open, tls_options = TLSOptions}}; handle_call({_, {user, User, Password}}, From, @@ -1081,17 +1081,17 @@ handle_call({_, {user, User, Password, Acc}}, From, handle_call({_, {account, Acc}}, From, State)-> handle_user_account(Acc, State#state{client = From}); -handle_call({_, pwd}, From, #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("PWD", [])), - activate_ctrl_connection(State), +handle_call({_, pwd}, From, #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("PWD", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From, caller = pwd}}; handle_call({_, lpwd}, From, #state{ldir = LDir} = State) -> {reply, {ok, LDir}, State#state{client = From}}; -handle_call({_, {cd, Dir}}, From, #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("CWD ~s", [Dir])), - activate_ctrl_connection(State), +handle_call({_, {cd, Dir}}, From, #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("CWD ~s", [Dir])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From, caller = cd}}; handle_call({_,{lcd, Dir}}, _From, #state{ldir = LDir0} = State) -> @@ -1108,41 +1108,41 @@ handle_call({_, {dir, Len, Dir}}, {_Pid, _} = From, setup_data_connection(State#state{caller = {dir, Dir, Len}, client = From}); handle_call({_, {rename, CurrFile, NewFile}}, From, - #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("RNFR ~s", [CurrFile])), - activate_ctrl_connection(State), + #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("RNFR ~s", [CurrFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {rename, NewFile}, client = From}}; handle_call({_, {delete, File}}, {_Pid, _} = From, - #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("DELE ~s", [File])), - activate_ctrl_connection(State), + #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("DELE ~s", [File])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From}}; -handle_call({_, {mkdir, Dir}}, From, #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("MKD ~s", [Dir])), - activate_ctrl_connection(State), +handle_call({_, {mkdir, Dir}}, From, #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("MKD ~s", [Dir])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From}}; -handle_call({_,{rmdir, Dir}}, From, #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("RMD ~s", [Dir])), - activate_ctrl_connection(State), +handle_call({_,{rmdir, Dir}}, From, #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("RMD ~s", [Dir])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From}}; -handle_call({_,{type, Type}}, From, #state{chunk = false} = State) -> +handle_call({_,{type, Type}}, From, #state{chunk = false} = State0) -> case Type of ascii -> - _ = send_ctrl_message(State, mk_cmd("TYPE A", [])), - activate_ctrl_connection(State), + _ = send_ctrl_message(State0, mk_cmd("TYPE A", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = type, type = ascii, client = From}}; binary -> - _ = send_ctrl_message(State, mk_cmd("TYPE I", [])), - activate_ctrl_connection(State), + _ = send_ctrl_message(State0, mk_cmd("TYPE I", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = type, type = binary, client = From}}; _ -> - {reply, {error, etype}, State} + {reply, {error, etype}, State0} end; handle_call({_,{recv, RemoteFile, LocalFile}}, From, @@ -1181,8 +1181,8 @@ handle_call({_, recv_chunk}, _From, #state{chunk = true, } = State0) -> %% The ftp:recv_chunk call was the last event we waited for, finnish and clean up ?DBG("recv_chunk_closing ftp:recv_chunk, last event",[]), - activate_ctrl_connection(State0), - {reply, ok, State0#state{caller = undefined, + State = activate_ctrl_connection(State0), + {reply, ok, State#state{caller = undefined, chunk = false, client = undefined}}; @@ -1238,18 +1238,18 @@ handle_call({_, {transfer_chunk, Bin}}, _, #state{chunk = true} = State) -> handle_call({_, {transfer_chunk, _}}, _, #state{chunk = false} = State) -> {reply, {error, echunk}, State}; -handle_call({_, chunk_end}, From, #state{chunk = true} = State) -> - close_data_connection(State), - activate_ctrl_connection(State), +handle_call({_, chunk_end}, From, #state{chunk = true} = State0) -> + close_data_connection(State0), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From, dsock = undefined, caller = end_chunk_transfer, chunk = false}}; handle_call({_, chunk_end}, _, #state{chunk = false} = State) -> {reply, {error, echunk}, State}; -handle_call({_, {quote, Cmd}}, From, #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd(Cmd, [])), - activate_ctrl_connection(State), +handle_call({_, {quote, Cmd}}, From, #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd(Cmd, [])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From, caller = quote}}; handle_call({_, _Req}, _From, #state{csock = CSock} = State) @@ -1325,38 +1325,38 @@ handle_info({Trpt, Socket, Data}, #state{dsock = {Trpt,Socket}} = State0) when T Data/binary>>}}; handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, - caller = {recv_file, Fd}} = State) + caller = {recv_file, Fd}} = State0) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> file_close(Fd), - progress_report({transfer_size, 0}, State), - activate_ctrl_connection(State), + progress_report({transfer_size, 0}, State0), + State = activate_ctrl_connection(State0), ?DBG("Data channel close",[]), {noreply, State#state{dsock = undefined, data = <<>>}}; handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, client = Client, - caller = recv_chunk} = State) + caller = recv_chunk} = State0) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> ?DBG("Data channel close recv_chunk",[]), - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{dsock = undefined, caller = #recv_chunk_closing{dconn_closed = true, client_called_us = Client =/= undefined} }}; handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, caller = recv_bin, - data = Data} = State) + data = Data} = State0) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> ?DBG("Data channel close",[]), - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{dsock = undefined, data = <<>>, caller = {recv_bin, Data}}}; handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, data = Data, - caller = {handle_dir_result, Dir}} - = State) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> + caller = {handle_dir_result, Dir}} + = State0) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> ?DBG("Data channel close",[]), - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{dsock = undefined, caller = {handle_dir_result, Dir, Data}, % data = <<?CR,?LF>>}}; @@ -1377,7 +1377,7 @@ handle_info({Transport, Socket, Data}, #state{csock = {Transport, Socket}, client = From, ctrl_data = {CtrlData, AccLines, LineStatus}} - = State) -> + = State0) -> ?DBG('--ctrl ~p ----> ~s~p~n',[Socket,<<CtrlData/binary, Data/binary>>,State]), case ftp_response:parse_lines(<<CtrlData/binary, Data/binary>>, AccLines, LineStatus) of @@ -1387,21 +1387,21 @@ handle_info({Transport, Socket, Data}, #state{csock = {Transport, Socket}, case Caller of quote -> gen_server:reply(From, string:tokens(Lines, [?CR, ?LF])), - {noreply, State#state{client = undefined, - caller = undefined, - latest_ctrl_response = Lines, - ctrl_data = {NextMsgData, [], - start}}}; + {noreply, State0#state{client = undefined, + caller = undefined, + latest_ctrl_response = Lines, + ctrl_data = {NextMsgData, [], + start}}}; _ -> ?DBG(' ...handle_ctrl_result(~p,...) ctrl_data=~p~n',[CtrlResult,{NextMsgData, [], start}]), handle_ctrl_result(CtrlResult, - State#state{latest_ctrl_response = Lines, - ctrl_data = - {NextMsgData, [], start}}) + State0#state{latest_ctrl_response = Lines, + ctrl_data = + {NextMsgData, [], start}}) end; {continue, NewCtrlData} -> ?DBG(' ...Continue... ctrl_data=~p~n',[NewCtrlData]), - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{ctrl_data = NewCtrlData}} end; @@ -1567,19 +1567,19 @@ start_link(Opts, GenServerOptions) -> %%% Help functions to handle_call and/or handle_ctrl_result %%-------------------------------------------------------------------------- %% User handling -handle_user(User, Password, Acc, State) -> - _ = send_ctrl_message(State, mk_cmd("USER ~s", [User])), - activate_ctrl_connection(State), +handle_user(User, Password, Acc, State0) -> + _ = send_ctrl_message(State0, mk_cmd("USER ~s", [User])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {handle_user, Password, Acc}}}. -handle_user_passwd(Password, Acc, State) -> - _ = send_ctrl_message(State, mk_cmd("PASS ~s", [Password])), - activate_ctrl_connection(State), +handle_user_passwd(Password, Acc, State0) -> + _ = send_ctrl_message(State0, mk_cmd("PASS ~s", [Password])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {handle_user_passwd, Acc}}}. -handle_user_account(Acc, State) -> - _ = send_ctrl_message(State, mk_cmd("ACCT ~s", [Acc])), - activate_ctrl_connection(State), +handle_user_account(Acc, State0) -> + _ = send_ctrl_message(State0, mk_cmd("ACCT ~s", [Acc])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = handle_user_account}}. @@ -1594,9 +1594,9 @@ handle_ctrl_result({tls_upgrade, _}, #state{csock = {tcp, Socket}, ?DBG('<--ctrl ssl:connect(~p, ~p)~n~p~n',[Socket,TLSOptions,State0]), case ssl:connect(Socket, TLSOptions, Timeout) of {ok, TLSSocket} -> - State = State0#state{csock = {ssl,TLSSocket}}, - _ = send_ctrl_message(State, mk_cmd("PBSZ 0", [])), - activate_ctrl_connection(State), + State1 = State0#state{csock = {ssl,TLSSocket}}, + _ = send_ctrl_message(State1, mk_cmd("PBSZ 0", [])), + State = activate_ctrl_connection(State1), {noreply, State#state{tls_upgrading_data_connection = {true, pbsz}} }; {error, _} = Error -> gen_server:reply(From, {Error, self()}), @@ -1605,9 +1605,9 @@ handle_ctrl_result({tls_upgrade, _}, #state{csock = {tcp, Socket}, tls_upgrading_data_connection = false}} end; -handle_ctrl_result({pos_compl, _}, #state{tls_upgrading_data_connection = {true, pbsz}} = State) -> - _ = send_ctrl_message(State, mk_cmd("PROT P", [])), - activate_ctrl_connection(State), +handle_ctrl_result({pos_compl, _}, #state{tls_upgrading_data_connection = {true, pbsz}} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("PROT P", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{tls_upgrading_data_connection = {true, prot}}}; handle_ctrl_result({pos_compl, _}, #state{tls_upgrading_data_connection = {true, prot}, @@ -1801,10 +1801,10 @@ handle_ctrl_result({pos_compl, _}, #state{caller = {handle_dir_result, Dir, handle_ctrl_result({pos_compl, Lines}, #state{caller = {handle_dir_data, Dir, DirData}} = - State) -> + State0) -> OldDir = pwd_result(Lines), - _ = send_ctrl_message(State, mk_cmd("CWD ~s", [Dir])), - activate_ctrl_connection(State), + _ = send_ctrl_message(State0, mk_cmd("CWD ~s", [Dir])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {handle_dir_data_second_phase, OldDir, DirData}}}; handle_ctrl_result({Status, _}, @@ -1818,9 +1818,9 @@ handle_ctrl_result(S={_Status, _}, handle_ctrl_result({pos_compl, _}, #state{caller = {handle_dir_data_second_phase, OldDir, - DirData}} = State) -> - _ = send_ctrl_message(State, mk_cmd("CWD ~s", [OldDir])), - activate_ctrl_connection(State), + DirData}} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("CWD ~s", [OldDir])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {handle_dir_data_third_phase, DirData}}}; handle_ctrl_result({Status, _}, #state{caller = {handle_dir_data_second_phase, _, _}} @@ -1840,9 +1840,9 @@ handle_ctrl_result(Status={epath, _}, #state{caller = {dir,_}} = State) -> %%-------------------------------------------------------------------------- %% File renaming handle_ctrl_result({pos_interm, _}, #state{caller = {rename, NewFile}} - = State) -> - _ = send_ctrl_message(State, mk_cmd("RNTO ~s", [NewFile])), - activate_ctrl_connection(State), + = State0) -> + _ = send_ctrl_message(State0, mk_cmd("RNTO ~s", [NewFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = rename_second_phase}}; handle_ctrl_result({Status, _}, @@ -1916,10 +1916,10 @@ handle_ctrl_result({pos_compl, _}, #state{client = From, %% The pos_compl was the last event we waited for, finnish and clean up ?DBG("recv_chunk_closing pos_compl, last event",[]), gen_server:reply(From, ok), - activate_ctrl_connection(State0), - {noreply, State0#state{caller = undefined, - chunk = false, - client = undefined}}; + State = activate_ctrl_connection(State0), + {noreply, State#state{caller = undefined, + chunk = false, + client = undefined}}; handle_ctrl_result({pos_compl, _}, #state{caller = #recv_chunk_closing{}=R} = State0) -> @@ -2013,71 +2013,71 @@ ctrl_result_response(_, #state{client = From} = State, ErrorMsg) -> {noreply, State#state{client = undefined, caller = undefined}}. %%-------------------------------------------------------------------------- -handle_caller(#state{caller = {dir, Dir, Len}} = State) -> +handle_caller(#state{caller = {dir, Dir, Len}} = State0) -> Cmd = case Len of short -> "NLST"; long -> "LIST" end, _ = case Dir of "" -> - send_ctrl_message(State, mk_cmd(Cmd, "")); + send_ctrl_message(State0, mk_cmd(Cmd, "")); _ -> - send_ctrl_message(State, mk_cmd(Cmd ++ " ~s", [Dir])) + send_ctrl_message(State0, mk_cmd(Cmd ++ " ~s", [Dir])) end, - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {dir, Dir}}}; -handle_caller(#state{caller = {recv_bin, RemoteFile}} = State) -> - _ = send_ctrl_message(State, mk_cmd("RETR ~s", [RemoteFile])), - activate_ctrl_connection(State), +handle_caller(#state{caller = {recv_bin, RemoteFile}} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("RETR ~s", [RemoteFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = recv_bin}}; handle_caller(#state{caller = {start_chunk_transfer, Cmd, RemoteFile}} = - State) -> - _ = send_ctrl_message(State, mk_cmd("~s ~s", [Cmd, RemoteFile])), - activate_ctrl_connection(State), + State0) -> + _ = send_ctrl_message(State0, mk_cmd("~s ~s", [Cmd, RemoteFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = start_chunk_transfer}}; -handle_caller(#state{caller = {recv_file, RemoteFile, Fd}} = State) -> - _ = send_ctrl_message(State, mk_cmd("RETR ~s", [RemoteFile])), - activate_ctrl_connection(State), +handle_caller(#state{caller = {recv_file, RemoteFile, Fd}} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("RETR ~s", [RemoteFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {recv_file, Fd}}}; handle_caller(#state{caller = {transfer_file, {Cmd, LocalFile, RemoteFile}}, - ldir = LocalDir, client = From} = State) -> + ldir = LocalDir, client = From} = State0) -> case file_open(filename:absname(LocalFile, LocalDir), read) of {ok, Fd} -> - _ = send_ctrl_message(State, mk_cmd("~s ~s", [Cmd, RemoteFile])), - activate_ctrl_connection(State), + _ = send_ctrl_message(State0, mk_cmd("~s ~s", [Cmd, RemoteFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {transfer_file, Fd}}}; {error, _} -> gen_server:reply(From, {error, epath}), - {noreply, State#state{client = undefined, caller = undefined, - dsock = undefined}} + {noreply, State0#state{client = undefined, caller = undefined, + dsock = undefined}} end; handle_caller(#state{caller = {transfer_data, {Cmd, Bin, RemoteFile}}} = - State) -> - _ = send_ctrl_message(State, mk_cmd("~s ~s", [Cmd, RemoteFile])), - activate_ctrl_connection(State), + State0) -> + _ = send_ctrl_message(State0, mk_cmd("~s ~s", [Cmd, RemoteFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {transfer_data, Bin}}}. %% ----------- FTP SERVER COMMUNICATION ------------------------- %% Connect to FTP server at Host (default is TCP port 21) %% in order to establish a control connection. -setup_ctrl_connection(Host, Port, Timeout, #state{sockopts_ctrl = SockOpts} = State) -> +setup_ctrl_connection(Host, Port, Timeout, #state{sockopts_ctrl = SockOpts} = State0) -> MsTime = erlang:monotonic_time(), - case connect(Host, Port, SockOpts, Timeout, State) of + case connect(Host, Port, SockOpts, Timeout, State0) of {ok, IpFam, CSock} -> - NewState = State#state{csock = {tcp, CSock}, ipfamily = IpFam}, - activate_ctrl_connection(NewState), + State1 = State0#state{csock = {tcp, CSock}, ipfamily = IpFam}, + State = activate_ctrl_connection(State1), case Timeout - millisec_passed(MsTime) of Timeout2 when (Timeout2 >= 0) -> - {ok, NewState#state{caller = open}, Timeout2}; + {ok, State#state{caller = open}, Timeout2}; _ -> %% Oups: Simulate timeout - {ok, NewState#state{caller = open}, 0} + {ok, State#state{caller = open}, 0} end; Error -> Error @@ -2087,7 +2087,7 @@ setup_data_connection(#state{mode = active, caller = Caller, csock = CSock, sockopts_data_active = SockOpts, - ftp_extension = FtpExt} = State) -> + ftp_extension = FtpExt} = State0) -> case (catch sockname(CSock)) of {ok, {{_, _, _, _, _, _, _, _} = IP0, _}} -> IP = proplists:get_value(ip, SockOpts, IP0), @@ -2098,8 +2098,8 @@ setup_data_connection(#state{mode = active, {ok, {_, Port}} = sockname({tcp,LSock}), IpAddress = inet_parse:ntoa(IP), Cmd = mk_cmd("EPRT |2|~s|~p|", [IpAddress, Port]), - _ = send_ctrl_message(State, Cmd), - activate_ctrl_connection(State), + _ = send_ctrl_message(State0, Cmd), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {setup_data_connection, {LSock, Caller}}}}; {ok, {{_,_,_,_} = IP0, _}} -> @@ -2112,37 +2112,37 @@ setup_data_connection(#state{mode = active, false -> {IP1, IP2, IP3, IP4} = IP, {Port1, Port2} = {Port div 256, Port rem 256}, - send_ctrl_message(State, + send_ctrl_message(State0, mk_cmd("PORT ~w,~w,~w,~w,~w,~w", [IP1, IP2, IP3, IP4, Port1, Port2])); true -> IpAddress = inet_parse:ntoa(IP), Cmd = mk_cmd("EPRT |1|~s|~p|", [IpAddress, Port]), - send_ctrl_message(State, Cmd) + send_ctrl_message(State0, Cmd) end, - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {setup_data_connection, {LSock, Caller}}}} end; setup_data_connection(#state{mode = passive, ipfamily = inet6, - caller = Caller} = State) -> - _ = send_ctrl_message(State, mk_cmd("EPSV", [])), - activate_ctrl_connection(State), + caller = Caller} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("EPSV", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {setup_data_connection, Caller}}}; setup_data_connection(#state{mode = passive, ipfamily = inet, caller = Caller, - ftp_extension = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("PASV", [])), - activate_ctrl_connection(State), + ftp_extension = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("PASV", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {setup_data_connection, Caller}}}; setup_data_connection(#state{mode = passive, ipfamily = inet, caller = Caller, - ftp_extension = true} = State) -> - _ = send_ctrl_message(State, mk_cmd("EPSV", [])), - activate_ctrl_connection(State), + ftp_extension = true} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("EPSV", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {setup_data_connection, Caller}}}. connect(Host, Port, SockOpts, Timeout, #state{ipfamily = inet = IpFam}) -> @@ -2248,14 +2248,15 @@ send_message({tcp, Socket}, Message) -> send_message({ssl, Socket}, Message) -> ssl:send(Socket, Message). -activate_ctrl_connection(#state{csock = CSock, ctrl_data = {<<>>, _, _}}) -> - activate_connection(CSock); -activate_ctrl_connection(#state{csock = CSock}) -> +activate_ctrl_connection(#state{csock = CSock, ctrl_data = {<<>>, _, _}} = State) -> + activate_connection(CSock), + State; +activate_ctrl_connection(#state{csock = CSock} = State0) -> activate_connection(CSock), %% We have already received at least part of the next control message, %% that has been saved in ctrl_data, process this first. - self() ! {socket_type(CSock), unwrap_socket(CSock), <<>>}, - ok. + {noreply, State} = handle_info({socket_type(CSock), unwrap_socket(CSock), <<>>}, State0), + State. activate_data_connection(#state{dsock = DSock} = State) -> activate_connection(DSock), @@ -2290,22 +2291,22 @@ close_connection({ssl, Socket}) -> ignore_return_value( ssl:close(Socket) ). %% ------------ FILE HANDLING ---------------------------------------- send_file(#state{tls_upgrading_data_connection = {true, CTRL, _}} = State, Fd) -> {noreply, State#state{tls_upgrading_data_connection = {true, CTRL, ?MODULE, send_file, Fd}}}; -send_file(State, Fd) -> +send_file(State0, Fd) -> case file_read(Fd) of {ok, N, Bin} when N > 0 -> - send_data_message(State, Bin), - progress_report({binary, Bin}, State), - send_file(State, Fd); + send_data_message(State0, Bin), + progress_report({binary, Bin}, State0), + send_file(State0, Fd); {ok, _, _} -> file_close(Fd), - close_data_connection(State), - progress_report({transfer_size, 0}, State), - activate_ctrl_connection(State), + close_data_connection(State0), + progress_report({transfer_size, 0}, State0), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = transfer_file_second_phase, dsock = undefined}}; {error, Reason} -> - gen_server:reply(State#state.client, {error, Reason}), - {stop, normal, State#state{client = undefined}} + gen_server:reply(State0#state.client, {error, Reason}), + {stop, normal, State0#state{client = undefined}} end. file_open(File, Option) -> @@ -2347,10 +2348,10 @@ cast(GenServer, Msg) -> send_bin(#state{tls_upgrading_data_connection = {true, CTRL, _}} = State, Bin) -> State#state{tls_upgrading_data_connection = {true, CTRL, ?MODULE, send_bin, Bin}}; -send_bin(State, Bin) -> - send_data_message(State, Bin), - close_data_connection(State), - activate_ctrl_connection(State), +send_bin(State0, Bin) -> + send_data_message(State0, Bin), + close_data_connection(State0), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = transfer_data_second_phase, dsock = undefined}}. diff --git a/lib/ftp/test/ftp_SUITE.erl b/lib/ftp/test/ftp_SUITE.erl index 7c87d5cbdb..0b070ee8cb 100644 --- a/lib/ftp/test/ftp_SUITE.erl +++ b/lib/ftp/test/ftp_SUITE.erl @@ -96,6 +96,7 @@ ftp_tests()-> recv_chunk, recv_chunk_twice, recv_chunk_three_times, + recv_chunk_delay, type, quote, error_elogin, @@ -669,9 +670,9 @@ recv_chunk(Config0) -> Contents = list_to_binary( lists:duplicate(1000, lists:seq(0,255)) ), Config = set_state([reset, {mkfile,File,Contents}], Config0), Pid = proplists:get_value(ftp, Config), - {{error, "ftp:recv_chunk_start/2 not called"},_} = recv_chunk(Pid, <<>>), + {error, "ftp:recv_chunk_start/2 not called"} = do_recv_chunk(Pid), ok = ftp:recv_chunk_start(Pid, id2ftp(File,Config)), - {ok, ReceivedContents, _Ncunks} = recv_chunk(Pid, <<>>), + {ok, ReceivedContents} = do_recv_chunk(Pid), find_diff(ReceivedContents, Contents). recv_chunk_twice() -> @@ -683,11 +684,11 @@ recv_chunk_twice(Config0) -> Contents2 = crypto:strong_rand_bytes(1200), Config = set_state([reset, {mkfile,File1,Contents1}, {mkfile,File2,Contents2}], Config0), Pid = proplists:get_value(ftp, Config), - {{error, "ftp:recv_chunk_start/2 not called"},_} = recv_chunk(Pid, <<>>), + {error, "ftp:recv_chunk_start/2 not called"} = do_recv_chunk(Pid), ok = ftp:recv_chunk_start(Pid, id2ftp(File1,Config)), - {ok, ReceivedContents1, _Ncunks1} = recv_chunk(Pid, <<>>), + {ok, ReceivedContents1} = do_recv_chunk(Pid), ok = ftp:recv_chunk_start(Pid, id2ftp(File2,Config)), - {ok, ReceivedContents2, _Ncunks2} = recv_chunk(Pid, <<>>), + {ok, ReceivedContents2} = do_recv_chunk(Pid), find_diff(ReceivedContents1, Contents1), find_diff(ReceivedContents2, Contents2). @@ -704,46 +705,56 @@ recv_chunk_three_times(Config0) -> Config = set_state([reset, {mkfile,File1,Contents1}, {mkfile,File2,Contents2}, {mkfile,File3,Contents3}], Config0), Pid = proplists:get_value(ftp, Config), - {{error, "ftp:recv_chunk_start/2 not called"},_} = recv_chunk(Pid, <<>>), + {error, "ftp:recv_chunk_start/2 not called"} = do_recv_chunk(Pid), + ok = ftp:recv_chunk_start(Pid, id2ftp(File3,Config)), + {ok, ReceivedContents3} = do_recv_chunk(Pid), + ok = ftp:recv_chunk_start(Pid, id2ftp(File1,Config)), - {ok, ReceivedContents1, Nchunks1} = recv_chunk(Pid, <<>>), + {ok, ReceivedContents1} = do_recv_chunk(Pid), ok = ftp:recv_chunk_start(Pid, id2ftp(File2,Config)), - {ok, ReceivedContents2, _Nchunks2} = recv_chunk(Pid, <<>>), - - ok = ftp:recv_chunk_start(Pid, id2ftp(File3,Config)), - {ok, ReceivedContents3, _Nchunks3} = recv_chunk(Pid, <<>>, 10000, 0, Nchunks1), + {ok, ReceivedContents2} = do_recv_chunk(Pid), find_diff(ReceivedContents1, Contents1), find_diff(ReceivedContents2, Contents2), find_diff(ReceivedContents3, Contents3). - +do_recv_chunk(Pid) -> + recv_chunk(Pid, <<>>). recv_chunk(Pid, Acc) -> - recv_chunk(Pid, Acc, 0, 0, undefined). - - - -%% ExpectNchunks :: integer() | undefined -recv_chunk(Pid, Acc, DelayMilliSec, N, ExpectNchunks) when N+1 < ExpectNchunks -> - %% for all I in integer(), I < undefined - recv_chunk1(Pid, Acc, DelayMilliSec, N, ExpectNchunks); - -recv_chunk(Pid, Acc, DelayMilliSec, N, ExpectNchunks) -> - %% N >= ExpectNchunks-1 - timer:sleep(DelayMilliSec), - recv_chunk1(Pid, Acc, DelayMilliSec, N, ExpectNchunks). + case ftp:recv_chunk(Pid) of + ok -> + {ok, Acc}; + {ok, Bin} -> + recv_chunk(Pid, <<Acc/binary, Bin/binary>>); + Error -> + Error + end. +recv_chunk_delay(Config0) when is_list(Config0) -> + File1 = "big_file1.txt", + Contents = list_to_binary(lists:duplicate(1000, lists:seq(0,255))), + Config = set_state([reset, {mkfile,File1,Contents}], Config0), + Pid = proplists:get_value(ftp, Config), + ok = ftp:recv_chunk_start(Pid, id2ftp(File1,Config)), + {ok, ReceivedContents} = delay_recv_chunk(Pid), + find_diff(ReceivedContents, Contents). -recv_chunk1(Pid, Acc, DelayMilliSec, N, ExpectNchunks) -> - ct:log("Call ftp:recv_chunk",[]), +delay_recv_chunk(Pid) -> + delay_recv_chunk(Pid, <<>>). +delay_recv_chunk(Pid, Acc) -> + ct:pal("Recived size ~p", [byte_size(Acc)]), case ftp:recv_chunk(Pid) of - ok -> {ok, Acc, N}; - {ok, Bin} -> recv_chunk(Pid, <<Acc/binary, Bin/binary>>, DelayMilliSec, N+1, ExpectNchunks); - Error -> {Error, N} - end. + ok -> + {ok, Acc}; + {ok, Bin} -> + ct:sleep(100), + delay_recv_chunk(Pid, <<Acc/binary, Bin/binary>>); + Error -> + Error + end. %%------------------------------------------------------------------------- type() -> diff --git a/lib/kernel/doc/src/kernel_app.xml b/lib/kernel/doc/src/kernel_app.xml index 15dbdb47dc..7f9609d5c1 100644 --- a/lib/kernel/doc/src/kernel_app.xml +++ b/lib/kernel/doc/src/kernel_app.xml @@ -369,6 +369,14 @@ MaxT = TickTime + TickTime / 4</code> performed. This option ensures that <c>global</c> is synchronized.</p> </item> + <tag><c>start_distribution = true | false</c></tag> + <item> + <p>Starts all distribution services, such as <c>rpc</c>, + <c>global</c>, and <c>net_kernel</c> if the parameter is + <c>true</c>. This parameter is to be set to <c>false</c> + for systems who want to disable all distribution functionality.</p> + <p>Defaults to <c>true</c>.</p> + </item> <tag><c>start_dist_ac = true | false</c></tag> <item> <p>Starts the <c>dist_ac</c> server if the parameter is @@ -510,11 +518,13 @@ MaxT = TickTime + TickTime / 4</code> parameters for Logger are not set.</p> <taglist> <tag><c>error_logger</c></tag> - <item>Replaced by setting the type of the default - <seealso marker="logger_std_h#type"><c>logger_std_h</c></seealso> - to the same value. Example: + <item>Replaced by setting the <seealso + marker="logger_std_h#type"><c>type</c></seealso>, and possibly + <seealso marker="logger_std_h#file"><c>file</c></seealso> and + <seealso marker="logger_std_h#modes"><c>modes</c></seealso> + parameters of the default <c>logger_std_h</c> handler. Example: <code type="none"> -erl -kernel logger '[{handler,default,logger_std_h,#{config=>#{type=>{file,"/tmp/erlang.log"}}}}]' +erl -kernel logger '[{handler,default,logger_std_h,#{config=>#{file=>"/tmp/erlang.log"}}}]' </code> </item> <tag><c>error_logger_format_depth</c></tag> diff --git a/lib/kernel/doc/src/logger.xml b/lib/kernel/doc/src/logger.xml index e6448e144e..ebebcaa1ae 100644 --- a/lib/kernel/doc/src/logger.xml +++ b/lib/kernel/doc/src/logger.xml @@ -66,7 +66,7 @@ logger:error("error happened because: ~p", [Reason]). % Without macro [{kernel, [{logger, [{handler, default, logger_std_h, - #{config => #{type => {file, "path/to/file.log"}}}}]}]}]. + #{config => #{file => "path/to/file.log"}}}]}]}]. </code> <p> For more information about: diff --git a/lib/kernel/doc/src/logger_chapter.xml b/lib/kernel/doc/src/logger_chapter.xml index 03b9edcf8f..8458ffa042 100644 --- a/lib/kernel/doc/src/logger_chapter.xml +++ b/lib/kernel/doc/src/logger_chapter.xml @@ -801,7 +801,7 @@ logger:debug(#{got => connection_request, id => Id, state => State}, [{kernel, [{logger, [{handler, default, logger_std_h, % {handler, HandlerId, Module, - #{config => #{type => {file,"log/erlang.log"}}}} % Config} + #{config => #{file => "log/erlang.log"}}} % Config} ]}]}]. </code> <p>Modify the default handler to print each log event as a @@ -831,10 +831,10 @@ logger:debug(#{got => connection_request, id => Id, state => State}, [{logger, [{handler, default, logger_std_h, #{level => error, - config => #{type => {file, "log/erlang.log"}}}}, + config => #{file => "log/erlang.log"}}}, {handler, info, logger_std_h, #{level => debug, - config => #{type => {file, "log/debug.log"}}}} + config => #{file => "log/debug.log"}}} ]}]}]. </code> </section> @@ -1004,10 +1004,10 @@ ok</pre> <p>Then, add a new handler which prints to file. You can use the handler module <seealso marker="logger_std_h"><c>logger_std_h</c></seealso>, - and specify type <c>{file,File}</c>.:</p> + and configure it to log to file:</p> <pre> -4> <input>Config = #{config => #{type => {file,"./info.log"}}, level => info}.</input> -#{config => #{type => {file,"./info.log"}},level => info} +4> <input>Config = #{config => #{file => "./info.log"}, level => info}.</input> +#{config => #{file => "./info.log"},level => info} 5> <input>logger:add_handler(myhandler, logger_std_h, Config).</input> ok</pre> <p>Since <c>filter_default</c> defaults to <c>log</c>, this @@ -1246,7 +1246,7 @@ do_log(Fd, LogEvent, #{formatter := {FModule, FConfig}}) -> <p>A configuration example:</p> <code type="none"> logger:add_handler(my_standard_h, logger_std_h, - #{config => #{type => {file,"./system_info.log"}, + #{config => #{file => "./system_info.log", sync_mode_qlen => 100, drop_mode_qlen => 1000, flush_qlen => 2000}}). diff --git a/lib/kernel/doc/src/logger_std_h.xml b/lib/kernel/doc/src/logger_std_h.xml index fcd180abd6..5ed1a2f210 100644 --- a/lib/kernel/doc/src/logger_std_h.xml +++ b/lib/kernel/doc/src/logger_std_h.xml @@ -55,30 +55,105 @@ is stored in a sub map with the key <c>config</c>, and can contain the following parameters:</p> <taglist> - <tag><marker id="type"/><c>type</c></tag> + <tag><marker id="type"/><c>type = standard_io | standard_error | file</c></tag> <item> - <p>This has the value <c>standard_io</c>, <c>standard_error</c>, - <c>{file,LogFileName}</c>, or <c>{file,LogFileName,LogFileOpts}</c>.</p> - <p>If <c>LogFileOpts</c> is specified, it replaces the default - list of options used when opening the log file. The default - list is <c>[raw,append,delayed_write]</c>. One reason to do - so can be to change <c>append</c> to, for - example, <c>write</c>, ensuring that the old log is - truncated when a node is restarted. See the reference manual - for <seealso marker="file#open-2"><c>file:open/2</c></seealso> - for more information about file options.</p> + <p>Specifies the log destination.</p> + <p>The value is set when the handler is added, and it can not + be changed in runtime.</p> + <p>Defaults to <c>standard_io</c>, unless + parameter <seealso marker="#file"><c>file</c></seealso> is + given, in which case it defaults to <c>file</c>.</p> + </item> + <tag><marker id="file"/><c>file = </c><seealso marker="file#type-filename"><c>file:filename()</c></seealso></tag> + <item> + <p>This specifies the name of the log file when the handler is + of type <c>file</c>.</p> + <p>The value is set when the handler is added, and it can not + be changed in runtime.</p> + <p>Defaults to the same name as the handler identity, in the + current directory.</p> + </item> + <tag><marker id="modes"/><c>modes = [</c><seealso marker="file#type-mode"><c>file:mode()</c></seealso><c>]</c></tag> + <item> + <p>This specifies the file modes to use when opening the log + file, + see <seealso marker="file#open-2"><c>file:open/2</c></seealso>. + If <c>modes</c> are not specified, the default list used + is <c>[raw,append,delayed_write]</c>. If <c>modes</c> are + specified, the list replaces the default modes list with the + following adjustments:</p> + <list> + <item> + If <c>raw</c> is not found in the list, it is added. + </item> + <item> + If none of <c>write</c>, <c>append</c> or <c>exclusive</c> is + found in the list, <c>append</c> is added.</item> + <item>If none of <c>delayed_write</c> + or <c>{delayed_write,Size,Delay}</c> is found in the + list, <c>delayed_write</c> is added.</item> + </list> <p>Log files are always UTF-8 encoded. The encoding can not be - changed by setting the option <c>{encoding,Encoding}</c> - in <c>LogFileOpts</c>.</p> - <p>Notice that the standard handler does not have support for - circular logging. Use the disk_log handler, - <seealso marker="logger_disk_log_h"><c>logger_disk_log_h</c></seealso>, - for this.</p> + changed by setting the mode <c>{encoding,Encoding}</c>.</p> <p>The value is set when the handler is added, and it can not be changed in runtime.</p> - <p>Defaults to <c>standard_io</c>.</p> + <p>Defaults to <c>[raw,append,delayed_write]</c>.</p> + </item> + <tag><marker id="max_no_bytes"/><c>max_no_bytes = pos_integer() | infinity</c></tag> + <item> + <p>This parameter specifies if the log file should be rotated + or not. The value <c>infinity</c> means the log file will + grow indefinitely, while an integer value specifies at which + file size (bytes) the file is rotated.</p> + <p>Defaults to <c>infinity</c>.</p> + </item> + <tag><marker id="max_no_files"/><c>max_no_files = non_neg_integer()</c></tag> + <item> + <p>This parameter specifies the number of rotated log file + archives to keep. This has meaning only + if <seealso marker="#max_no_bytes"><c>max_no_bytes</c></seealso> + is set to an integer value.</p> + <p>The log archives are + named <c>FileName.0</c>, <c>FileName.1</c>, + ... <c>FileName.N</c>, where <c>FileName</c> is the name of + the current log file. <c>FileName.0</c> is the newest of the + archives. The maximum value for <c>N</c> is the value + of <c>max_no_files</c> minus 1.</p> + <p>Notice that setting this value to <c>0</c> does not turn of + rotation. It only specifies that no archives are kept.</p> + <p>Defaults to <c>0</c>.</p> + </item> + <tag><marker id="compress_on_rotate"/><c>compress_on_rotate = boolean()</c></tag> + <item> + <p>This parameter specifies if the rotated log file archives + shall be compressed or not. If set to <c>true</c>, all + archives are compressed with <c>gzip</c>, and renamed + to <c>FileName.N.gz</c></p> + <p><c>compress_on_rotate</c> has no meaning if <seealso + marker="#max_no_bytes"><c>max_no_bytes</c></seealso> has the + value <c>infinity</c>.</p> + <p>Defaults to <c>false</c>.</p> + </item> + <tag><marker id="file_check"/><c>file_check = non_neg_integer()</c></tag> + <item> + <p>When <c>logger_std_h</c> logs to a file, it reads the file + information of the log file prior to each write + operation. This is to make sure the file still exists and + has the same inode as when it was opened. This implies some + performance loss, but ensures that no log events are lost in + the case when the file has been removed or renamed by an + external actor.</p> + <p>In order to allow minimizing the performance loss, the + <c>file_check</c> parameter can be set to a positive integer + value, <c>N</c>. The handler will then skip reading the file + information prior to writing, as long as no more + than <c>N</c> milliseconds have passed since it was last + read.</p> + <p>Notice that the risk of loosing log events grows when + the <c>file_check</c> value grows.</p> + <p>Defaults to 0.</p> </item> - <tag><c>filesync_repeat_interval</c></tag> + <tag><c>filesync_repeat_interval = pos_integer() | no_repeat</c></tag> <item> <p>This value, in milliseconds, specifies how often the handler does a file sync operation to write buffered data to disk. The handler attempts @@ -97,12 +172,13 @@ standard handler and the disk_log handler, and are documented in the <seealso marker="logger_chapter#overload_protection"><c>User's Guide</c> </seealso>.</p> - <p>Notice that if changing the configuration of the handler in runtime, - the <c>type</c> parameter must not be modified.</p> + <p>Notice that if changing the configuration of the handler in + runtime, the <c>type</c>, <c>file</c>, or <c>modes</c> parameters + must not be modified.</p> <p>Example of adding a standard handler:</p> <code type="none"> logger:add_handler(my_standard_h, logger_std_h, - #{config => #{type => {file,"./system_info.log"}, + #{config => #{file => "./system_info.log", filesync_repeat_interval => 1000}}). </code> <p>To set the default handler, that starts initially with @@ -110,7 +186,7 @@ logger:add_handler(my_standard_h, logger_std_h, change the Kernel default logger configuration. Example:</p> <code type="none"> erl -kernel logger '[{handler,default,logger_std_h, - #{config => #{type => {file,"./log.log"}}}}]' + #{config => #{file => "./log.log"}}}]' </code> <p>An example of how to replace the standard handler with a disk_log handler at startup is found in the diff --git a/lib/kernel/doc/src/notes.xml b/lib/kernel/doc/src/notes.xml index 021ecfa40d..1f4d9be1f5 100644 --- a/lib/kernel/doc/src/notes.xml +++ b/lib/kernel/doc/src/notes.xml @@ -31,6 +31,25 @@ </header> <p>This document describes the changes made to the Kernel application.</p> +<section><title>Kernel 6.2.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Setting the <c>recbuf</c> size of an inet socket the + <c>buffer</c> is also automatically increased. Fix a bug + where the auto adjustment of inet buffer size would be + triggered even if an explicit inet buffer size had + already been set.</p> + <p> + Own Id: OTP-15651 Aux Id: ERIERL-304 </p> + </item> + </list> + </section> + +</section> + <section><title>Kernel 6.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/kernel/src/kernel.app.src b/lib/kernel/src/kernel.app.src index 4b48f6cd1d..8fe6bdd1ca 100644 --- a/lib/kernel/src/kernel.app.src +++ b/lib/kernel/src/kernel.app.src @@ -147,6 +147,6 @@ {logger_sasl_compatible, false} ]}, {mod, {kernel, []}}, - {runtime_dependencies, ["erts-10.1", "stdlib-3.5", "sasl-3.0"]} + {runtime_dependencies, ["erts-10.2.5", "stdlib-3.5", "sasl-3.0"]} ] }. diff --git a/lib/kernel/src/kernel.appup.src b/lib/kernel/src/kernel.appup.src index ccf0a82ced..66fbcbf78d 100644 --- a/lib/kernel/src/kernel.appup.src +++ b/lib/kernel/src/kernel.appup.src @@ -40,7 +40,9 @@ {<<"^6\\.0\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^6\\.1$">>,[restart_new_emulator]}, {<<"^6\\.1\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], + {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^6\\.2$">>,[restart_new_emulator]}, + {<<"^6\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}], [{<<"^5\\.3$">>,[restart_new_emulator]}, {<<"^5\\.3\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^5\\.3\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, @@ -54,4 +56,6 @@ {<<"^6\\.0\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^6\\.1$">>,[restart_new_emulator]}, {<<"^6\\.1\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. + {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^6\\.2$">>,[restart_new_emulator]}, + {<<"^6\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}]}. diff --git a/lib/kernel/src/kernel.erl b/lib/kernel/src/kernel.erl index c68d04e279..111d103df2 100644 --- a/lib/kernel/src/kernel.erl +++ b/lib/kernel/src/kernel.erl @@ -145,26 +145,11 @@ init([]) -> {ok, {SupFlags, [Code, File, StdError, User, LoggerSup, Config, RefC, SafeSup]}}; _ -> - Rpc = #{id => rex, - start => {rpc, start_link, []}, - restart => permanent, - shutdown => 2000, - type => worker, - modules => [rpc]}, - - Global = #{id => global_name_server, - start => {global, start_link, []}, - restart => permanent, - shutdown => 2000, - type => worker, - modules => [global]}, - - GlGroup = #{id => global_group, - start => {global_group,start_link,[]}, - restart => permanent, - shutdown => 2000, - type => worker, - modules => [global_group]}, + DistChildren = + case application:get_env(kernel, start_distribution) of + {ok, false} -> []; + _ -> start_distribution() + end, InetDb = #{id => inet_db, start => {inet_db, start_link, []}, @@ -173,13 +158,6 @@ init([]) -> type => worker, modules => [inet_db]}, - NetSup = #{id => net_sup, - start => {erl_distribution, start_link, []}, - restart => permanent, - shutdown => infinity, - type => supervisor, - modules => [erl_distribution]}, - SigSrv = #{id => erl_signal_server, start => {gen_event, start_link, [{local, erl_signal_server}]}, restart => permanent, @@ -187,14 +165,11 @@ init([]) -> type => worker, modules => dynamic}, - DistAC = start_dist_ac(), - Timer = start_timer(), {ok, {SupFlags, - [Code, Rpc, Global, InetDb | DistAC] ++ - [NetSup, GlGroup, File, SigSrv, - StdError, User, Config, RefC, SafeSup, LoggerSup] ++ Timer}} + [Code, InetDb | DistChildren] ++ + [File, SigSrv, StdError, User, Config, RefC, SafeSup, LoggerSup] ++ Timer}} end; init(safe) -> SupFlags = #{strategy => one_for_one, @@ -213,6 +188,39 @@ init(safe) -> {ok, {SupFlags, Boot ++ DiskLog ++ Pg2}}. +start_distribution() -> + Rpc = #{id => rex, + start => {rpc, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [rpc]}, + + Global = #{id => global_name_server, + start => {global, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [global]}, + + DistAC = start_dist_ac(), + + NetSup = #{id => net_sup, + start => {erl_distribution, start_link, []}, + restart => permanent, + shutdown => infinity, + type => supervisor, + modules => [erl_distribution]}, + + GlGroup = #{id => global_group, + start => {global_group,start_link,[]}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [global_group]}, + + [Rpc, Global | DistAC] ++ [NetSup, GlGroup]. + start_dist_ac() -> Spec = [#{id => dist_ac, start => {dist_ac,start_link,[]}, diff --git a/lib/kernel/src/logger_formatter.erl b/lib/kernel/src/logger_formatter.erl index ded89bac9f..8696adbd72 100644 --- a/lib/kernel/src/logger_formatter.erl +++ b/lib/kernel/src/logger_formatter.erl @@ -64,7 +64,7 @@ format(#{level:=Level,msg:=Msg0,meta:=Meta},Config0) Config; Size0 -> Size = - case Size0 - string:length([B,A]) of + case Size0 - io_lib:chars_length([B,A]) of S when S>=0 -> S; _ -> 0 end, @@ -75,7 +75,11 @@ format(#{level:=Level,msg:=Msg0,meta:=Meta},Config0) true -> %% Trim leading and trailing whitespaces, and replace %% newlines with ", " - re:replace(string:trim(MsgStr0),",?\r?\n\s*",", ", + T = lists:reverse( + trim( + lists:reverse( + trim(MsgStr0,false)),true)), + re:replace(T,",?\r?\n\s*",", ", [{return,list},global,unicode]); _false -> MsgStr0 @@ -83,7 +87,26 @@ format(#{level:=Level,msg:=Msg0,meta:=Meta},Config0) true -> "" end, - truncate([B,MsgStr,A],maps:get(max_size,Config)). + truncate(B,MsgStr,A,maps:get(max_size,Config)). + +trim([H|T],Rev) when H==$\s; H==$\r; H==$\n -> + trim(T,Rev); +trim([H|T],false) when is_list(H) -> + case trim(H,false) of + [] -> + trim(T,false); + TrimmedH -> + [TrimmedH|T] + end; +trim([H|T],true) when is_list(H) -> + case trim(lists:reverse(H),true) of + [] -> + trim(T,true); + TrimmedH -> + [lists:reverse(TrimmedH)|T] + end; +trim(String,_) -> + String. do_format(Level,Data,[level|Format],Config) -> [to_string(level,Level,Config)|do_format(Level,Data,Format,Config)]; @@ -239,21 +262,47 @@ chardata_to_list(Chardata) -> throw(Error) end. -truncate(String,unlimited) -> - String; -truncate(String,Size) -> - Length = string:length(String), +truncate(B,Msg,A,unlimited) -> + [B,Msg,A]; +truncate(B,Msg,A,Size) -> + String = [B,Msg,A], + Length = io_lib:chars_length(String), if Length>Size -> - case lists:reverse(lists:flatten(String)) of - [$\n|_] -> - string:slice(String,0,Size-4)++"...\n"; + {Last,FlatString} = + case A of + [] -> + case Msg of + [] -> + {get_last(B),lists:flatten(B)}; + _ -> + {get_last(Msg),lists:flatten([B,Msg])} + end; + _ -> + {get_last(A),lists:flatten(String)} + end, + case Last of + $\n-> + lists:sublist(FlatString,1,Size-4)++"...\n"; _ -> - string:slice(String,0,Size-3)++"..." + lists:sublist(FlatString,1,Size-3)++"..." end; true -> String end. +get_last(L) -> + get_first(lists:reverse(L)). + +get_first([]) -> + error; +get_first([C|_]) when is_integer(C) -> + C; +get_first([L|Rest]) when is_list(L) -> + case get_last(L) of + error -> get_first(Rest); + First -> First + end. + %% SysTime is the system time in microseconds format_time(SysTime,#{time_offset:=Offset,time_designator:=Des}) when is_integer(SysTime) -> diff --git a/lib/kernel/src/logger_h_common.erl b/lib/kernel/src/logger_h_common.erl index 4b5e0a7dd0..16946ff97c 100644 --- a/lib/kernel/src/logger_h_common.erl +++ b/lib/kernel/src/logger_h_common.erl @@ -142,8 +142,9 @@ changing_config(SetOrUpdate, maps:with(?OLP_KEYS,NewHConfig0)), case logger_olp:set_opts(Olp,NewOlpOpts) of ok -> - maybe_set_repeated_filesync(Olp,OldCommonConfig, - NewCommonConfig), + logger_olp:cast(Olp, {config_changed, + NewCommonConfig, + NewHandlerConfig}), ReadOnly = maps:with(?READ_ONLY_KEYS,OldHConfig), NewHConfig = maps:merge( @@ -281,11 +282,24 @@ handle_cast(repeated_filesync, State#{handler_state => HS, last_op => sync} end, {noreply,set_repeated_filesync(State1)}; - -handle_cast({set_repeated_filesync,FSyncInt},State) -> - State1 = State#{filesync_repeat_interval=>FSyncInt}, - State2 = set_repeated_filesync(cancel_repeated_filesync(State1)), - {noreply, State2}. +handle_cast({config_changed, CommonConfig, HConfig}, + State = #{id := Name, + module := Module, + handler_state := HandlerState, + filesync_repeat_interval := OldFSyncInt}) -> + State1 = + case maps:get(filesync_repeat_interval,CommonConfig) of + OldFSyncInt -> + State; + FSyncInt -> + set_repeated_filesync( + cancel_repeated_filesync( + State#{filesync_repeat_interval=>FSyncInt})) + end, + HS = try Module:config_changed(Name, HConfig, HandlerState) + catch error:undef -> HandlerState + end, + {noreply, State1#{handler_state => HS}}. handle_info(Info, #{id := Name, module := Module, handler_state := HandlerState} = State) -> @@ -447,10 +461,3 @@ cancel_repeated_filesync(State) -> end. error_notify(Term) -> ?internal_log(error, Term). - -maybe_set_repeated_filesync(_Olp, - #{filesync_repeat_interval:=FSyncInt}, - #{filesync_repeat_interval:=FSyncInt}) -> - ok; -maybe_set_repeated_filesync(Olp,_,#{filesync_repeat_interval:=FSyncInt}) -> - logger_olp:cast(Olp,{set_repeated_filesync,FSyncInt}). diff --git a/lib/kernel/src/logger_olp.erl b/lib/kernel/src/logger_olp.erl index 009280a9c9..8365383fe2 100644 --- a/lib/kernel/src/logger_olp.erl +++ b/lib/kernel/src/logger_olp.erl @@ -515,10 +515,11 @@ check_load(State = #{id:=_Name, mode_ref := ModeRef, mode := Mode, end, State1 = ?update_other(drops,DROPS,_NewDrops,State), State2 = ?update_max_qlen(QLen,State1), - State3 = maybe_notify_mode_change(Mode1,State2), + State3 = ?update_max_mem(Mem,State2), + State4 = maybe_notify_mode_change(Mode1,State3), {Mode1, QLen, Mem, ?update_other(flushes,FLUSHES,_NewFlushes, - State3#{last_qlen => QLen})}. + State4#{last_qlen => QLen})}. limit_burst(#{burst_limit_enable := false}=State) -> {true,State}; diff --git a/lib/kernel/src/logger_olp.hrl b/lib/kernel/src/logger_olp.hrl index 9b4f5ebf27..d68b5c048d 100644 --- a/lib/kernel/src/logger_olp.hrl +++ b/lib/kernel/src/logger_olp.hrl @@ -114,12 +114,16 @@ flushes => 0, flushed => 0, drops => 0, burst_drops => 0, casts => 0, calls => 0, writes => 0, max_qlen => 0, max_time => 0, - freq => {TIME,0,0}} end). + max_mem => 0, freq => {TIME,0,0}} end). -define(update_max_qlen(QLEN, STATE), begin #{max_qlen := QLEN0} = STATE, STATE#{max_qlen => ?max(QLEN0,QLEN)} end). + -define(update_max_mem(MEM, STATE), + begin #{max_mem := MEM0} = STATE, + STATE#{max_mem => ?max(MEM0,MEM)} end). + -define(update_calls_or_casts(CALL_OR_CAST, INC, STATE), case CALL_OR_CAST of cast -> @@ -154,6 +158,7 @@ -else. % DEFAULT! -define(merge_with_stats(STATE), STATE). -define(update_max_qlen(_QLEN, STATE), STATE). + -define(update_max_mem(_MEM, STATE), STATE). -define(update_calls_or_casts(_CALL_OR_CAST, _INC, STATE), STATE). -define(update_max_time(_TIME, STATE), STATE). -define(update_other(_OTHER, _VAR, _INCVAL, STATE), STATE). diff --git a/lib/kernel/src/logger_std_h.erl b/lib/kernel/src/logger_std_h.erl index 65f5b3876e..c8f1acfca4 100644 --- a/lib/kernel/src/logger_std_h.erl +++ b/lib/kernel/src/logger_std_h.erl @@ -29,7 +29,7 @@ -export([filesync/1]). %% logger_h_common callbacks --export([init/2, check_config/4, reset_state/2, +-export([init/2, check_config/4, config_changed/3, reset_state/2, filesync/3, write/4, handle_info/3, terminate/3]). %% logger callbacks @@ -105,85 +105,169 @@ filter_config(Config) -> %%%=================================================================== %%% logger_h_common callbacks %%%=================================================================== -init(Name, #{type := Type}) -> - case open_log_file(Name, Type) of +init(Name, Config) -> + MyConfig = maps:with([type,file,modes,file_check,max_no_bytes, + max_no_files,compress_on_rotate],Config), + case file_ctrl_start(Name, MyConfig) of {ok,FileCtrlPid} -> - {ok,#{type=>Type,file_ctrl_pid=>FileCtrlPid}}; + {ok,MyConfig#{file_ctrl_pid=>FileCtrlPid}}; Error -> Error end. -check_config(_Name,set,undefined,NewHConfig) -> - check_config(maps:merge(get_default_config(),NewHConfig)); -check_config(_Name,SetOrUpdate,OldHConfig,NewHConfig0) -> - WriteOnce = maps:with([type],OldHConfig), +check_config(Name,set,undefined,NewHConfig) -> + check_h_config(merge_default_config(Name,normalize_config(NewHConfig))); +check_config(Name,SetOrUpdate,OldHConfig,NewHConfig0) -> + WriteOnce = maps:with([type,file,modes],OldHConfig), Default = case SetOrUpdate of set -> %% Do not reset write-once fields to defaults - maps:merge(get_default_config(),WriteOnce); + merge_default_config(Name,WriteOnce); update -> OldHConfig end, - NewHConfig = maps:merge(Default, NewHConfig0), + NewHConfig = maps:merge(Default, normalize_config(NewHConfig0)), %% Fail if write-once fields are changed - case maps:with([type],NewHConfig) of + case maps:with([type,file,modes],NewHConfig) of WriteOnce -> - check_config(NewHConfig); + check_h_config(NewHConfig); Other -> {error,{illegal_config_change,?MODULE,WriteOnce,Other}} end. -check_config(#{type:=Type}=HConfig) -> - case check_h_config(maps:to_list(HConfig)) of - ok when is_atom(Type) -> - {ok,HConfig#{filesync_repeat_interval=>no_repeat}}; +check_h_config(HConfig) -> + case check_h_config(maps:get(type,HConfig),maps:to_list(HConfig)) of ok -> - {ok,HConfig}; + {ok,fix_file_opts(HConfig)}; {error,{Key,Value}} -> {error,{invalid_config,?MODULE,#{Key=>Value}}} end. -check_h_config([{type,Type} | Config]) when Type == standard_io; - Type == standard_error -> - check_h_config(Config); -check_h_config([{type,{file,File}} | Config]) when is_list(File) -> - check_h_config(Config); -check_h_config([{type,{file,File,Modes}} | Config]) when is_list(File), - is_list(Modes) -> - check_h_config(Config); -check_h_config([Other | _]) -> +check_h_config(Type,[{type,Type} | Config]) when Type =:= standard_io; + Type =:= standard_error; + Type =:= file -> + check_h_config(Type,Config); +check_h_config(file,[{file,File} | Config]) when is_list(File) -> + check_h_config(file,Config); +check_h_config(file,[{modes,Modes} | Config]) when is_list(Modes) -> + check_h_config(file,Config); +check_h_config(file,[{max_no_bytes,Size} | Config]) + when (is_integer(Size) andalso Size>0) orelse Size=:=infinity -> + check_h_config(file,Config); +check_h_config(file,[{max_no_files,Num} | Config]) when is_integer(Num), Num>=0 -> + check_h_config(file,Config); +check_h_config(file,[{compress_on_rotate,Bool} | Config]) when is_boolean(Bool) -> + check_h_config(file,Config); +check_h_config(file,[{file_check,FileCheck} | Config]) + when is_integer(FileCheck), FileCheck>=0 -> + check_h_config(file,Config); +check_h_config(_Type,[Other | _]) -> {error,Other}; -check_h_config([]) -> +check_h_config(_Type,[]) -> ok. -get_default_config() -> - #{type => standard_io}. +normalize_config(#{type:={file,File}}=HConfig) -> + HConfig#{type=>file,file=>File}; +normalize_config(#{type:={file,File,Modes}}=HConfig) -> + HConfig#{type=>file,file=>File,modes=>Modes}; +normalize_config(HConfig) -> + HConfig. + +merge_default_config(Name,#{type:=Type}=HConfig) -> + merge_default_config(Name,Type,HConfig); +merge_default_config(Name,#{file:=_}=HConfig) -> + merge_default_config(Name,file,HConfig); +merge_default_config(Name,HConfig) -> + merge_default_config(Name,standard_io,HConfig). + +merge_default_config(Name,Type,HConfig) -> + maps:merge(get_default_config(Name,Type),HConfig). + +get_default_config(Name,file) -> + #{type => file, + file => atom_to_list(Name), + modes => [raw,append], + file_check => 0, + max_no_bytes => infinity, + max_no_files => 0, + compress_on_rotate => false}; +get_default_config(_Name,Type) -> + #{type => Type}. + +fix_file_opts(#{modes:=Modes}=HConfig) -> + HConfig#{modes=>fix_modes(Modes)}; +fix_file_opts(HConfig) -> + HConfig#{filesync_repeat_interval=>no_repeat}. + +fix_modes(Modes) -> + %% Ensure write|append|exclusive + Modes1 = + case [M || M <- Modes, + lists:member(M,[write,append,exclusive])] of + [] -> [append|Modes]; + _ -> Modes + end, + %% Ensure raw + Modes2 = + case lists:member(raw,Modes) of + false -> [raw|Modes1]; + true -> Modes1 + end, + %% Ensure delayed_write + case lists:partition(fun(delayed_write) -> true; + ({delayed_write,_,_}) -> true; + (_) -> false + end, Modes2) of + {[],_} -> + [delayed_write|Modes2]; + _ -> + Modes2 + end. -filesync(_Name, _Mode, #{type := Type}=State) when is_atom(Type) -> - {ok,State}; -filesync(_Name, async, #{file_ctrl_pid := FileCtrlPid} = State) -> - ok = file_ctrl_filesync_async(FileCtrlPid), - {ok,State}; -filesync(_Name, sync, #{file_ctrl_pid := FileCtrlPid} = State) -> - Result = file_ctrl_filesync_sync(FileCtrlPid), +config_changed(_Name, + #{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}, + #{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}=State) -> + State; +config_changed(_Name, + #{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}, + #{file_ctrl_pid := FileCtrlPid} = State) -> + FileCtrlPid ! {update_config,#{file_check=>FileCheck, + max_no_bytes=>Size, + max_no_files=>Count, + compress_on_rotate=>Compress}}, + State#{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}; +config_changed(_Name,_NewHConfig,State) -> + State. + +filesync(_Name, SyncAsync, #{file_ctrl_pid := FileCtrlPid} = State) -> + Result = file_ctrl_filesync(SyncAsync, FileCtrlPid), {Result,State}. -write(_Name, async, Bin, #{file_ctrl_pid:=FileCtrlPid} = State) -> - ok = file_write_async(FileCtrlPid, Bin), - {ok,State}; -write(_Name, sync, Bin, #{file_ctrl_pid:=FileCtrlPid} = State) -> - Result = file_write_sync(FileCtrlPid, Bin), +write(_Name, SyncAsync, Bin, #{file_ctrl_pid:=FileCtrlPid} = State) -> + Result = file_write(SyncAsync, FileCtrlPid, Bin), {Result,State}. reset_state(_Name, State) -> State. -handle_info(_Name, {'EXIT',Pid,Why}, #{type := FileInfo, file_ctrl_pid := Pid}) -> +handle_info(_Name, {'EXIT',Pid,Why}, #{file_ctrl_pid := Pid}=State) -> %% file_ctrl_pid died, file error, terminate handler - exit({error,{write_failed,FileInfo,Why}}); + exit({error,{write_failed,maps:with([type,file,modes],State),Why}}); handle_info(_, _, State) -> State. @@ -211,27 +295,33 @@ terminate(_Name, _Reason, #{file_ctrl_pid:=FWPid}) -> %%%----------------------------------------------------------------- %%% -open_log_file(HandlerName, FileInfo) -> - case file_ctrl_start(HandlerName, FileInfo) of - OK = {ok,_FileCtrlPid} -> OK; - Error -> Error - end. - -do_open_log_file({file,FileName}) -> - do_open_log_file({file,FileName,[raw,append,delayed_write]}); - -do_open_log_file({file,FileName,[]}) -> - do_open_log_file({file,FileName,[raw,append,delayed_write]}); - -do_open_log_file({file,FileName,Modes}) -> +open_log_file(HandlerName,#{type:=file, + file:=FileName, + modes:=Modes, + file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}) -> try case filelib:ensure_dir(FileName) of ok -> case file:open(FileName, Modes) of {ok, Fd} -> {ok,#file_info{inode=INode}} = - file:read_file_info(FileName), - {ok, {Fd, INode}}; + file:read_file_info(FileName,[raw]), + UpdateModes = [append | Modes--[write,append,exclusive]], + State0 = #{handler_name=>HandlerName, + file_name=>FileName, + modes=>UpdateModes, + file_check=>FileCheck, + fd=>Fd, + inode=>INode, + last_check=>timestamp(), + synced=>false, + write_res=>ok, + sync_res=>ok}, + State = update_rotation({Size,Count,Compress},State0), + {ok,State}; Error -> Error end; @@ -242,21 +332,23 @@ do_open_log_file({file,FileName,Modes}) -> _:Reason -> {error,Reason} end. -close_log_file(Std) when Std == standard_io; Std == standard_error -> - ok; -close_log_file({Fd,_}) -> +close_log_file(#{fd:=Fd}) -> _ = file:datasync(Fd), - _ = file:close(Fd). + _ = file:close(Fd), + ok; +close_log_file(_) -> + ok. + %%%----------------------------------------------------------------- %%% File control process -file_ctrl_start(HandlerName, FileInfo) -> +file_ctrl_start(HandlerName, HConfig) -> Starter = self(), FileCtrlPid = spawn_link(fun() -> - file_ctrl_init(HandlerName, FileInfo, Starter) + file_ctrl_init(HandlerName, HConfig, Starter) end), receive {FileCtrlPid,ok} -> @@ -271,18 +363,16 @@ file_ctrl_start(HandlerName, FileInfo) -> file_ctrl_stop(Pid) -> Pid ! stop. -file_write_async(Pid, Bin) -> +file_write(async, Pid, Bin) -> Pid ! {log,Bin}, - ok. - -file_write_sync(Pid, Bin) -> + ok; +file_write(sync, Pid, Bin) -> file_ctrl_call(Pid, {log,Bin}). -file_ctrl_filesync_async(Pid) -> +file_ctrl_filesync(async, Pid) -> Pid ! filesync, - ok. - -file_ctrl_filesync_sync(Pid) -> + ok; +file_ctrl_filesync(sync, Pid) -> file_ctrl_call(Pid, filesync). file_ctrl_call(Pid, Msg) -> @@ -299,98 +389,255 @@ file_ctrl_call(Pid, Msg) -> {error,{no_response,Pid}} end. -file_ctrl_init(HandlerName, FileInfo, Starter) when is_tuple(FileInfo) -> +file_ctrl_init(HandlerName, + #{type:=file, + file:=FileName} = HConfig, + Starter) -> process_flag(message_queue_data, off_heap), - FileName = element(2, FileInfo), - case do_open_log_file(FileInfo) of - {ok,File} -> + case open_log_file(HandlerName,HConfig) of + {ok,State} -> Starter ! {self(),ok}, - file_ctrl_loop(File, FileName, false, ok, ok, HandlerName); + file_ctrl_loop(State); {error,Reason} -> Starter ! {self(),{error,{open_failed,FileName,Reason}}} end; -file_ctrl_init(HandlerName, StdDev, Starter) -> +file_ctrl_init(HandlerName, #{type:=StdDev}, Starter) -> Starter ! {self(),ok}, - file_ctrl_loop(StdDev, StdDev, false, ok, ok, HandlerName). + file_ctrl_loop(#{handler_name=>HandlerName,dev=>StdDev}). -file_ctrl_loop(File, DevName, Synced, - PrevWriteResult, PrevSyncResult, HandlerName) -> +file_ctrl_loop(State) -> receive %% asynchronous event {log,Bin} -> - File1 = ensure(File, DevName), - Result = write_to_dev(File1, Bin, DevName, - PrevWriteResult, HandlerName), - file_ctrl_loop(File1, DevName, false, - Result, PrevSyncResult, HandlerName); + State1 = write_to_dev(Bin,State), + file_ctrl_loop(State1); %% synchronous event {{log,Bin},{From,MRef}} -> - File1 = ensure(File, DevName), - Result = write_to_dev(File1, Bin, DevName, - PrevWriteResult, HandlerName), + State1 = ensure_file(State), + State2 = write_to_dev(Bin,State1), From ! {MRef,ok}, - file_ctrl_loop(File1, DevName, false, - Result, PrevSyncResult, HandlerName); + file_ctrl_loop(State2); filesync -> - File1 = ensure(File, DevName), - Result = sync_dev(File1, DevName, Synced, - PrevSyncResult, HandlerName), - file_ctrl_loop(File1, DevName, true, - PrevWriteResult, Result, HandlerName); + State1 = sync_dev(State), + file_ctrl_loop(State1); {filesync,{From,MRef}} -> - File1 = ensure(File, DevName), - Result = sync_dev(File1, DevName, Synced, - PrevSyncResult, HandlerName), + State1 = ensure_file(State), + State2 = sync_dev(State1), From ! {MRef,ok}, - file_ctrl_loop(File1, DevName, true, - PrevWriteResult, Result, HandlerName); + file_ctrl_loop(State2); + + {update_config,#{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}} -> + State1 = update_rotation({Size,Count,Compress},State), + file_ctrl_loop(State1#{file_check=>FileCheck}); stop -> - _ = close_log_file(File), + close_log_file(State), stopped end. +maybe_ensure_file(#{file_check:=0}=State) -> + ensure_file(State); +maybe_ensure_file(#{last_check:=T0,file_check:=CheckInt}=State) + when is_integer(CheckInt) -> + T = timestamp(), + if T-T0 > CheckInt -> ensure_file(State); + true -> State + end; +maybe_ensure_file(State) -> + State. + %% In order to play well with tools like logrotate, we need to be able %% to re-create the file if it has disappeared (e.g. if rotated by %% logrotate) -ensure(Fd,DevName) when is_atom(DevName) -> - Fd; -ensure({Fd,INode},FileName) -> - case file:read_file_info(FileName) of - {ok,#file_info{inode=INode}} -> - {Fd,INode}; +ensure_file(#{fd:=Fd0,inode:=INode0,file_name:=FileName,modes:=Modes}=State) -> + case file:read_file_info(FileName,[raw]) of + {ok,#file_info{inode=INode0}} -> + State#{last_check=>timestamp()}; _ -> - _ = file:close(Fd), - _ = file:close(Fd), % delayed_write cause close not to close - case do_open_log_file({file,FileName}) of - {ok,File} -> - File; + close_log_file(Fd0), + case file:open(FileName,Modes) of + {ok,Fd} -> + {ok,#file_info{inode=INode}} = + file:read_file_info(FileName,[raw]), + State#{fd=>Fd,inode=>INode, + last_check=>timestamp(), + synced=>true,sync_res=>ok}; Error -> exit({could_not_reopen_file,Error}) end - end. + end; +ensure_file(State) -> + State. -write_to_dev(DevName, Bin, _DevName, _PrevWriteResult, _HandlerName) - when is_atom(DevName) -> - io:put_chars(DevName, Bin); -write_to_dev({Fd,_}, Bin, FileName, PrevWriteResult, HandlerName) -> +write_to_dev(Bin,#{dev:=DevName}=State) -> + io:put_chars(DevName, Bin), + State; +write_to_dev(Bin, State) -> + State1 = #{fd:=Fd} = maybe_ensure_file(State), Result = ?file_write(Fd, Bin), - maybe_notify_error(write,Result,PrevWriteResult,FileName,HandlerName). + State2 = maybe_rotate_file(Bin,State1), + maybe_notify_error(write,Result,State2), + State2#{synced=>false,write_res=>Result}. -sync_dev(_, _FileName, true, PrevSyncResult, _HandlerName) -> - PrevSyncResult; -sync_dev({Fd,_}, FileName, false, PrevSyncResult, HandlerName) -> +sync_dev(#{synced:=false}=State) -> + State1 = #{fd:=Fd} = maybe_ensure_file(State), Result = ?file_datasync(Fd), - maybe_notify_error(filesync,Result,PrevSyncResult,FileName,HandlerName). + maybe_notify_error(filesync,Result,State1), + State1#{synced=>true,sync_res=>Result}; +sync_dev(State) -> + State. -maybe_notify_error(_Op, ok, _PrevResult, _FileName, _HandlerName) -> +update_rotation({infinity,_,_},State) -> + maybe_remove_archives(0,State), + maps:remove(rotation,State); +update_rotation({Size,Count,Compress},#{file_name:=FileName} = State) -> + maybe_remove_archives(Count,State), + {ok,#file_info{size=CurrSize}} = file:read_file_info(FileName,[raw]), + State1 = State#{rotation=>#{size=>Size, + count=>Count, + compress=>Compress, + curr_size=>CurrSize}}, + maybe_update_compress(0,State1), + maybe_rotate_file(0,State1). + +maybe_remove_archives(Count,#{file_name:=FileName}=State) -> + Archive = rot_file_name(FileName,Count,false), + CompressedArchive = rot_file_name(FileName,Count,true), + case {file:read_file_info(Archive,[raw]), + file:read_file_info(CompressedArchive,[raw])} of + {{error,enoent},{error,enoent}} -> + ok; + _ -> + _ = file:delete(Archive), + _ = file:delete(CompressedArchive), + maybe_remove_archives(Count+1,State) + end. + +maybe_update_compress(Count,#{rotation:=#{count:=Count}}) -> + ok; +maybe_update_compress(N,#{file_name:=FileName, + rotation:=#{compress:=Compress}}=State) -> + Archive = rot_file_name(FileName,N,not Compress), + case file:read_file_info(Archive,[raw]) of + {ok,_} when Compress -> + compress_file(Archive); + {ok,_} -> + decompress_file(Archive); + _ -> + ok + end, + maybe_update_compress(N+1,State). + +maybe_rotate_file(Bin,#{rotation:=_}=State) when is_binary(Bin) -> + maybe_rotate_file(byte_size(Bin),State); +maybe_rotate_file(AddSize,#{rotation:=#{size:=RotSize, + curr_size:=CurrSize}=Rotation}=State) -> + NewSize = CurrSize + AddSize, + if NewSize>RotSize -> + rotate_file(State#{rotation=>Rotation#{curr_size=>NewSize}}); + true -> + State#{rotation=>Rotation#{curr_size=>NewSize}} + end; +maybe_rotate_file(_Bin,State) -> + State. + +rotate_file(#{fd:=Fd0,file_name:=FileName,modes:=Modes,rotation:=Rotation}=State) -> + State1 = sync_dev(State), + _ = file:close(Fd0), + _ = file:close(Fd0), + rotate_files(FileName,maps:get(count,Rotation),maps:get(compress,Rotation)), + case file:open(FileName,Modes) of + {ok,Fd} -> + {ok,#file_info{inode=INode}} = file:read_file_info(FileName,[raw]), + State1#{fd=>Fd,inode=>INode,rotation=>Rotation#{curr_size=>0}}; + Error -> + exit({could_not_reopen_file,Error}) + end. + +rotate_files(FileName,0,_Compress) -> + _ = file:delete(FileName), + ok; +rotate_files(FileName,1,Compress) -> + FileName0 = FileName++".0", + _ = file:rename(FileName,FileName0), + if Compress -> compress_file(FileName0); + true -> ok + end, ok; -maybe_notify_error(_Op, PrevResult, PrevResult, _FileName, _HandlerName) -> +rotate_files(FileName,Count,Compress) -> + _ = file:rename(rot_file_name(FileName,Count-2,Compress), + rot_file_name(FileName,Count-1,Compress)), + rotate_files(FileName,Count-1,Compress). + +rot_file_name(FileName,Count,false) -> + FileName ++ "." ++ integer_to_list(Count); +rot_file_name(FileName,Count,true) -> + rot_file_name(FileName,Count,false) ++ ".gz". + +compress_file(FileName) -> + {ok,In} = file:open(FileName,[read,binary]), + {ok,Out} = file:open(FileName++".gz",[write]), + Z = zlib:open(), + zlib:deflateInit(Z, default, deflated, 31, 8, default), + compress_data(Z,In,Out), + zlib:deflateEnd(Z), + zlib:close(Z), + _ = file:close(In), + _ = file:close(Out), + _ = file:delete(FileName), + ok. + +compress_data(Z,In,Out) -> + case file:read(In,100000) of + {ok,Data} -> + Compressed = zlib:deflate(Z, Data), + _ = file:write(Out,Compressed), + compress_data(Z,In,Out); + eof -> + Compressed = zlib:deflate(Z, <<>>, finish), + _ = file:write(Out,Compressed), + ok + end. + +decompress_file(FileName) -> + {ok,In} = file:open(FileName,[read,binary]), + {ok,Out} = file:open(filename:rootname(FileName,".gz"),[write]), + Z = zlib:open(), + zlib:inflateInit(Z, 31), + decompress_data(Z,In,Out), + zlib:inflateEnd(Z), + zlib:close(Z), + _ = file:close(In), + _ = file:close(Out), + _ = file:delete(FileName), + ok. + +decompress_data(Z,In,Out) -> + case file:read(In,1000) of + {ok,Data} -> + Decompressed = zlib:inflate(Z, Data), + _ = file:write(Out,Decompressed), + decompress_data(Z,In,Out); + eof -> + ok + end. + +maybe_notify_error(_Op, ok, _State) -> + ok; +maybe_notify_error(Op, Result, #{write_res:=WR,sync_res:=SR}) + when (Op==write andalso Result==WR) orelse + (Op==filesync andalso Result==SR) -> %% don't report same error twice - PrevResult; -maybe_notify_error(Op, Error, _PrevResult, FileName, HandlerName) -> + ok; +maybe_notify_error(Op, Error, #{handler_name:=HandlerName,file_name:=FileName}) -> logger_h_common:error_notify({HandlerName,Op,FileName,Error}), - Error. + ok. + +timestamp() -> + erlang:monotonic_time(millisecond). diff --git a/lib/kernel/test/kernel.spec b/lib/kernel/test/kernel.spec index 62afc9f97b..eaa17f3a59 100644 --- a/lib/kernel/test/kernel.spec +++ b/lib/kernel/test/kernel.spec @@ -2,3 +2,4 @@ {config, "../test_server/ts.unix.config"}. {suites,"../kernel_test", all}. +{skip_suites,"../kernel_test",[logger_stress_SUITE],"Benchmarks only"}. diff --git a/lib/kernel/test/kernel_config_SUITE.erl b/lib/kernel/test/kernel_config_SUITE.erl index 9207025a2c..57c44c2498 100644 --- a/lib/kernel/test/kernel_config_SUITE.erl +++ b/lib/kernel/test/kernel_config_SUITE.erl @@ -21,7 +21,8 @@ -include_lib("common_test/include/ct.hrl"). --export([all/0, suite/0,groups/0,init_per_group/2,end_per_group/2, sync/1]). +-export([all/0, suite/0,groups/0,init_per_group/2,end_per_group/2, + start_distribution_false/1, sync/1]). -export([init_per_suite/1, end_per_suite/1]). @@ -30,7 +31,7 @@ suite() -> {timetrap,{minutes,2}}]. all() -> - [sync]. + [sync, start_distribution_false]. groups() -> []. @@ -59,12 +60,9 @@ from(H, [H | T]) -> T; from(H, [_ | T]) -> from(H, T); from(_, []) -> []. -%%----------------------------------------------------------------- -%% Test suite for sync_nodes. This is quite tricky. -%% +%% Test sync_nodes. This is quite tricky. %% Should be started in a CC view with: %% erl -sname XXX where XX not in [cp1, cp2] -%%----------------------------------------------------------------- sync(Conf) when is_list(Conf) -> %% Write a config file Dir = proplists:get_value(priv_dir,Conf), @@ -106,9 +104,18 @@ wait_for_node(Node) -> _Other -> wait_for_node(Node) end. - stop_node(Node) -> M = list_to_atom(lists:concat([Node, [$@], from($@,atom_to_list(node()))])), rpc:cast(M, erlang, halt, []). + +start_distribution_false(Config) when is_list(Config) -> + %% When distribution is disabled, -sname/-name has no effect + Str = os:cmd(ct:get_progname() + ++ " -kernel start_distribution false" + ++ " -sname no_distribution" + ++ " -eval \"erlang:display(node())\"" + ++ " -noshell -s erlang halt"), + "'nonode@nohost'" ++ _ = Str, + ok. diff --git a/lib/kernel/test/logger_SUITE.erl b/lib/kernel/test/logger_SUITE.erl index 70bb775db8..035e5d8974 100644 --- a/lib/kernel/test/logger_SUITE.erl +++ b/lib/kernel/test/logger_SUITE.erl @@ -1048,8 +1048,11 @@ kernel_config(Config) -> ok = rpc:call(Node,logger,internal_init_logger,[]), ok = rpc:call(Node,logger,add_handlers,[kernel]), #{primary:=#{filter_default:=log,filters:=[]}, - handlers:=[#{id:=default,filters:=DF,config:=#{type:={file,F}}}], + handlers:=[#{id:=default,filters:=DF, + config:=#{type:=file,file:=F,modes:=Modes}}], module_levels:=[]} = rpc:call(Node,logger,get_config,[]), + [append,delayed_write,raw] = lists:sort(Modes), + %% Same, but using 'logger' parameter instead of 'error_logger' ok = rpc:call(Node,logger,remove_handler,[default]),% so it can be added again @@ -1060,26 +1063,27 @@ kernel_config(Config) -> ok = rpc:call(Node,logger,internal_init_logger,[]), ok = rpc:call(Node,logger,add_handlers,[kernel]), #{primary:=#{filter_default:=log,filters:=[]}, - handlers:=[#{id:=default,filters:=DF,config:=#{type:={file,F}}}], + handlers:=[#{id:=default,filters:=DF, + config:=#{type:=file,file:=F,modes:=Modes}}], module_levels:=[]} = rpc:call(Node,logger,get_config,[]), %% Same, but with type={file,File,Modes} ok = rpc:call(Node,logger,remove_handler,[default]),% so it can be added again ok = rpc:call(Node,application,unset_env,[kernel,error_logger]), - M = [raw,write,delayed_write], + M = [raw,write], ok = rpc:call(Node,application,set_env,[kernel,logger, [{handler,default,logger_std_h, #{config=>#{type=>{file,F,M}}}}]]), ok = rpc:call(Node,logger,internal_init_logger,[]), ok = rpc:call(Node,logger,add_handlers,[kernel]), #{primary:=#{filter_default:=log,filters:=[]}, - handlers:=[#{id:=default,filters:=DF,config:=#{type:={file,F,M}}}], + handlers:=[#{id:=default,filters:=DF, + config:=#{type:=file,file:=F,modes:=[delayed_write|M]}}], module_levels:=[]} = rpc:call(Node,logger,get_config,[]), %% Same, but with disk_log handler ok = rpc:call(Node,logger,remove_handler,[default]),% so it can be added again ok = rpc:call(Node,application,unset_env,[kernel,error_logger]), - M = [raw,write,delayed_write], ok = rpc:call(Node,application,set_env,[kernel,logger, [{handler,default,logger_disk_log_h, #{config=>#{file=>F}}}]]), diff --git a/lib/kernel/test/logger_std_h_SUITE.erl b/lib/kernel/test/logger_std_h_SUITE.erl index b2c2c8ba67..0c5516f82b 100644 --- a/lib/kernel/test/logger_std_h_SUITE.erl +++ b/lib/kernel/test/logger_std_h_SUITE.erl @@ -112,6 +112,8 @@ all() -> add_remove_instance_standard_error, add_remove_instance_file1, add_remove_instance_file2, + add_remove_instance_file3, + add_remove_instance_file4, default_formatter, filter_config, errors, @@ -142,7 +144,12 @@ all() -> restart_after, handler_requests_under_load, recreate_deleted_log, - reopen_changed_log + reopen_changed_log, + rotate_size, + rotate_size_compressed, + rotate_size_reopen, + rotation_opts, + rotation_opts_restart_handler ]. add_remove_instance_tty(_Config) -> @@ -179,10 +186,27 @@ add_remove_instance_file2(Config) -> add_remove_instance_file2(cleanup,_Config) -> logger_std_h_remove(). -add_remove_instance_file(Log, Type) -> +add_remove_instance_file3(_Config) -> + Log = atom_to_list(?MODULE), + StdHConfig = #{type=>file}, + add_remove_instance_file(Log, StdHConfig). +add_remove_instance_file3(cleanup,_Config) -> + logger_std_h_remove(). + +add_remove_instance_file4(Config) -> + Dir = ?config(priv_dir,Config), + Log = filename:join(Dir,"stdlog4.txt"), + StdHConfig = #{file=>Log,modes=>[]}, + add_remove_instance_file(Log, StdHConfig). +add_remove_instance_file4(cleanup,_Config) -> + logger_std_h_remove(). + +add_remove_instance_file(Log, Type) when not is_map(Type) -> + add_remove_instance_file(Log,#{type=>Type}); +add_remove_instance_file(Log, StdHConfig) when is_map(StdHConfig) -> ok = logger:add_handler(?MODULE, logger_std_h, - #{config => #{type => Type}, + #{config => StdHConfig, filter_default=>stop, filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), formatter=>{?MODULE,self()}}), @@ -257,9 +281,10 @@ errors(Config) -> end, {error, - {handler_not_added,{open_failed,Log,_}}} = + {handler_not_added, + {invalid_config,logger_std_h,#{modes:=bad_file_opt}}}} = logger:add_handler(myh3,logger_std_h, - #{config=>#{type=>{file,Log,[bad_file_opt]}}}), + #{config=>#{type=>{file,Log,bad_file_opt}}}), ok = logger:notice(?msg). @@ -607,24 +632,51 @@ reconfig(cleanup, _Config) -> file_opts(Config) -> Dir = ?config(priv_dir,Config), Log = filename:join(Dir, lists:concat([?FUNCTION_NAME,".log"])), - BadFileOpts = [raw], - BadType = {file,Log,BadFileOpts}, - {error,{handler_not_added,{open_failed,Log,enoent}}} = - logger:add_handler(?MODULE, logger_std_h, - #{config => #{type => BadType}}), + MissingOpts = [raw], + Type1 = {file,Log,MissingOpts}, + ok = logger:add_handler(?MODULE, logger_std_h, + #{config => #{type => Type1}}), + {ok,#{config:=#{type:=file,file:=Log,modes:=Modes1}}} = + logger:get_handler_config(?MODULE), + [append,delayed_write,raw] = lists:sort(Modes1), + ok = logger:remove_handler(?MODULE), OkFileOpts = [raw,append], OkType = {file,Log,OkFileOpts}, ok = logger:add_handler(?MODULE, logger_std_h, - #{config => #{type => OkType}, + #{config => #{type => OkType}, % old format filter_default=>log, filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), formatter=>{?MODULE,self()}}), - #{cb_state := #{handler_state := #{type := OkType}}} = + ModOpts = [delayed_write|OkFileOpts], + #{cb_state := #{handler_state := #{type:=file, + file:=Log, + modes:=ModOpts}}} = logger_olp:info(h_proc_name()), - {ok,#{config := #{type := OkType}}} = logger:get_handler_config(?MODULE), + {ok,#{config := #{type:=file, + file:=Log, + modes:=ModOpts}}} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + + ok = logger:add_handler(?MODULE, + logger_std_h, + #{config => #{type => file, + file => Log, + modes => OkFileOpts}, % new format + filter_default=>log, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), + formatter=>{?MODULE,self()}}), + + #{cb_state := #{handler_state := #{type:=file, + file:=Log, + modes:=ModOpts}}} = + logger_olp:info(h_proc_name()), + {ok,#{config := #{type:=file, + file:=Log, + modes:=ModOpts}}} = + logger:get_handler_config(?MODULE), logger:notice(M1=?msg,?domain), ?check(M1), B1 = ?bin(M1), @@ -640,13 +692,14 @@ sync(Config) -> Type = {file,Log}, ok = logger:add_handler(?MODULE, logger_std_h, - #{config => #{type => Type}, + #{config => #{type => Type, + file_check => 10000}, filter_default=>log, filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), formatter=>{?MODULE,nl}}), %% check repeated filesync happens - start_tracer([{logger_std_h, write_to_dev, 5}, + start_tracer([{logger_std_h, write_to_dev, 2}, {file, datasync, 1}], [{logger_std_h, write_to_dev, <<"first\n">>}, {file,datasync}]), @@ -656,7 +709,7 @@ sync(Config) -> check_tracer(filesync_rep_int()*2), %% check that explicit filesync is only done once - start_tracer([{logger_std_h, write_to_dev, 5}, + start_tracer([{logger_std_h, write_to_dev, 2}, {file, datasync, 1}], [{logger_std_h, write_to_dev, <<"second\n">>}, {file,datasync}, @@ -675,7 +728,7 @@ sync(Config) -> #{filesync_repeat_interval => no_repeat}), no_repeat = maps:get(filesync_repeat_interval, maps:get(cb_state, logger_olp:info(h_proc_name()))), - start_tracer([{logger_std_h, write_to_dev, 5}, + start_tracer([{logger_std_h, write_to_dev, 2}, {file, datasync, 1}], [{logger_std_h, write_to_dev, <<"third\n">>}, {file,datasync}, @@ -1285,6 +1338,331 @@ reopen_changed_log(Config) -> reopen_changed_log(cleanup, _Config) -> ok = stop_handler(?MODULE). +rotate_size(Config) -> + {Log,_HConfig,_StdHConfig} = + start_handler(?MODULE, ?FUNCTION_NAME, Config), + ok = logger:update_handler_config(?MODULE,#{config=>#{max_no_bytes=>1000, + max_no_files=>2}}), + + Str = lists:duplicate(19,$a), + [logger:notice(Str,?domain) || _ <- lists:seq(1,50)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=1000}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + + logger:notice(Str,?domain), + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".0"), + {error,enoent} = file:read_file_info(Log++".1"), + + [logger:notice(Str,?domain) || _ <- lists:seq(1,51)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".0"), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".1"), + {error,enoent} = file:read_file_info(Log++".2"), + + [logger:notice(Str,?domain) || _ <- lists:seq(1,50)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=1000}} = file:read_file_info(Log), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".0"), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".1"), + {error,enoent} = file:read_file_info(Log++".2"), + + logger:notice("bbbb",?domain), + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=1005}} = file:read_file_info(Log++".0"), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".1"), + {error,enoent} = file:read_file_info(Log++".2"), + + ok. +rotate_size(cleanup,_Config) -> + ok = stop_handler(?MODULE). + +rotate_size_compressed(Config) -> + {Log,_HConfig,_StdHConfig} = + start_handler(?MODULE, ?FUNCTION_NAME, Config), + ok = logger:update_handler_config(?MODULE, + #{config=>#{max_no_bytes=>1000, + max_no_files=>2, + compress_on_rotate=>true}}), + Str = lists:duplicate(19,$a), + [logger:notice(Str,?domain) || _ <- lists:seq(1,50)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=1000}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + {error,enoent} = file:read_file_info(Log++".0.gz"), + + logger:notice(Str,?domain), + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".0.gz"), + {error,enoent} = file:read_file_info(Log++".1"), + {error,enoent} = file:read_file_info(Log++".1.gz"), + + [logger:notice(Str,?domain) || _ <- lists:seq(1,51)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".0.gz"), + {error,enoent} = file:read_file_info(Log++".1"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".1.gz"), + {error,enoent} = file:read_file_info(Log++".2"), + {error,enoent} = file:read_file_info(Log++".2.gz"), + + [logger:notice(Str,?domain) || _ <- lists:seq(1,50)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=1000}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".0.gz"), + {error,enoent} = file:read_file_info(Log++".1"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".1.gz"), + {error,enoent} = file:read_file_info(Log++".2"), + {error,enoent} = file:read_file_info(Log++".2.gz"), + + logger:notice("bbbb",?domain), + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + {ok,#file_info{size=38}} = file:read_file_info(Log++".0.gz"), + {error,enoent} = file:read_file_info(Log++".1"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".1.gz"), + {error,enoent} = file:read_file_info(Log++".2"), + {error,enoent} = file:read_file_info(Log++".2.gz"), + + ok. +rotate_size_compressed(cleanup,_Config) -> + ok = stop_handler(?MODULE). + +rotate_size_reopen(Config) -> + {Log,_HConfig,_StdHConfig} = + start_handler(?MODULE, ?FUNCTION_NAME, Config), + ok = logger:update_handler_config(?MODULE,#{config=>#{max_no_bytes=>1000, + max_no_files=>2}}), + + Str = lists:duplicate(19,$a), + [logger:notice(Str,?domain) || _ <- lists:seq(1,40)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=800}} = file:read_file_info(Log), + + {ok,HConfig} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + ok = logger:add_handler(?MODULE,maps:get(module,HConfig),HConfig), + {ok,#file_info{size=800}} = file:read_file_info(Log), + + [logger:notice(Str,?domain) || _ <- lists:seq(1,40)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=580}} = file:read_file_info(Log), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".0"), + ok. +rotate_size_reopen(cleanup,_Config) -> + ok = stop_handler(?MODULE). + +rotation_opts(Config) -> + {Log,_HConfig,StdHConfig} = + start_handler(?MODULE, ?FUNCTION_NAME, Config), + #{max_no_bytes:=infinity, + max_no_files:=0, + compress_on_rotate:=false} = StdHConfig, + + %% Test bad rotation config + {error,{invalid_config,_,_}} = + logger:update_handler_config(?MODULE,config,#{max_no_bytes=>0}), + {error,{invalid_config,_,_}} = + logger:update_handler_config(?MODULE,config,#{max_no_files=>infinity}), + {error,{invalid_config,_,_}} = + logger:update_handler_config(?MODULE,config, + #{compress_on_rotate=>undefined}), + + + %% Test good rotation config - start with no rotation + Str = lists:duplicate(19,$a), + [logger:notice(Str,?domain) || _ <- lists:seq(1,10)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=200}} = file:read_file_info(Log), + [] = filelib:wildcard(Log++".*"), + + %% Turn on rotation, check that existing file is rotated since its + %% size exceeds max_no_bytes + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_bytes=>100, + max_no_files=>2}), + timer:sleep(100), % give some time to execute config_changed + {ok,#file_info{size=0}} = file:read_file_info(Log), + Log0 = Log++".0", + {ok,#file_info{size=200}} = file:read_file_info(Log0), + [Log0] = filelib:wildcard(Log++".*"), + + %% Fill all logs + [logger:notice(Str,?domain) || _ <- lists:seq(1,13)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=20}} = file:read_file_info(Log), + {ok,#file_info{size=120}} = file:read_file_info(Log0), + {ok,#file_info{size=120}} = file:read_file_info(Log++".1"), + [_,_] = filelib:wildcard(Log++".*"), + + %% Extend size and count and check that nothing changes with existing files + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_bytes=>200, + max_no_files=>3}), + timer:sleep(100), % give some time to execute config_changed + {ok,#file_info{size=20}} = file:read_file_info(Log), + {ok,#file_info{size=120}} = file:read_file_info(Log0), + {ok,#file_info{size=120}} = file:read_file_info(Log++".1"), + [_,_] = filelib:wildcard(Log++".*"), + + %% Add more log events and see that extended size and count works + [logger:notice(Str,?domain) || _ <- lists:seq(1,10)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=220}} = file:read_file_info(Log0), + {ok,#file_info{size=120}} = file:read_file_info(Log++".1"), + {ok,#file_info{size=120}} = file:read_file_info(Log++".2"), + [_,_,_] = filelib:wildcard(Log++".*"), + + %% Reduce count and check that archive files that exceed the new + %% count are moved + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_files=>1}), + timer:sleep(100), % give some time to execute config_changed + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=220}} = file:read_file_info(Log0), + [Log0] = filelib:wildcard(Log++".*"), + + %% Extend size and count again, and turn on compression. Check + %% that archives are compressed + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_bytes=>100, + max_no_files=>2, + compress_on_rotate=>true}), + timer:sleep(100), % give some time to execute config_changed + {ok,#file_info{size=0}} = file:read_file_info(Log), + Log0gz = Log0++".gz", + {ok,#file_info{size=29}} = file:read_file_info(Log0gz), + [Log0gz] = filelib:wildcard(Log++".*"), + + %% Fill all logs + [logger:notice(Str,?domain) || _ <- lists:seq(1,13)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=20}} = file:read_file_info(Log), + {ok,#file_info{size=29}} = file:read_file_info(Log0gz), + {ok,#file_info{size=29}} = file:read_file_info(Log++".1.gz"), + [_,_] = filelib:wildcard(Log++".*"), + + %% Reduce count and turn off compression. Check that archives that + %% exceeds the new count are removed, and the rest are + %% uncompressed. + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_files=>1, + compress_on_rotate=>false}), + timer:sleep(100), % give some time to execute config_changed + {ok,#file_info{size=20}} = file:read_file_info(Log), + {ok,#file_info{size=120}} = file:read_file_info(Log0), + [Log0] = filelib:wildcard(Log++".*"), + + %% Check that config and handler state agree on the current rotation settings + {ok,#{config:=#{max_no_bytes:=100, + max_no_files:=1, + compress_on_rotate:=false}}} = + logger:get_handler_config(?MODULE), + #{cb_state:=#{handler_state:=#{max_no_bytes:=100, + max_no_files:=1, + compress_on_rotate:=false}}} = + logger_olp:info(h_proc_name()), + ok. +rotation_opts(cleanup,_Config) -> + ok = stop_handler(?MODULE). + +rotation_opts_restart_handler(Config) -> + {Log,_HConfig,_StdHConfig} = + start_handler(?MODULE, ?FUNCTION_NAME, Config), + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_bytes=>100, + max_no_files=>2}), + + %% Fill all logs + Str = lists:duplicate(19,$a), + [logger:notice(Str,?domain) || _ <- lists:seq(1,15)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=60}} = file:read_file_info(Log), + {ok,#file_info{size=120}} = file:read_file_info(Log++".0"), + {ok,#file_info{size=120}} = file:read_file_info(Log++".1"), + [_,_] = filelib:wildcard(Log++".*"), + + %% Stop/start handler and turn off rotation. Check that archives are removed. + {ok,#{config:=StdHConfig1}=HConfig1} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + ok = logger:add_handler( + ?MODULE,logger_std_h, + HConfig1#{config=>StdHConfig1#{max_no_bytes=>infinity}}), + timer:sleep(100), + {ok,#file_info{size=60}} = file:read_file_info(Log), + [] = filelib:wildcard(Log++".*"), + + %% Add some log events and check that file is no longer rotated. + [logger:notice(Str,?domain) || _ <- lists:seq(1,10)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=260}} = file:read_file_info(Log), + [] = filelib:wildcard(Log++".*"), + + %% Stop/start handler and trun on rotation. Check that file is rotated. + {ok,#{config:=StdHConfig2}=HConfig2} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + ok = logger:add_handler( + ?MODULE,logger_std_h, + HConfig2#{config=>StdHConfig2#{max_no_bytes=>100, + max_no_files=>2}}), + timer:sleep(100), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=260}} = file:read_file_info(Log++".0"), + [_] = filelib:wildcard(Log++".*"), + + %% Fill all logs + [logger:notice(Str,?domain) || _ <- lists:seq(1,10)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=80}} = file:read_file_info(Log), + {ok,#file_info{size=120}} = file:read_file_info(Log++".0"), + {ok,#file_info{size=260}} = file:read_file_info(Log++".1"), + + %% Stop/start handler, reduce count and turn on compression. Check + %% that excess archives are removed, and the rest compressed. + {ok,#{config:=StdHConfig3}=HConfig3} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + ok = logger:add_handler( + ?MODULE,logger_std_h, + HConfig3#{config=>StdHConfig3#{max_no_bytes=>75, + max_no_files=>1, + compress_on_rotate=>true}}), + timer:sleep(100), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=29}} = file:read_file_info(Log++".0.gz"), + [_] = filelib:wildcard(Log++".*"), + + %% Stop/start handler and turn off compression. Check that achives + %% are decompressed. + {ok,#{config:=StdHConfig4}=HConfig4} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + ok = logger:add_handler( + ?MODULE,logger_std_h, + HConfig4#{config=>StdHConfig4#{compress_on_rotate=>false}}), + timer:sleep(100), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=80}} = file:read_file_info(Log++".0"), + [_] = filelib:wildcard(Log++".*"), + + ok. +rotation_opts_restart_handler(cleanup,_Config) -> + ok = stop_handler(?MODULE). + %%%----------------------------------------------------------------- %%% send_requests(TO, Reqs = [{Mod,Func,Args,Res}|Rs]) -> @@ -1305,8 +1683,8 @@ start_handler(Name, TTY, _Config) when TTY == standard_io; ok = logger:add_handler(Name, logger_std_h, #{config => #{type => TTY}, - filter_default=>log, - filters=>?DEFAULT_HANDLER_FILTERS([Name]), + filter_default=>stop, + filters=>filter_only_this_domain(Name), formatter=>{?MODULE,op}}), {ok,HConfig = #{config := StdHConfig}} = logger:get_handler_config(Name), {HConfig,StdHConfig}; @@ -1320,12 +1698,17 @@ start_handler(Name, FuncName, Config) -> ok = logger:add_handler(Name, logger_std_h, #{config => #{type => Type}, - filter_default=>log, - filters=>?DEFAULT_HANDLER_FILTERS([Name]), + filter_default=>stop, + filters=>filter_only_this_domain(Name), formatter=>{?MODULE,op}}), {ok,HConfig = #{config := StdHConfig}} = logger:get_handler_config(Name), {Log,HConfig,StdHConfig}. + +filter_only_this_domain(Name) -> + [{remote_gl,{fun logger_filters:remote_gl/2,stop}}, + {domain,{fun logger_filters:domain/2,{log,super,[Name]}}}]. + stop_handler(Name) -> R = logger:remove_handler(Name), ct:pal("Handler ~p stopped! Result: ~p", [Name,R]), @@ -1658,7 +2041,7 @@ tpl([]) -> tracer({trace,_,call,{logger_h_common,handle_cast,[Op|_]}}, {Pid,[{Mod,Func,Op}|Expected]}) -> maybe_tracer_done(Pid,Expected,{Mod,Func,Op}); -tracer({trace,_,call,{Mod=logger_std_h,Func=write_to_dev,[_,Data,_,_,_]}}, +tracer({trace,_,call,{Mod=logger_std_h,Func=write_to_dev,[Data,_]}}, {Pid,[{Mod,Func,Data}|Expected]}) -> maybe_tracer_done(Pid,Expected,{Mod,Func,Data}); tracer({trace,_,call,{Mod,Func,_}}, {Pid,[{Mod,Func}|Expected]}) -> @@ -1742,4 +2125,3 @@ filesync_rep_int() -> file_delete(Log) -> file:delete(Log). - diff --git a/lib/kernel/test/logger_stress_SUITE.erl b/lib/kernel/test/logger_stress_SUITE.erl index 4072e8c86a..1a278fb1b2 100644 --- a/lib/kernel/test/logger_stress_SUITE.erl +++ b/lib/kernel/test/logger_stress_SUITE.erl @@ -67,6 +67,10 @@ all() -> reject_events, std_handler, disk_log_handler, + std_handler_time, + std_handler_time_big, + disk_log_handler_time, + disk_log_handler_time_big, emulator_events, remote_events, remote_to_disk_log, @@ -123,6 +127,20 @@ std_handler(cleanup,_Config) -> _ = file:delete("default.log"), ok. +%% Disable overload protection and just print a lot - measure time. +%% The IOPS reported is the number of log events written per millisecond. +std_handler_time(Config) -> + measure_handler_time(logger_std_h,#{type=>{file,"default.log"}},Config). +std_handler_time(cleanup,_Config) -> + _ = file:delete("default.log"), + ok. + +std_handler_time_big(Config) -> + measure_handler_time_big(logger_std_h,#{type=>{file,"default.log"}},Config). +std_handler_time_big(cleanup,_Config) -> + _ = file:delete("default.log"), + ok. + %% Cascading failure that produce gen_server and proc_lib reports - %% how many of the produced log events are actually written to a log %% with logger_disk_log_h wrap file handler. @@ -140,6 +158,20 @@ disk_log_handler(cleanup,_Config) -> [_ = file:delete(F) || F <- Files], ok. +%% Disable overload protection and just print a lot - measure time. +%% The IOPS reported is the number of log events written per millisecond. +disk_log_handler_time(Config) -> + measure_handler_time(logger_disk_log_h,#{type=>halt},Config). +disk_log_handler_time(cleanup,_Config) -> + _ = file:delete("default"), + ok. + +disk_log_handler_time_big(Config) -> + measure_handler_time_big(logger_disk_log_h,#{type=>halt},Config). +disk_log_handler_time_big(cleanup,_Config) -> + _ = file:delete("default"), + ok. + %% Cascading failure that produce log events from the emulator - how %% many of the produced log events pass through the proxy. emulator_events(Config) -> @@ -221,6 +253,68 @@ remote_emulator_to_disk_log(cleanup,_Config) -> %%%----------------------------------------------------------------- %%% Internal functions +measure_handler_time(Module,HCfg,Config) -> + measure_handler_time(Module,100000,small_fa(),millisecond,HCfg,#{},Config). + +measure_handler_time_big(Module,HCfg,Config) -> + FCfg = #{chars_limit=>4096, max_size=>1024}, + measure_handler_time(Module,100,big_fa(),second,HCfg,FCfg,Config). + +measure_handler_time(Module,N,FA,Unit,HCfg,FCfg,Config) -> + {ok,_,Node} = + logger_test_lib:setup( + Config, + [{logger, + [{handler,default,Module, + #{formatter => {logger_formatter, + maps:merge(#{legacy_header=>false, + single_line=>true},FCfg)}, + config=>maps:merge(#{sync_mode_qlen => N+1, + drop_mode_qlen => N+1, + flush_qlen => N+1, + burst_limit_enable => false, + filesync_repeat_interval => no_repeat}, + HCfg)}}]}]), + %% HPid = rpc:call(Node,erlang,whereis,[?name_to_reg_name(Module,default)]), + %% {links,[_,FCPid]} = rpc:call(Node,erlang,process_info,[HPid,links]), + T0 = erlang:monotonic_time(millisecond), + ok = rpc:call(Node,?MODULE,nlogs_wait,[N,FA,Module]), + %% ok = rpc:call(Node,fprof,apply,[fun ?MODULE:nlogs_wait/2,[N div 10,FA,Module],[{procs,[HPid,FCPid]}]]), + T1 = erlang:monotonic_time(millisecond), + T = T1-T0, + M = case Unit of + millisecond -> 1; + second -> 1000 + end, + IOPS = M*N/T, + ct:pal("N: ~p~nT: ~p~nIOPS: ~.2f events pr ~w",[N,T,IOPS,Unit]), + %% Stats = rpc:call(Node,logger_olp,info,[?name_to_reg_name(Module,default)]), + %% ct:pal("Stats: ~p",[Stats]), + ct_event:notify(#event{name = benchmark_data, + data = [{value,IOPS}]}), + {comment,io_lib:format("~.2f events written pr ~w",[IOPS,Unit])}. + +nlogs_wait(N,{F,A},Module) -> + group_leader(whereis(user),self()), + [?LOG_NOTICE(F,A) || _ <- lists:seq(1,N)], + wait(Module). + +wait(Module) -> + case Module:filesync(default) of + {error,handler_busy} -> + wait(Module); + ok -> + ok + end. + +small_fa() -> + Str = "\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "[\\]^_`abcdefghijklmnopqr", + {"~ts",[Str]}. + +big_fa() -> + {"~p",[lists:duplicate(1048576,"a")]}. + nlogs(N) -> group_leader(whereis(user),self()), Str = "\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ" diff --git a/lib/kernel/vsn.mk b/lib/kernel/vsn.mk index 4b43c6ae9d..6e0eea7824 100644 --- a/lib/kernel/vsn.mk +++ b/lib/kernel/vsn.mk @@ -1 +1 @@ -KERNEL_VSN = 6.2 +KERNEL_VSN = 6.2.1 diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index 422c4c94ae..e5b01cce5f 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -2736,6 +2736,11 @@ ssl_options_list([Key | Keys], [Value | Values], Acc) -> handle_active_option(false, connection = StateName, To, Reply, State) -> hibernate_after(StateName, State, [{reply, To, Reply}]); +handle_active_option(_, connection = StateName, To, _Reply, #state{connection_env = #connection_env{terminated = true}, + user_data_buffer = {_,0,_}} = State) -> + handle_normal_shutdown(?ALERT_REC(?FATAL, ?CLOSE_NOTIFY, all_data_deliverd), StateName, + State#state{start_or_recv_from = To}), + {stop,{shutdown, peer_close}, State}; handle_active_option(_, connection = StateName0, To, Reply, #state{static_env = #static_env{protocol_cb = Connection}, user_data_buffer = {_,0,_}} = State0) -> case Connection:next_event(StateName0, no_record, State0) of diff --git a/lib/stdlib/src/io_lib_pretty.erl b/lib/stdlib/src/io_lib_pretty.erl index 5483ea87b5..8f2fd7ea8f 100644 --- a/lib/stdlib/src/io_lib_pretty.erl +++ b/lib/stdlib/src/io_lib_pretty.erl @@ -721,7 +721,7 @@ printable_list(_L, 1, _T, _Enc) -> printable_list(L, _D, T, latin1) when T < 0 -> io_lib:printable_latin1_list(L); printable_list(L, _D, T, Enc) when T >= 0 -> - case slice(L, tsub(T, 2)) of + case slice(L, tsub(T, 2), Enc) of false -> false; {prefix, Prefix} when Enc =:= latin1 -> @@ -737,20 +737,46 @@ printable_list(L, _D, T, Enc) when T >= 0 -> printable_list(L, _D, T, _Uni) when T < 0-> io_lib:printable_list(L). -slice(L, N) -> - try io_lib:chars_length(L) =< N of - true -> +slice(L, N, latin1) -> + try lists:split(N, L) of + {_, []} -> all; - false -> - case string:slice(L, 0, N) of - "" -> - false; - Prefix -> - {prefix, Prefix} + {[], _} -> + false; + {L1, _} -> + {prefix, L1} + catch + _:_ -> + all + end; +slice(L, N, _Uni) -> + %% Be careful not to traverse more of L than necessary. + try string:slice(L, 0, N) of + "" -> + false; + Prefix -> + %% Assume no binaries are introduced by string:slice(). + case is_flat(L, lists:flatlength(Prefix)) of + true -> + case string:equal(Prefix, L) of + true -> + all; + false -> + {prefix, Prefix} + end; + false -> + false end catch _:_ -> false end. +is_flat(_L, 0) -> + true; +is_flat([C|Cs], N) when is_integer(C) -> + is_flat(Cs, N - 1); +is_flat(_, _N) -> + false. + printable_bin0(Bin, D, T, Enc) -> Len = case D >= 0 of true -> diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index 22c77aa172..7703198c4c 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -59,6 +59,7 @@ -export([otp_5340/1]). -export([otp_6338/1]). -export([otp_6842_select_1000/1]). +-export([select_mbuf_trapping/1]). -export([otp_7665/1]). -export([meta_wb/1]). -export([grow_shrink/1, grow_pseudo_deleted/1, shrink_pseudo_deleted/1]). @@ -132,6 +133,7 @@ all() -> t_named_select, select_fail, t_insert_new, t_repair_continuation, otp_5340, otp_6338, otp_6842_select_1000, otp_7665, + select_mbuf_trapping, otp_8732, meta_wb, grow_shrink, grow_pseudo_deleted, shrink_pseudo_deleted, {group, meta_smp}, smp_insert, smp_fixed_delete, smp_unfix_fix, smp_select_replace, @@ -5292,6 +5294,61 @@ otp_6338(Config) when is_list(Config) -> end), ok. +%% OTP-15660: Verify select not doing excessive trapping +%% when process have mbuf heap fragments. +select_mbuf_trapping(Config) when is_list(Config) -> + select_mbuf_trapping_do(set), + select_mbuf_trapping_do(ordered_set). + +select_mbuf_trapping_do(Type) -> + T = ets:new(xxx, [Type]), + NKeys = 50, + [ets:insert(T, {K, value}) || K <- lists:seq(1,NKeys)], + + {priority, Prio} = process_info(self(), priority), + Tracee = self(), + [SchedTracer] + = start_loopers(1, Prio, + fun (SC) -> + receive + {trace, Tracee, out, _} -> + SC+1; + done -> + Tracee ! {schedule_count, SC}, + exit(normal) + end + end, + 0), + + erlang:garbage_collect(), + 1 = erlang:trace(self(), true, [running,{tracer,SchedTracer}]), + + %% Artificially create an mbuf heap fragment + MbufTerm = "Frag me up", + MbufTerm = erts_debug:set_internal_state(mbuf, MbufTerm), + + Keys = ets:select(T, [{{'$1', value}, [], ['$1']}]), + NKeys = length(Keys), + + 1 = erlang:trace(self(), false, [running]), + Ref = erlang:trace_delivered(Tracee), + receive + {trace_delivered, Tracee, Ref} -> + SchedTracer ! done + end, + receive + {schedule_count, N} -> + io:format("~p context switches: ~p", [Type,N]), + if + N < 3 -> ok; + true -> ct:fail(failed) + end + end, + true = ets:delete(T), + ok. + + + %% Elements could come in the wrong order in a bag if a rehash occurred. otp_5340(Config) when is_list(Config) -> repeat_for_opts(fun otp_5340_do/1). diff --git a/lib/stdlib/test/io_SUITE.erl b/lib/stdlib/test/io_SUITE.erl index f097552e8c..824f5d19f2 100644 --- a/lib/stdlib/test/io_SUITE.erl +++ b/lib/stdlib/test/io_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2018. All Rights Reserved. +%% Copyright Ericsson AB 1999-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -32,7 +32,7 @@ io_with_huge_message_queue/1, format_string/1, maps/1, coverage/1, otp_14178_unicode_atoms/1, otp_14175/1, otp_14285/1, limit_term/1, otp_14983/1, otp_15103/1, otp_15076/1, - otp_15159/1]). + otp_15159/1, otp_15639/1]). -export([pretty/2, trf/3]). @@ -64,7 +64,8 @@ all() -> io_lib_print_binary_depth_one, otp_10302, otp_10755, otp_10836, io_lib_width_too_small, io_with_huge_message_queue, format_string, maps, coverage, otp_14178_unicode_atoms, otp_14175, - otp_14285, limit_term, otp_14983, otp_15103, otp_15076, otp_15159]. + otp_14285, limit_term, otp_14983, otp_15103, otp_15076, otp_15159, + otp_15639]. %% Error cases for output. error_1(Config) when is_list(Config) -> @@ -2647,3 +2648,35 @@ otp_15076(_Config) -> {'EXIT', {badarg, _}} = (catch io_lib:build_text(L)), {'EXIT', {badarg, _}} = (catch io_lib:build_text(L, [])), ok. + +otp_15639(_Config) -> + L = lists:duplicate(10, "a"), + LOpts = [{encoding, latin1}, {chars_limit, 10}], + UOpts = [{encoding, unicode}, {chars_limit, 10}], + "[[...]|...]" = pretty(L, LOpts), + "[[...]|...]" = pretty(L, UOpts), + "[\"a\",[...]|...]" = + pretty(L, [{chars_limit, 12}, {encoding, latin1}]), + "[\"a\",[...]|...]" = + pretty(L, [{chars_limit, 12}, {encoding, unicode}]), + + %% Latin-1 + "\"12345678\"" = pretty("12345678", LOpts), + "\"12345678\"..." = pretty("123456789", LOpts), + "\"\\r\\n123456\"..." = pretty("\r\n1234567", LOpts), + "\"\\r1234567\"..." = pretty("\r12345678", LOpts), + "\"\\r\\n123456\"..." = pretty("\r\n12345678", LOpts), + "\"12345678\"..." = pretty("12345678"++[x], LOpts), + "[49,50|...]" = pretty("1234567"++[x], LOpts), + "[49,x]" = pretty("1"++[x], LOpts), + "[[...]|...]" = pretty(["1","2","3","4","5","6","7","8"], LOpts), + %% Unicode + "\"12345678\"" = pretty("12345678", UOpts), + "\"12345678\"..." = pretty("123456789", UOpts), + "\"\\r\\n1234567\"" = pretty("\r\n1234567", UOpts), + "\"\\r1234567\"..." = pretty("\r12345678", UOpts), + "\"\\r\\n1234567\"..." = pretty("\r\n12345678", UOpts), + "[49,50|...]" = pretty("12345678"++[x], UOpts), + "\"12345678\"..." = pretty("123456789"++[x], UOpts), + "[[...]|...]" = pretty(["1","2","3","4","5","6","7","8"], UOpts), + ok. diff --git a/lib/tools/c_src/Makefile.in b/lib/tools/c_src/Makefile.in index cfe91917f8..289322b6fa 100644 --- a/lib/tools/c_src/Makefile.in +++ b/lib/tools/c_src/Makefile.in @@ -29,7 +29,6 @@ CC=@CC@ LD=@LD@ AR=@AR@ RANLIB=@RANLIB@ -RM=@RM@ MKDIR=@MKDIR@ INSTALL=@INSTALL@ INSTALL_DIR=@INSTALL_DIR@ @@ -158,9 +157,9 @@ $(ERTS_LIB): docs: clean: - $(RM) -rf ../obj/* - $(RM) -rf ../bin/* - $(RM) -f ./*~ + $(RM) -r ../obj/* + $(RM) -r ../bin/* + $(RM) ./*~ .PHONY: all erts_lib docs clean |