diff options
Diffstat (limited to 'lib/kernel/src')
38 files changed, 2152 insertions, 1082 deletions
diff --git a/lib/kernel/src/Makefile b/lib/kernel/src/Makefile index 57f17defc8..fcb599859b 100644 --- a/lib/kernel/src/Makefile +++ b/lib/kernel/src/Makefile @@ -118,10 +118,11 @@ MODULES = \ logger_h_common \ logger_filters \ logger_formatter \ + logger_olp \ + logger_proxy \ logger_server \ logger_simple_h \ logger_sup \ - net \ net_adm \ net_kernel \ os \ @@ -151,7 +152,7 @@ INTERNAL_HRL_FILES= application_master.hrl disk_log.hrl \ inet_dns.hrl inet_res.hrl \ inet_boot.hrl inet_config.hrl inet_int.hrl \ inet_dns_record_adts.hrl \ - logger_internal.hrl logger_h_common.hrl + logger_internal.hrl logger_olp.hrl logger_h_common.hrl ERL_FILES= $(MODULES:%=%.erl) @@ -174,8 +175,10 @@ APPUP_TARGET= $(EBIN)/$(APPUP_FILE) ifeq ($(NATIVE_LIBS_ENABLED),yes) ERL_COMPILE_FLAGS += +native +else +ERL_COMPILE_FLAGS += -Werror endif -ERL_COMPILE_FLAGS += -I../include -Werror +ERL_COMPILE_FLAGS += -I../include # ---------------------------------------------------- # Targets @@ -279,6 +282,8 @@ $(EBIN)/logger_config.beam: logger_internal.hrl ../include/logger.hrl $(EBIN)/logger_disk_log_h.beam: logger_h_common.hrl logger_internal.hrl ../include/logger.hrl ../include/file.hrl $(EBIN)/logger_filters.beam: logger_internal.hrl ../include/logger.hrl $(EBIN)/logger_formatter.beam: logger_internal.hrl ../include/logger.hrl +$(EBIN)/logger_olp.beam: logger_olp.hrl logger_internal.hrl +$(EBIN)/logger_proxy.beam: logger_internal.hrl $(EBIN)/logger_server.beam: logger_internal.hrl ../include/logger.hrl $(EBIN)/logger_simple_h.beam: logger_internal.hrl ../include/logger.hrl $(EBIN)/logger_std_h.beam: logger_h_common.hrl logger_internal.hrl ../include/logger.hrl ../include/file.hrl diff --git a/lib/kernel/src/application.erl b/lib/kernel/src/application.erl index bc6be2f8f5..5c2e981e4b 100644 --- a/lib/kernel/src/application.erl +++ b/lib/kernel/src/application.erl @@ -25,7 +25,7 @@ which_applications/0, which_applications/1, loaded_applications/0, permit/2]). -export([ensure_started/1, ensure_started/2]). --export([set_env/3, set_env/4, unset_env/2, unset_env/3]). +-export([set_env/1, set_env/2, set_env/3, set_env/4, unset_env/2, unset_env/3]). -export([get_env/1, get_env/2, get_env/3, get_all_env/0, get_all_env/1]). -export([get_key/1, get_key/2, get_all_key/0, get_all_key/1]). -export([get_application/0, get_application/1, info/0]). @@ -279,6 +279,26 @@ loaded_applications() -> info() -> application_controller:info(). +-spec set_env(Config) -> 'ok' when + Config :: [{Application, Env}], + Application :: atom(), + Env :: [{Par :: atom(), Val :: term()}]. + +set_env(Config) when is_list(Config) -> + set_env(Config, []). + +-spec set_env(Config, Opts) -> 'ok' when + Config :: [{Application, Env}], + Application :: atom(), + Env :: [{Par :: atom(), Val :: term()}], + Opts :: [{timeout, timeout()} | {persistent, boolean()}]. + +set_env(Config, Opts) when is_list(Config), is_list(Opts) -> + case application_controller:set_env(Config, Opts) of + ok -> ok; + {error, Msg} -> erlang:error({badarg, Msg}, [Config, Opts]) + end. + -spec set_env(Application, Par, Val) -> 'ok' when Application :: atom(), Par :: atom(), diff --git a/lib/kernel/src/application_controller.erl b/lib/kernel/src/application_controller.erl index a074d2e74b..7715dca7c6 100644 --- a/lib/kernel/src/application_controller.erl +++ b/lib/kernel/src/application_controller.erl @@ -26,7 +26,7 @@ control_application/1, change_application_data/2, prep_config_change/0, config_change/1, which_applications/0, which_applications/1, - loaded_applications/0, info/0, + loaded_applications/0, info/0, set_env/2, get_pid_env/2, get_env/2, get_pid_all_env/1, get_all_env/1, get_pid_key/2, get_key/2, get_pid_all_key/1, get_all_key/1, get_master/1, get_application/1, get_application_module/1, @@ -345,9 +345,6 @@ get_all_env(AppName) -> map(fun([Key, Val]) -> {Key, Val} end, ets:match(ac_tab, {{env, AppName, '$1'}, '$2'})). - - - get_pid_key(Master, Key) -> case ets:match(ac_tab, {{application_master, '$1'}, Master}) of [[AppName]] -> get_key(AppName, Key); @@ -461,6 +458,15 @@ permit_application(ApplName, Flag) -> {permit_application, ApplName, Flag}, infinity). +set_env(Config, Opts) -> + case check_conf_data(Config) of + ok -> + Timeout = proplists:get_value(timeout, Opts, 5000), + gen_server:call(?AC, {set_env, Config, Opts}, Timeout); + + {error, _} = Error -> + Error + end. set_env(AppName, Key, Val) -> gen_server:call(?AC, {set_env, AppName, Key, Val, []}). @@ -528,19 +534,15 @@ check_conf_data([]) -> check_conf_data(ConfData) when is_list(ConfData) -> [Application | ConfDataRem] = ConfData, case Application of - {kernel, List} when is_list(List) -> - case check_para_kernel(List) of - ok -> - check_conf_data(ConfDataRem); - Error1 -> - Error1 - end; {AppName, List} when is_atom(AppName), is_list(List) -> - case check_para(List, atom_to_list(AppName)) of - ok -> - check_conf_data(ConfDataRem); - Error2 -> - Error2 + case lists:keymember(AppName, 1, ConfDataRem) of + true -> + {error, "duplicate application config: " ++ atom_to_list(AppName)}; + false -> + case check_para(List, AppName) of + ok -> check_conf_data(ConfDataRem); + Error -> Error + end end; {AppName, List} when is_list(List) -> ErrMsg = "application: " @@ -553,36 +555,39 @@ check_conf_data(ConfData) when is_list(ConfData) -> ++ "; parameters must be a list", {error, ErrMsg}; Else -> - ErrMsg = "invalid application name: " ++ - lists:flatten(io_lib:format(" ~tp",[Else])), + ErrMsg = "invalid application config: " + ++ lists:flatten(io_lib:format("~tp",[Else])), {error, ErrMsg} end; check_conf_data(_ConfData) -> - {error, 'configuration must be a list ended by <dot><whitespace>'}. - + {error, "configuration must be a list ended by <dot><whitespace>"}. -%% Special check of distributed parameter for kernel -check_para_kernel([]) -> + +check_para([], _AppName) -> ok; -check_para_kernel([{distributed, Apps} | ParaList]) when is_list(Apps) -> - case check_distributed(Apps) of - {error, _ErrorMsg} = Error -> - Error; - _ -> - check_para_kernel(ParaList) +check_para([{Para, Val} | ParaList], AppName) when is_atom(Para) -> + case lists:keymember(Para, 1, ParaList) of + true -> + ErrMsg = "application: " ++ atom_to_list(AppName) + ++ "; duplicate parameter: " ++ atom_to_list(Para), + {error, ErrMsg}; + false -> + case check_para_value(Para, Val, AppName) of + ok -> check_para(ParaList, AppName); + {error, _} = Error -> Error + end end; -check_para_kernel([{distributed, _Apps} | _ParaList]) -> - {error, "application: kernel; erroneous parameter: distributed"}; -check_para_kernel([{Para, _Val} | ParaList]) when is_atom(Para) -> - check_para_kernel(ParaList); -check_para_kernel([{Para, _Val} | _ParaList]) -> - {error, "application: kernel; invalid parameter: " ++ +check_para([{Para, _Val} | _ParaList], AppName) -> + {error, "application: " ++ atom_to_list(AppName) ++ "; invalid parameter name: " ++ lists:flatten(io_lib:format("~tp",[Para]))}; -check_para_kernel(Else) -> - {error, "application: kernel; invalid parameter list: " ++ +check_para([Else | _ParaList], AppName) -> + {error, "application: " ++ atom_to_list(AppName) ++ "; invalid parameter: " ++ lists:flatten(io_lib:format("~tp",[Else]))}. - +check_para_value(distributed, Apps, kernel) -> check_distributed(Apps); +check_para_value(_Para, _Val, _AppName) -> ok. + +%% Special check of distributed parameter for kernel check_distributed([]) -> ok; check_distributed([{App, List} | Apps]) when is_atom(App), is_list(List) -> @@ -595,18 +600,6 @@ check_distributed(_Else) -> {error, "application: kernel; erroneous parameter: distributed"}. -check_para([], _AppName) -> - ok; -check_para([{Para, _Val} | ParaList], AppName) when is_atom(Para) -> - check_para(ParaList, AppName); -check_para([{Para, _Val} | _ParaList], AppName) -> - {error, "application: " ++ AppName ++ "; invalid parameter: " ++ - lists:flatten(io_lib:format("~tp",[Para]))}; -check_para([Else | _ParaList], AppName) -> - {error, "application: " ++ AppName ++ "; invalid parameter: " ++ - lists:flatten(io_lib:format("~tp",[Else]))}. - - -type calls() :: 'info' | 'prep_config_change' | 'which_applications' | {'config_change' | 'control_application' | 'load_application' | 'start_type' | 'stop_application' | @@ -863,6 +856,16 @@ handle_call(which_applications, _From, S) -> end, S#state.running), {reply, Reply, S}; +handle_call({set_env, Config, Opts}, _From, S) -> + _ = [add_env(AppName, Env) || {AppName, Env} <- Config], + + case proplists:get_value(persistent, Opts, false) of + true -> + {reply, ok, S#state{conf_data = merge_env(S#state.conf_data, Config)}}; + false -> + {reply, ok, S} + end; + handle_call({set_env, AppName, Key, Val, Opts}, _From, S) -> ets:insert(ac_tab, {{env, AppName, Key}, Val}), case proplists:get_value(persistent, Opts, false) of diff --git a/lib/kernel/src/code_server.erl b/lib/kernel/src/code_server.erl index 1b4a67ecb7..68e1205301 100644 --- a/lib/kernel/src/code_server.erl +++ b/lib/kernel/src/code_server.erl @@ -1434,19 +1434,25 @@ all_loaded(Db) -> -spec error_msg(io:format(), [term()]) -> 'ok'. error_msg(Format, Args) -> + %% This is equal to calling logger:error/3 which we don't want to + %% do from code_server. We don't want to call logger:timestamp() + %% either. logger ! {log,error,Format,Args, #{pid=>self(), gl=>group_leader(), - time=>erlang:system_time(microsecond), + time=>os:system_time(microsecond), error_logger=>#{tag=>error}}}, ok. -spec info_msg(io:format(), [term()]) -> 'ok'. info_msg(Format, Args) -> + %% This is equal to calling logger:info/3 which we don't want to + %% do from code_server. We don't want to call logger:timestamp() + %% either. logger ! {log,info,Format,Args, #{pid=>self(), gl=>group_leader(), - time=>erlang:system_time(microsecond), + time=>os:system_time(microsecond), error_logger=>#{tag=>info_msg}}}, ok. diff --git a/lib/kernel/src/dist_util.erl b/lib/kernel/src/dist_util.erl index ecc022b28d..09ed31f10c 100644 --- a/lib/kernel/src/dist_util.erl +++ b/lib/kernel/src/dist_util.erl @@ -116,6 +116,10 @@ dflag2str(?DFLAG_SEND_SENDER) -> "SEND_SENDER"; dflag2str(?DFLAG_BIG_SEQTRACE_LABELS) -> "BIG_SEQTRACE_LABELS"; +dflag2str(?DFLAG_EXIT_PAYLOAD) -> + "EXIT_PAYLOAD"; +dflag2str(?DFLAG_FRAGMENTS) -> + "FRAGMENTS"; dflag2str(_) -> "UNKNOWN". diff --git a/lib/kernel/src/erl_epmd.erl b/lib/kernel/src/erl_epmd.erl index b7e8868911..f31a1722ce 100644 --- a/lib/kernel/src/erl_epmd.erl +++ b/lib/kernel/src/erl_epmd.erl @@ -33,10 +33,10 @@ -define(erlang_daemon_port, 4369). -endif. -ifndef(epmd_dist_high). --define(epmd_dist_high, 4370). +-define(epmd_dist_high, 6). -endif. -ifndef(epmd_dist_low). --define(epmd_dist_low, 4370). +-define(epmd_dist_low, 5). -endif. %% External exports @@ -77,8 +77,8 @@ stop() -> %% -spec port_please(Name, Host) -> {ok, Port, Version} | noport when - Name :: string(), - Host :: inet:ip_address(), + Name :: atom() | string(), + Host :: atom() | string() | inet:ip_address(), Port :: non_neg_integer(), Version :: non_neg_integer(). @@ -86,8 +86,8 @@ port_please(Node, Host) -> port_please(Node, Host, infinity). -spec port_please(Name, Host, Timeout) -> {ok, Port, Version} | noport when - Name :: string(), - Host :: inet:ip_address(), + Name :: atom() | string(), + Host :: atom() | string() | inet:ip_address(), Timeout :: non_neg_integer() | infinity, Port :: non_neg_integer(), Version :: non_neg_integer(). @@ -342,6 +342,13 @@ wait_for_reg_reply(Socket, SoFar) -> receive {tcp, Socket, Data0} -> case SoFar ++ Data0 of + [$v, Result, A, B, C, D] -> + case Result of + 0 -> + {alive, Socket, ?u32(A, B, C, D)}; + _ -> + {error, duplicate_name} + end; [$y, Result, A, B] -> case Result of 0 -> diff --git a/lib/kernel/src/erts_debug.erl b/lib/kernel/src/erts_debug.erl index 1270de4144..c4d276f9e8 100644 --- a/lib/kernel/src/erts_debug.erl +++ b/lib/kernel/src/erts_debug.erl @@ -36,7 +36,8 @@ map_info/1, same/2, set_internal_state/2, size_shared/1, copy_shared/1, dirty_cpu/2, dirty_io/2, dirty/3, lcnt_control/1, lcnt_control/2, lcnt_collect/0, lcnt_clear/0, - lc_graph/0, lc_graph_to_dot/2, lc_graph_merge/2]). + lc_graph/0, lc_graph_to_dot/2, lc_graph_merge/2, + alloc_blocks_size/1]). -spec breakpoint(MFA, Flag) -> non_neg_integer() when MFA :: {Module :: module(), @@ -495,3 +496,58 @@ lcg_print_locks(Out, [LastLock]) -> lcg_print_locks(Out, [Lock | Rest]) -> io:format(Out, "~w,\n", [Lock]), lcg_print_locks(Out, Rest). + + +%% Returns the amount of memory allocated by the given allocator type. +-spec alloc_blocks_size(Type) -> non_neg_integer() | undefined when + Type :: atom(). + +alloc_blocks_size(Type) -> + Allocs = erlang:system_info(alloc_util_allocators), + Sizes = erlang:system_info({allocator_sizes, Allocs}), + alloc_blocks_size_1(Sizes, Type, 0). + +alloc_blocks_size_1([], _Type, 0) -> + undefined; +alloc_blocks_size_1([{_Type, false} | Rest], Type, Acc) -> + alloc_blocks_size_1(Rest, Type, Acc); +alloc_blocks_size_1([{Type, Instances} | Rest], Type, Acc0) -> + F = fun ({instance, _, L}, Acc) -> + MBCSPool = case lists:keyfind(mbcs_pool, 1, L) of + {_, Pool} -> Pool; + false -> [] + end, + {_,MBCS} = lists:keyfind(mbcs, 1, L), + {_,SBCS} = lists:keyfind(sbcs, 1, L), + Acc + + sum_block_sizes(MBCSPool) + + sum_block_sizes(MBCS) + + sum_block_sizes(SBCS) + end, + alloc_blocks_size_1(Rest, Type, lists:foldl(F, Acc0, Instances)); +alloc_blocks_size_1([{_Type, Instances} | Rest], Type, Acc0) -> + F = fun ({instance, _, L}, Acc) -> + Acc + sum_foreign_sizes(Type, L) + end, + alloc_blocks_size_1(Rest, Type, lists:foldl(F, Acc0, Instances)); +alloc_blocks_size_1([], _Type, Acc) -> + Acc. + +sum_foreign_sizes(Type, L) -> + case lists:keyfind(mbcs_pool, 1, L) of + {_,Pool} -> + {_,ForeignBlocks} = lists:keyfind(foreign_blocks, 1, Pool), + case lists:keyfind(Type, 1, ForeignBlocks) of + {_,TypeSizes} -> sum_block_sizes(TypeSizes); + false -> 0 + end; + _ -> + 0 + end. + +sum_block_sizes(Blocks) -> + lists:foldl( + fun({blocks_size, Sz,_,_}, Sz0) -> Sz0+Sz; + ({blocks_size, Sz}, Sz0) -> Sz0+Sz; + (_, Sz) -> Sz + end, 0, Blocks). diff --git a/lib/kernel/src/gen_tcp.erl b/lib/kernel/src/gen_tcp.erl index 7f7833ec23..5d4764f8ff 100644 --- a/lib/kernel/src/gen_tcp.erl +++ b/lib/kernel/src/gen_tcp.erl @@ -156,7 +156,7 @@ connect(Address, Port, Opts) -> Options :: [connect_option()], Timeout :: timeout(), Socket :: socket(), - Reason :: inet:posix(). + Reason :: timeout | inet:posix(). connect(Address, Port, Opts, Time) -> Timer = inet:start_timer(Time), @@ -220,7 +220,7 @@ listen(Port, Opts0) -> -spec accept(ListenSocket) -> {ok, Socket} | {error, Reason} when ListenSocket :: socket(), Socket :: socket(), - Reason :: closed | timeout | system_limit | inet:posix(). + Reason :: closed | system_limit | inet:posix(). accept(S) -> case inet_db:lookup_socket(S) of @@ -312,7 +312,7 @@ recv(S, Length) when is_port(S) -> Length :: non_neg_integer(), Timeout :: timeout(), Packet :: string() | binary() | HttpPacket, - Reason :: closed | inet:posix(), + Reason :: closed | timeout | inet:posix(), HttpPacket :: term(). recv(S, Length, Time) when is_port(S) -> diff --git a/lib/kernel/src/gen_udp.erl b/lib/kernel/src/gen_udp.erl index d6e8652e77..fad7b2f887 100644 --- a/lib/kernel/src/gen_udp.erl +++ b/lib/kernel/src/gen_udp.erl @@ -95,7 +95,7 @@ -spec open(Port) -> {ok, Socket} | {error, Reason} when Port :: inet:port_number(), Socket :: socket(), - Reason :: inet:posix(). + Reason :: system_limit | inet:posix(). open(Port) -> open(Port, []). @@ -112,7 +112,7 @@ open(Port) -> | {bind_to_device, binary()} | option(), Socket :: socket(), - Reason :: inet:posix(). + Reason :: system_limit | inet:posix(). open(Port, Opts0) -> {Mod, Opts} = inet:udp_module(Opts0), @@ -186,7 +186,7 @@ recv(S,Len) when is_port(S), is_integer(Len) -> Port :: inet:port_number(), AncData :: inet:ancillary_data(), Packet :: string() | binary(), - Reason :: not_owner | inet:posix(). + Reason :: not_owner | timeout | inet:posix(). recv(S,Len,Time) when is_port(S) -> case inet_db:lookup_socket(S) of diff --git a/lib/kernel/src/global.erl b/lib/kernel/src/global.erl index a38522eb5c..3875074d74 100644 --- a/lib/kernel/src/global.erl +++ b/lib/kernel/src/global.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2017. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -50,10 +50,9 @@ %% This is for backward compatibility only; the functionality is broken. -define(WARN_DUPLICATED_NAME, global_multi_name_action). -%% Undocumented Kernel variable. Set this to 0 (zero) to get the old -%% behaviour. +%% Undocumented Kernel variable. -define(N_CONNECT_RETRIES, global_connect_retries). --define(DEFAULT_N_CONNECT_RETRIES, 5). +-define(DEFAULT_N_CONNECT_RETRIES, 0). %%% In certain places in the server, calling io:format hangs everything, %%% so we'd better use erlang:display/1. @@ -125,16 +124,12 @@ %%% There are also ETS tables used for bookkeeping of locks and names %%% (the first position is the key): %%% -%%% global_locks (set): {ResourceId, LockRequesterId, [{Pid,RPid,ref()]} +%%% global_locks (set): {ResourceId, LockRequesterId, [{Pid,ref()]} %%% Pid is locking ResourceId, ref() is the monitor ref. -%%% RPid =/= Pid if there is an extra process calling erlang:monitor(). -%%% global_names (set): {Name, Pid, Method, RPid, ref()} +%%% global_names (set): {Name, Pid, Method, ref()} %%% Registered names. ref() is the monitor ref. -%%% RPid =/= Pid if there is an extra process calling erlang:monitor(). %%% global_names_ext (set): {Name, Pid, RegNode} %%% External registered names (C-nodes). -%%% (The RPid:s can be removed when/if erlang:monitor() returns before -%%% trying to connect to the other node.) %%% %%% Helper tables: %%% global_pid_names (bag): {Pid, Name} | {ref(), Name} @@ -310,7 +305,7 @@ re_register_name(Name, Pid, Method0) when is_pid(Pid) -> -spec registered_names() -> [Name] when Name :: term(). registered_names() -> - MS = ets:fun2ms(fun({Name,_Pid,_M,_RP,_R}) -> Name end), + MS = ets:fun2ms(fun({Name,_Pid,_M,_R}) -> Name end), ets:select(global_names, MS). %%----------------------------------------------------------------- @@ -1235,7 +1230,7 @@ ins_name_ext(Name, Pid, Method, RegNode, FromPidOrNode, ExtraInfo, S0) -> where(Name) -> case ets:lookup(global_names, Name) of - [{_Name, Pid, _Method, _RPid, _Ref}] -> + [{_Name, Pid, _Method, _Ref}] -> if node(Pid) == node() -> case is_process_alive(Pid) of true -> Pid; @@ -1272,10 +1267,10 @@ can_set_lock({ResourceId, LockRequesterId}) -> end. insert_lock({ResourceId, LockRequesterId}=Id, Pid, PidRefs, S) -> - {RPid, Ref} = do_monitor(Pid), + Ref = erlang:monitor(process, Pid), true = ets:insert(global_pid_ids, {Pid, ResourceId}), true = ets:insert(global_pid_ids, {Ref, ResourceId}), - Lock = {ResourceId, LockRequesterId, [{Pid,RPid,Ref} | PidRefs]}, + Lock = {ResourceId, LockRequesterId, [{Pid,Ref} | PidRefs]}, true = ets:insert(global_locks, Lock), trace_message(S, {ins_lock, node(Pid)}, [Id, Pid]). @@ -1293,10 +1288,9 @@ handle_del_lock({ResourceId, LockReqId}, Pid, S0) -> _ -> S0 end. -remove_lock(ResourceId, LockRequesterId, Pid, [{Pid,RPid,Ref}], Down, S0) -> +remove_lock(ResourceId, LockRequesterId, Pid, [{Pid,Ref}], Down, S0) -> ?trace({remove_lock_1, {id,ResourceId},{pid,Pid}}), true = erlang:demonitor(Ref, [flush]), - kill_monitor_proc(RPid, Pid), true = ets:delete(global_locks, ResourceId), true = ets:delete_object(global_pid_ids, {Pid, ResourceId}), true = ets:delete_object(global_pid_ids, {Ref, ResourceId}), @@ -1309,9 +1303,8 @@ remove_lock(ResourceId, LockRequesterId, Pid, [{Pid,RPid,Ref}], Down, S0) -> remove_lock(ResourceId, LockRequesterId, Pid, PidRefs0, _Down, S) -> ?trace({remove_lock_2, {id,ResourceId},{pid,Pid}}), PidRefs = case lists:keyfind(Pid, 1, PidRefs0) of - {Pid, RPid, Ref} -> + {Pid, Ref} -> true = erlang:demonitor(Ref, [flush]), - kill_monitor_proc(RPid, Pid), true = ets:delete_object(global_pid_ids, {Ref, ResourceId}), lists:keydelete(Pid, 1, PidRefs0); @@ -1324,11 +1317,6 @@ remove_lock(ResourceId, LockRequesterId, Pid, PidRefs0, _Down, S) -> trace_message(S, {rem_lock, node(Pid)}, [{ResourceId, LockRequesterId}, Pid]). -kill_monitor_proc(Pid, Pid) -> - ok; -kill_monitor_proc(RPid, _Pid) -> - exit(RPid, kill). - do_ops(Ops, ConnNode, Names_ext, ExtraInfo, S0) -> ?trace({do_ops, {ops,Ops}}), @@ -1394,8 +1382,8 @@ sync_other(Node, N) -> % exit(normal). insert_global_name(Name, Pid, Method, FromPidOrNode, ExtraInfo, S) -> - {RPid, Ref} = do_monitor(Pid), - true = ets:insert(global_names, {Name, Pid, Method, RPid, Ref}), + Ref = erlang:monitor(process, Pid), + true = ets:insert(global_names, {Name, Pid, Method, Ref}), true = ets:insert(global_pid_names, {Pid, Name}), true = ets:insert(global_pid_names, {Ref, Name}), case lock_still_set(FromPidOrNode, ExtraInfo, S) of @@ -1437,7 +1425,7 @@ extra_info(Tag, ExtraInfo) -> del_name(Ref, S) -> NameL = [Name || {_, Name} <- ets:lookup(global_pid_names, Ref), - {_, _Pid, _Method, _RPid, Ref1} <- + {_, _Pid, _Method, Ref1} <- ets:lookup(global_names, Name), Ref1 =:= Ref], case NameL of @@ -1450,24 +1438,23 @@ del_name(Ref, S) -> %% Keeps the entry in global_names for whereis_name/1. delete_global_name_keep_pid(Name, S) -> case ets:lookup(global_names, Name) of - [{Name, Pid, _Method, RPid, Ref}] -> - delete_global_name2(Name, Pid, RPid, Ref, S); + [{Name, Pid, _Method, Ref}] -> + delete_global_name2(Name, Pid, Ref, S); [] -> S end. delete_global_name2(Name, S) -> case ets:lookup(global_names, Name) of - [{Name, Pid, _Method, RPid, Ref}] -> + [{Name, Pid, _Method, Ref}] -> true = ets:delete(global_names, Name), - delete_global_name2(Name, Pid, RPid, Ref, S); + delete_global_name2(Name, Pid, Ref, S); [] -> S end. -delete_global_name2(Name, Pid, RPid, Ref, S) -> +delete_global_name2(Name, Pid, Ref, S) -> true = erlang:demonitor(Ref, [flush]), - kill_monitor_proc(RPid, Pid), delete_global_name(Name, Pid), ?trace({delete_global_name,{item,Name},{pid,Pid}}), true = ets:delete_object(global_pid_names, {Pid, Name}), @@ -1929,9 +1916,9 @@ reset_node_state(Node) -> %% from the same partition. exchange_names([{Name, Pid, Method} | Tail], Node, Ops, Res) -> case ets:lookup(global_names, Name) of - [{Name, Pid, _Method, _RPid2, _Ref2}] -> + [{Name, Pid, _Method, _Ref2}] -> exchange_names(Tail, Node, Ops, Res); - [{Name, Pid2, Method2, _RPid2, _Ref2}] when node() < Node -> + [{Name, Pid2, Method2, _Ref2}] when node() < Node -> %% Name clash! Add the result of resolving to Res(olved). %% We know that node(Pid) =/= node(), so we don't %% need to link/unlink to Pid. @@ -1960,7 +1947,7 @@ exchange_names([{Name, Pid, Method} | Tail], Node, Ops, Res) -> Op = {delete, Name}, exchange_names(Tail, Node, [Op | Ops], [Op | Res]) end; - [{Name, _Pid2, _Method, _RPid, _Ref}] -> + [{Name, _Pid2, _Method, _Ref}] -> %% The other node will solve the conflict. exchange_names(Tail, Node, Ops, Res); _ -> @@ -2036,7 +2023,7 @@ pid_is_locking(Pid, PidRefs) -> delete_lock(Ref, S0) -> Locks = pid_locks(Ref), F = fun({ResourceId, LockRequesterId, PidRefs}, S) -> - {Pid, _RPid, Ref} = lists:keyfind(Ref, 3, PidRefs), + {Pid, Ref} = lists:keyfind(Ref, 2, PidRefs), remove_lock(ResourceId, LockRequesterId, Pid, PidRefs, true, S) end, lists:foldl(F, S0, Locks). @@ -2046,10 +2033,10 @@ pid_locks(Ref) -> ets:lookup(global_locks, ResourceId) end, ets:lookup(global_pid_ids, Ref)), [Lock || Lock = {_Id, _Req, PidRefs} <- L, - rpid_is_locking(Ref, PidRefs)]. + ref_is_locking(Ref, PidRefs)]. -rpid_is_locking(Ref, PidRefs) -> - lists:keyfind(Ref, 3, PidRefs) =/= false. +ref_is_locking(Ref, PidRefs) -> + lists:keyfind(Ref, 2, PidRefs) =/= false. handle_nodedown(Node, S) -> %% DOWN signals from monitors have removed locks and registered names. @@ -2062,7 +2049,7 @@ handle_nodedown(Node, S) -> get_names() -> ets:select(global_names, - ets:fun2ms(fun({Name, Pid, Method, _RPid, _Ref}) -> + ets:fun2ms(fun({Name, Pid, Method, _Ref}) -> {Name, Pid, Method} end)). @@ -2205,24 +2192,6 @@ unexpected_message(Message, What) -> %%% Utilities -%% When/if erlang:monitor() returns before trying to connect to the -%% other node this function can be removed. -do_monitor(Pid) -> - case (node(Pid) =:= node()) orelse lists:member(node(Pid), nodes()) of - true -> - %% Assume the node is still up - {Pid, erlang:monitor(process, Pid)}; - false -> - F = fun() -> - Ref = erlang:monitor(process, Pid), - receive - {'DOWN', Ref, process, Pid, _Info} -> - exit(normal) - end - end, - erlang:spawn_monitor(F) - end. - intersection(_, []) -> []; intersection(L1, L2) -> diff --git a/lib/kernel/src/inet_config.erl b/lib/kernel/src/inet_config.erl index 9f76360b8b..e771461b65 100644 --- a/lib/kernel/src/inet_config.erl +++ b/lib/kernel/src/inet_config.erl @@ -98,7 +98,7 @@ init() -> {win32,WinType} -> win32_load_from_registry(WinType); _ -> - error("can not read win32 system registry~n", []) + error("cannot read win32 system registry~n", []) end end, CfgFiles), diff --git a/lib/kernel/src/inet_db.erl b/lib/kernel/src/inet_db.erl index 6cbb6ac2da..3f5a2ea5ee 100644 --- a/lib/kernel/src/inet_db.erl +++ b/lib/kernel/src/inet_db.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2016. All Rights Reserved. +%% Copyright Ericsson AB 1997-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1223,7 +1223,10 @@ handle_set_file(Option, Fname, TagTm, TagInfo, ParseFun, From, {ok, B, _} -> B; _ -> <<>> end; - _ -> <<>> + _ -> + ets:insert(Db, {TagInfo, undefined}), + TimeZero = - (?RES_FILE_UPDATE_TM + 1), % Early enough + ets:insert(Db, {TagTm, TimeZero}) end, handle_set_file(ParseFun, Bin, From, State); false -> {reply,error,State} diff --git a/lib/kernel/src/inet_dns.erl b/lib/kernel/src/inet_dns.erl index f1f58bc872..6c98d2aab7 100644 --- a/lib/kernel/src/inet_dns.erl +++ b/lib/kernel/src/inet_dns.erl @@ -699,7 +699,7 @@ encode_labels(Bin, Comp0, Pos, [L|Ls]=Labels) none -> Comp = if Pos < (3 bsl 14) -> %% Just in case - compression - %% pointers can not reach further + %% pointers cannot reach further gb_trees:insert(Labels, Pos, Comp0); true -> Comp0 end, diff --git a/lib/kernel/src/inet_tcp_dist.erl b/lib/kernel/src/inet_tcp_dist.erl index c37212b0f9..c5a114a9ef 100644 --- a/lib/kernel/src/inet_tcp_dist.erl +++ b/lib/kernel/src/inet_tcp_dist.erl @@ -212,6 +212,7 @@ do_accept(Driver, Kernel, AcceptPid, Socket, MyNode, Allowed, SetupTime) -> [{active, true}, {deliver, port}, {packet, 4}, + binary, nodelay()]) end, f_getll = fun(S) -> diff --git a/lib/kernel/src/kernel.app.src b/lib/kernel/src/kernel.app.src index fe073621c8..8fe6bdd1ca 100644 --- a/lib/kernel/src/kernel.app.src +++ b/lib/kernel/src/kernel.app.src @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -68,11 +68,12 @@ logger_formatter, logger_h_common, logger_handler_watcher, + logger_olp, + logger_proxy, logger_server, logger_simple_h, logger_std_h, logger_sup, - net, net_adm, net_kernel, os, @@ -146,6 +147,6 @@ {logger_sasl_compatible, false} ]}, {mod, {kernel, []}}, - {runtime_dependencies, ["erts-10.1", "stdlib-3.5", "sasl-3.0"]} + {runtime_dependencies, ["erts-10.2.5", "stdlib-3.5", "sasl-3.0"]} ] }. diff --git a/lib/kernel/src/kernel.appup.src b/lib/kernel/src/kernel.appup.src index ccf0a82ced..aca3247c8f 100644 --- a/lib/kernel/src/kernel.appup.src +++ b/lib/kernel/src/kernel.appup.src @@ -40,7 +40,12 @@ {<<"^6\\.0\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^6\\.1$">>,[restart_new_emulator]}, {<<"^6\\.1\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], + {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^6\\.2$">>,[restart_new_emulator]}, + {<<"^6\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^6\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^6\\.3$">>,[restart_new_emulator]}, + {<<"^6\\.3\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}], [{<<"^5\\.3$">>,[restart_new_emulator]}, {<<"^5\\.3\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^5\\.3\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, @@ -54,4 +59,9 @@ {<<"^6\\.0\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^6\\.1$">>,[restart_new_emulator]}, {<<"^6\\.1\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. + {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^6\\.2$">>,[restart_new_emulator]}, + {<<"^6\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^6\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^6\\.3$">>,[restart_new_emulator]}, + {<<"^6\\.3\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}]}. diff --git a/lib/kernel/src/kernel.erl b/lib/kernel/src/kernel.erl index c68d04e279..111d103df2 100644 --- a/lib/kernel/src/kernel.erl +++ b/lib/kernel/src/kernel.erl @@ -145,26 +145,11 @@ init([]) -> {ok, {SupFlags, [Code, File, StdError, User, LoggerSup, Config, RefC, SafeSup]}}; _ -> - Rpc = #{id => rex, - start => {rpc, start_link, []}, - restart => permanent, - shutdown => 2000, - type => worker, - modules => [rpc]}, - - Global = #{id => global_name_server, - start => {global, start_link, []}, - restart => permanent, - shutdown => 2000, - type => worker, - modules => [global]}, - - GlGroup = #{id => global_group, - start => {global_group,start_link,[]}, - restart => permanent, - shutdown => 2000, - type => worker, - modules => [global_group]}, + DistChildren = + case application:get_env(kernel, start_distribution) of + {ok, false} -> []; + _ -> start_distribution() + end, InetDb = #{id => inet_db, start => {inet_db, start_link, []}, @@ -173,13 +158,6 @@ init([]) -> type => worker, modules => [inet_db]}, - NetSup = #{id => net_sup, - start => {erl_distribution, start_link, []}, - restart => permanent, - shutdown => infinity, - type => supervisor, - modules => [erl_distribution]}, - SigSrv = #{id => erl_signal_server, start => {gen_event, start_link, [{local, erl_signal_server}]}, restart => permanent, @@ -187,14 +165,11 @@ init([]) -> type => worker, modules => dynamic}, - DistAC = start_dist_ac(), - Timer = start_timer(), {ok, {SupFlags, - [Code, Rpc, Global, InetDb | DistAC] ++ - [NetSup, GlGroup, File, SigSrv, - StdError, User, Config, RefC, SafeSup, LoggerSup] ++ Timer}} + [Code, InetDb | DistChildren] ++ + [File, SigSrv, StdError, User, Config, RefC, SafeSup, LoggerSup] ++ Timer}} end; init(safe) -> SupFlags = #{strategy => one_for_one, @@ -213,6 +188,39 @@ init(safe) -> {ok, {SupFlags, Boot ++ DiskLog ++ Pg2}}. +start_distribution() -> + Rpc = #{id => rex, + start => {rpc, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [rpc]}, + + Global = #{id => global_name_server, + start => {global, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [global]}, + + DistAC = start_dist_ac(), + + NetSup = #{id => net_sup, + start => {erl_distribution, start_link, []}, + restart => permanent, + shutdown => infinity, + type => supervisor, + modules => [erl_distribution]}, + + GlGroup = #{id => global_group, + start => {global_group,start_link,[]}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [global_group]}, + + [Rpc, Global | DistAC] ++ [NetSup, GlGroup]. + start_dist_ac() -> Spec = [#{id => dist_ac, start => {dist_ac,start_link,[]}, diff --git a/lib/kernel/src/logger.erl b/lib/kernel/src/logger.erl index 6762998d4f..38bd2f481c 100644 --- a/lib/kernel/src/logger.erl +++ b/lib/kernel/src/logger.erl @@ -43,11 +43,14 @@ get_module_level/0, get_module_level/1, set_primary_config/1, set_primary_config/2, set_handler_config/2, set_handler_config/3, + set_proxy_config/1, update_primary_config/1, update_handler_config/2, update_handler_config/3, + update_proxy_config/1, update_formatter_config/2, update_formatter_config/3, get_primary_config/0, get_handler_config/1, get_handler_config/0, get_handler_ids/0, get_config/0, + get_proxy_config/0, add_handlers/1]). %% Private configuration @@ -57,6 +60,8 @@ -export([compare_levels/2]). -export([set_process_metadata/1, update_process_metadata/1, unset_process_metadata/0, get_process_metadata/0]). +-export([i/0, i/1]). +-export([timestamp/0]). %% Basic report formatting -export([format_report/1, format_otp_report/1]). @@ -122,6 +127,18 @@ {filters,log | stop,[{filter_id(),filter()}]} | {module_level,level(),[module()]}]. +-type olp_config() :: #{sync_mode_qlen => non_neg_integer(), + drop_mode_qlen => pos_integer(), + flush_qlen => pos_integer(), + burst_limit_enable => boolean(), + burst_limit_max_count => pos_integer(), + burst_limit_window_time => pos_integer(), + overload_kill_enable => boolean(), + overload_kill_qlen => pos_integer(), + overload_kill_mem_size => pos_integer(), + overload_kill_restart_after => + non_neg_integer() | infinity}. + -export_type([log_event/0, level/0, report/0, @@ -137,7 +154,9 @@ filter_arg/0, filter_return/0, config_handler/0, - formatter_config/0]). + formatter_config/0, + olp_config/0, + timestamp/0]). %%%----------------------------------------------------------------- %%% API @@ -337,6 +356,10 @@ internal_log(Level,Term) when is_atom(Level) -> erlang:display_string("Logger - "++ atom_to_list(Level) ++ ": "), erlang:display(Term). +-spec timestamp() -> timestamp(). +timestamp() -> + os:system_time(microsecond). + %%%----------------------------------------------------------------- %%% Configuration -spec add_primary_filter(FilterId,Filter) -> ok | {error,term()} when @@ -390,6 +413,7 @@ set_primary_config(Key,Value) -> set_primary_config(Config) -> logger_server:set_config(primary,Config). + -spec set_handler_config(HandlerId,level,Level) -> Return when HandlerId :: handler_id(), Level :: level() | all | none, @@ -419,6 +443,11 @@ set_handler_config(HandlerId,Key,Value) -> set_handler_config(HandlerId,Config) -> logger_server:set_config(HandlerId,Config). +-spec set_proxy_config(Config) -> ok | {error,term()} when + Config :: olp_config(). +set_proxy_config(Config) -> + logger_server:set_config(proxy,Config). + -spec update_primary_config(Config) -> ok | {error,term()} when Config :: primary_config(). update_primary_config(Config) -> @@ -453,6 +482,11 @@ update_handler_config(HandlerId,Key,Value) -> update_handler_config(HandlerId,Config) -> logger_server:update_config(HandlerId,Config). +-spec update_proxy_config(Config) -> ok | {error,term()} when + Config :: olp_config(). +update_proxy_config(Config) -> + logger_server:update_config(proxy,Config). + -spec get_primary_config() -> Config when Config :: primary_config(). get_primary_config() -> @@ -486,6 +520,12 @@ get_handler_ids() -> {ok,#{handlers:=HandlerIds}} = logger_config:get(?LOGGER_TABLE,primary), HandlerIds. +-spec get_proxy_config() -> Config when + Config :: olp_config(). +get_proxy_config() -> + {ok,Config} = logger_config:get(?LOGGER_TABLE,proxy), + Config. + -spec update_formatter_config(HandlerId,FormatterConfig) -> ok | {error,term()} when HandlerId :: handler_id(), @@ -606,12 +646,150 @@ unset_process_metadata() -> -spec get_config() -> #{primary=>primary_config(), handlers=>[handler_config()], + proxy=>olp_config(), module_levels=>[{module(),level() | all | none}]}. get_config() -> #{primary=>get_primary_config(), handlers=>get_handler_config(), + proxy=>get_proxy_config(), module_levels=>lists:keysort(1,get_module_level())}. +-spec i() -> ok. +i() -> + #{primary := Primary, + handlers := HandlerConfigs, + proxy := Proxy, + module_levels := Modules} = get_config(), + M = modifier(), + i_primary(Primary,M), + i_handlers(HandlerConfigs,M), + i_proxy(Proxy,M), + i_modules(Modules,M). + +-spec i(What) -> ok when + What :: primary | handlers | proxy | modules | handler_id(). +i(primary) -> + i_primary(get_primary_config(),modifier()); +i(handlers) -> + i_handlers(get_handler_config(),modifier()); +i(proxy) -> + i_proxy(get_proxy_config(),modifier()); +i(modules) -> + i_modules(get_module_level(),modifier()); +i(HandlerId) when is_atom(HandlerId) -> + case get_handler_config(HandlerId) of + {ok,HandlerConfig} -> + i_handlers([HandlerConfig],modifier()); + Error -> + Error + end; +i(What) -> + erlang:error(badarg,[What]). + + +i_primary(#{level := Level, + filters := Filters, + filter_default := FilterDefault}, + M) -> + io:format("Primary configuration: ~n",[]), + io:format(" Level: ~p~n",[Level]), + io:format(" Filter Default: ~p~n", [FilterDefault]), + io:format(" Filters: ~n", []), + print_filters(" ",Filters,M). + +i_handlers(HandlerConfigs,M) -> + io:format("Handler configuration: ~n", []), + print_handlers(HandlerConfigs,M). + +i_proxy(Proxy,M) -> + io:format("Proxy configuration: ~n", []), + print_custom(" ",Proxy,M). + +i_modules(Modules,M) -> + io:format("Level set per module: ~n", []), + print_module_levels(Modules,M). + +encoding() -> + case lists:keyfind(encoding, 1, io:getopts()) of + false -> latin1; + {encoding, Enc} -> Enc + end. + +modifier() -> + modifier(encoding()). + +modifier(latin1) -> ""; +modifier(_) -> "t". + +print_filters(Indent, {Id, {Fun, Arg}}, M) -> + io:format("~sId: ~"++M++"p~n" + "~s Fun: ~"++M++"p~n" + "~s Arg: ~"++M++"p~n", + [Indent, Id, Indent, Fun, Indent, Arg]); +print_filters(Indent,[],_M) -> + io:format("~s(none)~n",[Indent]); +print_filters(Indent,Filters,M) -> + [print_filters(Indent,Filter,M) || Filter <- Filters], + ok. + +print_handlers(#{id := Id, + module := Module, + level := Level, + filters := Filters, filter_default := FilterDefault, + formatter := {FormatterModule,FormatterConfig}} = Config, M) -> + io:format(" Id: ~"++M++"p~n" + " Module: ~p~n" + " Level: ~p~n" + " Formatter:~n" + " Module: ~p~n" + " Config:~n", + [Id, Module, Level, FormatterModule]), + print_custom(" ",FormatterConfig,M), + io:format(" Filter Default: ~p~n" + " Filters:~n", + [FilterDefault]), + print_filters(" ",Filters,M), + case maps:find(config,Config) of + {ok,HandlerConfig} -> + io:format(" Handler Config:~n"), + print_custom(" ",HandlerConfig,M); + error -> + ok + end, + MyKeys = [filter_default, filters, formatter, level, module, id, config], + case maps:without(MyKeys,Config) of + Empty when Empty==#{} -> + ok; + Unhandled -> + io:format(" Custom Config:~n"), + print_custom(" ",Unhandled,M) + end; +print_handlers([], _M) -> + io:format(" (none)~n"); +print_handlers(HandlerConfigs, M) -> + [print_handlers(HandlerConfig, M) || HandlerConfig <- HandlerConfigs], + ok. + +print_custom(Indent, {Key, Value}, M) -> + io:format("~s~"++M++"p: ~"++M++"p~n",[Indent,Key,Value]); +print_custom(Indent, Map, M) when is_map(Map) -> + print_custom(Indent,lists:keysort(1,maps:to_list(Map)), M); +print_custom(Indent, List, M) when is_list(List), is_tuple(hd(List)) -> + [print_custom(Indent, X, M) || X <- List], + ok; +print_custom(Indent, Value, M) -> + io:format("~s~"++M++"p~n",[Indent,Value]). + +print_module_levels({Module,Level},M) -> + io:format(" Module: ~"++M++"p~n" + " Level: ~p~n", + [Module,Level]); +print_module_levels([],_M) -> + io:format(" (none)~n"); +print_module_levels(Modules,M) -> + [print_module_levels(Module,M) || Module <- Modules], + ok. + -spec internal_init_logger() -> ok | {error,term()}. %% This function is responsible for config of the logger %% This is done before add_handlers because we want the @@ -672,6 +850,17 @@ init_kernel_handlers(Env) -> %% This function is responsible for resolving the handler config %% and then starting the correct handlers. This is done after the %% kernel supervisor tree has been started as it needs the logger_sup. +add_handlers(kernel) -> + Env = get_logger_env(kernel), + case get_proxy_opts(Env) of + undefined -> + add_handlers(kernel,Env); + Opts -> + case set_proxy_config(Opts) of + ok -> add_handlers(kernel,Env); + {error, Reason} -> {error,{bad_proxy_config,Reason}} + end + end; add_handlers(App) when is_atom(App) -> add_handlers(App,get_logger_env(App)); add_handlers(HandlerConfig) -> @@ -729,6 +918,8 @@ check_logger_config(kernel,[{filters,_,_}|Env]) -> check_logger_config(kernel,Env); check_logger_config(kernel,[{module_level,_,_}|Env]) -> check_logger_config(kernel,Env); +check_logger_config(kernel,[{proxy,_}|Env]) -> + check_logger_config(kernel,Env); check_logger_config(_,Bad) -> throw(Bad). @@ -784,6 +975,13 @@ get_primary_filters(Env) -> _ -> throw({multiple_filters,Env}) end. +get_proxy_opts(Env) -> + case [P || P={proxy,_} <- Env] of + [{proxy,Opts}] -> Opts; + [] -> undefined; + _ -> throw({multiple_proxies,Env}) + end. + %% This function looks at the kernel logger environment %% and updates it so that the correct logger is configured init_default_config(Type,Env) when Type==standard_io; @@ -880,30 +1078,30 @@ log_allowed(Location,Level,Msg,Meta0) when is_map(Meta0) -> maps:merge(Location,maps:merge(proc_meta(),Meta0))), case node(maps:get(gl,Meta)) of Node when Node=/=node() -> - log_remote(Node,Level,Msg,Meta), - do_log_allowed(Level,Msg,Meta); + log_remote(Node,Level,Msg,Meta); _ -> - do_log_allowed(Level,Msg,Meta) - end. + ok + end, + do_log_allowed(Level,Msg,Meta,tid()). -do_log_allowed(Level,{Format,Args}=Msg,Meta) +do_log_allowed(Level,{Format,Args}=Msg,Meta,Tid) when ?IS_LEVEL(Level), is_list(Format), is_list(Args), is_map(Meta) -> - logger_backend:log_allowed(#{level=>Level,msg=>Msg,meta=>Meta},tid()); -do_log_allowed(Level,Report,Meta) + logger_backend:log_allowed(#{level=>Level,msg=>Msg,meta=>Meta},Tid); +do_log_allowed(Level,Report,Meta,Tid) when ?IS_LEVEL(Level), ?IS_REPORT(Report), is_map(Meta) -> logger_backend:log_allowed(#{level=>Level,msg=>{report,Report},meta=>Meta}, - tid()); -do_log_allowed(Level,String,Meta) + Tid); +do_log_allowed(Level,String,Meta,Tid) when ?IS_LEVEL(Level), ?IS_STRING(String), is_map(Meta) -> logger_backend:log_allowed(#{level=>Level,msg=>{string,String},meta=>Meta}, - tid()). + Tid). tid() -> ets:whereis(?LOGGER_TABLE). @@ -913,7 +1111,7 @@ log_remote(Node,Level,Msg,Meta) -> log_remote(Node,{log,Level,Msg,Meta}). log_remote(Node,Request) -> - {logger,Node} ! Request, + logger_proxy:log({remote,Node,Request}), ok. add_default_metadata(Meta) -> @@ -937,7 +1135,7 @@ proc_meta() -> default(pid) -> self(); default(gl) -> group_leader(); -default(time) -> erlang:system_time(microsecond). +default(time) -> timestamp(). %% Remove everything upto and including this module from the stacktrace filter_stacktrace(Module,[{Module,_,_,_}|_]) -> diff --git a/lib/kernel/src/logger_config.erl b/lib/kernel/src/logger_config.erl index 5e9faf332c..5024d20cfe 100644 --- a/lib/kernel/src/logger_config.erl +++ b/lib/kernel/src/logger_config.erl @@ -66,6 +66,8 @@ get(Tid,What) -> case ets:lookup(Tid,table_key(What)) of [{_,_,Config}] -> {ok,Config}; + [{_,Config}] when What=:=proxy -> + {ok,Config}; [] -> {error,{not_found,What}} end. @@ -79,10 +81,15 @@ get(Tid,What,Level) -> [Data] -> {ok,Data} end. +create(Tid,proxy,Config) -> + ets:insert(Tid,{table_key(proxy),Config}); create(Tid,What,Config) -> LevelInt = level_to_int(maps:get(level,Config)), ets:insert(Tid,{table_key(What),LevelInt,Config}). +set(Tid,proxy,Config) -> + ets:insert(Tid,{table_key(proxy),Config}), + ok; set(Tid,What,Config) -> LevelInt = level_to_int(maps:get(level,Config)), %% Should do this only if the level has actually changed. Possibly @@ -148,5 +155,6 @@ int_to_level(?LOG_ALL) -> all. %%%----------------------------------------------------------------- %%% Internal +table_key(proxy) -> ?PROXY_KEY; table_key(primary) -> ?PRIMARY_KEY; table_key(HandlerId) -> {?HANDLER_KEY,HandlerId}. diff --git a/lib/kernel/src/logger_disk_log_h.erl b/lib/kernel/src/logger_disk_log_h.erl index 41e0d51a9d..47b39da900 100644 --- a/lib/kernel/src/logger_disk_log_h.erl +++ b/lib/kernel/src/logger_disk_log_h.erl @@ -24,7 +24,7 @@ -include("logger_h_common.hrl"). %%% API --export([info/1, filesync/1, reset/1]). +-export([filesync/1]). %% logger_h_common callbacks -export([init/2, check_config/4, reset_state/2, @@ -47,25 +47,6 @@ filesync(Name) -> logger_h_common:filesync(?MODULE,Name). -%%%----------------------------------------------------------------- -%%% --spec info(Name) -> Info | {error,Reason} when - Name :: atom(), - Info :: term(), - Reason :: handler_busy | {badarg,term()}. - -info(Name) -> - logger_h_common:info(?MODULE,Name). - -%%%----------------------------------------------------------------- -%%% --spec reset(Name) -> ok | {error,Reason} when - Name :: atom(), - Reason :: handler_busy | {badarg,term()}. - -reset(Name) -> - logger_h_common:reset(?MODULE,Name). - %%%=================================================================== %%% logger callbacks %%%=================================================================== diff --git a/lib/kernel/src/logger_formatter.erl b/lib/kernel/src/logger_formatter.erl index ded89bac9f..8696adbd72 100644 --- a/lib/kernel/src/logger_formatter.erl +++ b/lib/kernel/src/logger_formatter.erl @@ -64,7 +64,7 @@ format(#{level:=Level,msg:=Msg0,meta:=Meta},Config0) Config; Size0 -> Size = - case Size0 - string:length([B,A]) of + case Size0 - io_lib:chars_length([B,A]) of S when S>=0 -> S; _ -> 0 end, @@ -75,7 +75,11 @@ format(#{level:=Level,msg:=Msg0,meta:=Meta},Config0) true -> %% Trim leading and trailing whitespaces, and replace %% newlines with ", " - re:replace(string:trim(MsgStr0),",?\r?\n\s*",", ", + T = lists:reverse( + trim( + lists:reverse( + trim(MsgStr0,false)),true)), + re:replace(T,",?\r?\n\s*",", ", [{return,list},global,unicode]); _false -> MsgStr0 @@ -83,7 +87,26 @@ format(#{level:=Level,msg:=Msg0,meta:=Meta},Config0) true -> "" end, - truncate([B,MsgStr,A],maps:get(max_size,Config)). + truncate(B,MsgStr,A,maps:get(max_size,Config)). + +trim([H|T],Rev) when H==$\s; H==$\r; H==$\n -> + trim(T,Rev); +trim([H|T],false) when is_list(H) -> + case trim(H,false) of + [] -> + trim(T,false); + TrimmedH -> + [TrimmedH|T] + end; +trim([H|T],true) when is_list(H) -> + case trim(lists:reverse(H),true) of + [] -> + trim(T,true); + TrimmedH -> + [lists:reverse(TrimmedH)|T] + end; +trim(String,_) -> + String. do_format(Level,Data,[level|Format],Config) -> [to_string(level,Level,Config)|do_format(Level,Data,Format,Config)]; @@ -239,21 +262,47 @@ chardata_to_list(Chardata) -> throw(Error) end. -truncate(String,unlimited) -> - String; -truncate(String,Size) -> - Length = string:length(String), +truncate(B,Msg,A,unlimited) -> + [B,Msg,A]; +truncate(B,Msg,A,Size) -> + String = [B,Msg,A], + Length = io_lib:chars_length(String), if Length>Size -> - case lists:reverse(lists:flatten(String)) of - [$\n|_] -> - string:slice(String,0,Size-4)++"...\n"; + {Last,FlatString} = + case A of + [] -> + case Msg of + [] -> + {get_last(B),lists:flatten(B)}; + _ -> + {get_last(Msg),lists:flatten([B,Msg])} + end; + _ -> + {get_last(A),lists:flatten(String)} + end, + case Last of + $\n-> + lists:sublist(FlatString,1,Size-4)++"...\n"; _ -> - string:slice(String,0,Size-3)++"..." + lists:sublist(FlatString,1,Size-3)++"..." end; true -> String end. +get_last(L) -> + get_first(lists:reverse(L)). + +get_first([]) -> + error; +get_first([C|_]) when is_integer(C) -> + C; +get_first([L|Rest]) when is_list(L) -> + case get_last(L) of + error -> get_first(Rest); + First -> First + end. + %% SysTime is the system time in microseconds format_time(SysTime,#{time_offset:=Offset,time_designator:=Des}) when is_integer(SysTime) -> diff --git a/lib/kernel/src/logger_h_common.erl b/lib/kernel/src/logger_h_common.erl index 74a2d158fc..16946ff97c 100644 --- a/lib/kernel/src/logger_h_common.erl +++ b/lib/kernel/src/logger_h_common.erl @@ -24,11 +24,11 @@ -include("logger_internal.hrl"). %% API --export([start_link/1, info/2, filesync/2, reset/2]). +-export([filesync/2]). -%% gen_server and proc_lib callbacks --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). +%% logger_olp callbacks +-export([init/1, handle_load/2, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3, notify/2, reset_state/1]). %% logger callbacks -export([log/2, adding_handler/1, removing_handler/1, changing_config/3, @@ -37,52 +37,45 @@ %% Library functions for handlers -export([error_notify/1]). -%%%----------------------------------------------------------------- --define(CONFIG_KEYS,[sync_mode_qlen, - drop_mode_qlen, - flush_qlen, - burst_limit_enable, - burst_limit_max_count, - burst_limit_window_time, - overload_kill_enable, - overload_kill_qlen, - overload_kill_mem_size, - overload_kill_restart_after, - filesync_repeat_interval]). --define(READ_ONLY_KEYS,[handler_pid,mode_tab]). +-define(OLP_KEYS,[sync_mode_qlen, + drop_mode_qlen, + flush_qlen, + burst_limit_enable, + burst_limit_max_count, + burst_limit_window_time, + overload_kill_enable, + overload_kill_qlen, + overload_kill_mem_size, + overload_kill_restart_after]). + +-define(COMMON_KEYS,[filesync_repeat_interval]). + +-define(READ_ONLY_KEYS,[olp]). %%%----------------------------------------------------------------- %%% API %% This function is called by the logger_sup supervisor -start_link(Args) -> - proc_lib:start_link(?MODULE,init,[Args]). - filesync(Module, Name) -> call(Module, Name, filesync). -info(Module, Name) -> - call(Module, Name, info). - -reset(Module, Name) -> - call(Module, Name, reset). - %%%----------------------------------------------------------------- %%% Handler being added adding_handler(#{id:=Name,module:=Module}=Config) -> HConfig0 = maps:get(config, Config, #{}), - HandlerConfig0 = maps:without(?CONFIG_KEYS,HConfig0), + HandlerConfig0 = maps:without(?OLP_KEYS++?COMMON_KEYS,HConfig0), case Module:check_config(Name,set,undefined,HandlerConfig0) of {ok,HandlerConfig} -> - ModifiedCommon = maps:with(?CONFIG_KEYS,HandlerConfig), - CommonConfig0 = maps:with(?CONFIG_KEYS,HConfig0), + ModifiedCommon = maps:with(?COMMON_KEYS,HandlerConfig), + CommonConfig0 = maps:with(?COMMON_KEYS,HConfig0), CommonConfig = maps:merge( maps:merge(get_default_config(), CommonConfig0), ModifiedCommon), case check_config(CommonConfig) of ok -> HConfig = maps:merge(CommonConfig,HandlerConfig), - start(Config#{config => HConfig}); + OlpOpts = maps:with(?OLP_KEYS,HConfig0), + start(OlpOpts, Config#{config => HConfig}); {error,Faulty} -> {error,{invalid_config,Module,Faulty}} end; @@ -92,11 +85,11 @@ adding_handler(#{id:=Name,module:=Module}=Config) -> %%%----------------------------------------------------------------- %%% Handler being removed -removing_handler(#{id:=Name, module:=Module}) -> +removing_handler(#{id:=Name, module:=Module, config:=#{olp:=Olp}}) -> case whereis(?name_to_reg_name(Module,Name)) of undefined -> ok; - Pid -> + _Pid -> %% We don't want to do supervisor:terminate_child here %% since we need to distinguish this explicit stop from a %% system termination in order to avoid circular attempts @@ -106,7 +99,7 @@ removing_handler(#{id:=Name, module:=Module}) -> %% the restart type is temporary, which means that the %% child specification is automatically removed from the %% supervisor when the process dies. - _ = gen_server:call(Pid, stop), + _ = logger_olp:stop(Olp), ok end. @@ -116,34 +109,53 @@ changing_config(SetOrUpdate, #{id:=Name,config:=OldHConfig,module:=Module}, NewConfig0) -> NewHConfig0 = maps:get(config, NewConfig0, #{}), - OldHandlerConfig = maps:without(?CONFIG_KEYS++?READ_ONLY_KEYS,OldHConfig), - NewHandlerConfig0 = maps:without(?CONFIG_KEYS++?READ_ONLY_KEYS,NewHConfig0), + NoHandlerKeys = ?OLP_KEYS++?COMMON_KEYS++?READ_ONLY_KEYS, + OldHandlerConfig = maps:without(NoHandlerKeys,OldHConfig), + NewHandlerConfig0 = maps:without(NoHandlerKeys,NewHConfig0), case Module:check_config(Name, SetOrUpdate, OldHandlerConfig,NewHandlerConfig0) of {ok, NewHandlerConfig} -> - ModifiedCommon = maps:with(?CONFIG_KEYS,NewHandlerConfig), - NewCommonConfig0 = maps:with(?CONFIG_KEYS,NewHConfig0), + ModifiedCommon = maps:with(?COMMON_KEYS,NewHandlerConfig), + NewCommonConfig0 = maps:with(?COMMON_KEYS,NewHConfig0), + OldCommonConfig = maps:with(?COMMON_KEYS,OldHConfig), CommonDefault = case SetOrUpdate of set -> get_default_config(); update -> - maps:with(?CONFIG_KEYS,OldHConfig) + OldCommonConfig end, NewCommonConfig = maps:merge( maps:merge(CommonDefault,NewCommonConfig0), ModifiedCommon), case check_config(NewCommonConfig) of ok -> - ReadOnly = maps:with(?READ_ONLY_KEYS,OldHConfig), - NewHConfig = maps:merge( - maps:merge(NewCommonConfig,NewHandlerConfig), - ReadOnly), - NewConfig = NewConfig0#{config=>NewHConfig}, - HPid = maps:get(handler_pid,OldHConfig), - case call(HPid, {change_config,NewConfig}) of - ok -> {ok,NewConfig}; - Error -> Error + OlpDefault = + case SetOrUpdate of + set -> + logger_olp:get_default_opts(); + update -> + maps:with(?OLP_KEYS,OldHConfig) + end, + Olp = maps:get(olp,OldHConfig), + NewOlpOpts = maps:merge(OlpDefault, + maps:with(?OLP_KEYS,NewHConfig0)), + case logger_olp:set_opts(Olp,NewOlpOpts) of + ok -> + logger_olp:cast(Olp, {config_changed, + NewCommonConfig, + NewHandlerConfig}), + ReadOnly = maps:with(?READ_ONLY_KEYS,OldHConfig), + NewHConfig = + maps:merge( + maps:merge( + maps:merge(NewCommonConfig,NewHandlerConfig), + ReadOnly), + NewOlpOpts), + NewConfig = NewConfig0#{config=>NewHConfig}, + {ok,NewConfig}; + Error -> + Error end; {error,Faulty} -> {error,{invalid_config,Module,Faulty}} @@ -158,14 +170,12 @@ changing_config(SetOrUpdate, LogEvent :: logger:log_event(), Config :: logger:handler_config(). -log(LogEvent, Config = #{id := Name, - config := #{handler_pid := HPid, - mode_tab := ModeTab}}) -> +log(LogEvent, Config = #{config := #{olp:=Olp}}) -> %% if the handler has crashed, we must drop this event %% and hope the handler restarts so we can try again - true = is_process_alive(HPid), + true = is_process_alive(logger_olp:get_pid(Olp)), Bin = log_to_binary(LogEvent, Config), - call_cast_or_drop(Name, HPid, ModeTab, Bin). + logger_olp:load(Olp,Bin). %%%----------------------------------------------------------------- %%% Remove internal fields from configuration @@ -180,18 +190,23 @@ filter_config(#{config:=HConfig}=Config) -> %%% %%% The handler process is linked to logger_sup, which is part of the %%% kernel application's supervision tree. -start(#{id := Name} = Config0) -> +start(OlpOpts0, #{id := Name, module:=Module, config:=HConfig} = Config0) -> + RegName = ?name_to_reg_name(Module,Name), ChildSpec = #{id => Name, - start => {?MODULE, start_link, [Config0]}, + start => {logger_olp, start_link, [RegName,?MODULE, + Config0, OlpOpts0]}, restart => temporary, shutdown => 2000, type => worker, modules => [?MODULE]}, case supervisor:start_child(logger_sup, ChildSpec) of - {ok,Pid,Config} -> + {ok,Pid,Olp} -> ok = logger_handler_watcher:register_handler(Name,Pid), - {ok,Config}; + OlpOpts = logger_olp:get_opts(Olp), + {ok,Config0#{config=>(maps:merge(HConfig,OlpOpts))#{olp=>Olp}}}; + {error,{Reason,Ch}} when is_tuple(Ch), element(1,Ch)==child -> + {error,Reason}; Error -> Error end. @@ -200,103 +215,50 @@ start(#{id := Name} = Config0) -> %%% gen_server callbacks %%%=================================================================== -init(#{id := Name, module := Module, - formatter := Formatter, config := HConfig0} = Config0) -> - RegName = ?name_to_reg_name(Module,Name), - register(RegName, self()), +init(#{id := Name, module := Module, config := HConfig}) -> process_flag(trap_exit, true), - process_flag(message_queue_data, off_heap), ?init_test_hooks(), - ?start_observation(Name), - case Module:init(Name, HConfig0) of + case Module:init(Name, HConfig) of {ok,HState} -> - try ets:new(Name, [public]) of - ModeTab -> - ?set_mode(ModeTab, async), - T0 = ?timestamp(), - HConfig = HConfig0#{handler_pid => self(), - mode_tab => ModeTab}, - Config = Config0#{config => HConfig}, - proc_lib:init_ack({ok,self(),Config}), - %% Storing common config in state to avoid copying - %% (sending) the config data for each log message - CommonConfig = maps:with(?CONFIG_KEYS,HConfig), - State = - ?merge_with_stats( - CommonConfig#{id => Name, - module => Module, - mode_tab => ModeTab, - mode => async, - ctrl_sync_count => - ?CONTROLLER_SYNC_INTERVAL, - last_qlen => 0, - last_log_ts => T0, - last_op => sync, - burst_win_ts => T0, - burst_msg_count => 0, - formatter => Formatter, - handler_state => HState}), - State1 = set_repeated_filesync(State), - unset_restart_flag(State1), - gen_server:enter_loop(?MODULE, [], State1) - catch - _:Error -> - unregister(RegName), - error_notify({init_handler,Name,Error}), - proc_lib:init_ack(Error) - end; + %% Storing common config in state to avoid copying + %% (sending) the config data for each log message + CommonConfig = maps:with(?COMMON_KEYS,HConfig), + State = CommonConfig#{id => Name, + module => Module, + ctrl_sync_count => + ?CONTROLLER_SYNC_INTERVAL, + last_op => sync, + handler_state => HState}, + State1 = set_repeated_filesync(State), + {ok,State1}; Error -> - unregister(RegName), - error_notify({init_handler,Name,Error}), - proc_lib:init_ack(Error) + Error end. -%% This is the synchronous log event. -handle_call({log, Bin}, _From, State) -> - {Result,State1} = do_log(Bin, call, State), - %% Result == ok | dropped - {reply,Result, State1}; +%% This is the log event. +handle_load(Bin, #{id:=Name, + module:=Module, + handler_state:=HandlerState, + ctrl_sync_count := CtrlSync}=State) -> + if CtrlSync==0 -> + {_,HS1} = Module:write(Name, sync, Bin, HandlerState), + State#{handler_state => HS1, + ctrl_sync_count => ?CONTROLLER_SYNC_INTERVAL, + last_op=>write}; + true -> + {_,HS1} = Module:write(Name, async, Bin, HandlerState), + State#{handler_state => HS1, + ctrl_sync_count => CtrlSync-1, + last_op=>write} + end. handle_call(filesync, _From, State = #{id := Name, module := Module, handler_state := HandlerState}) -> {Result,HandlerState1} = Module:filesync(Name,sync,HandlerState), - {reply, Result, State#{handler_state=>HandlerState1, last_op=>sync}}; - -handle_call({change_config, #{formatter:=Formatter, config:=NewHConfig}}, _From, - State = #{filesync_repeat_interval := FSyncInt0}) -> - %% In the future, if handler_state must be updated due to config - %% change, then we need to add a callback to Module here. - CommonConfig = maps:with(?CONFIG_KEYS,NewHConfig), - State1 = maps:merge(State, CommonConfig), - State2 = - case maps:get(filesync_repeat_interval, NewHConfig) of - FSyncInt0 -> - State1; - _FSyncInt1 -> - set_repeated_filesync(cancel_repeated_filesync(State1)) - end, - {reply, ok, State2#{formatter:=Formatter}}; - -handle_call(info, _From, State) -> - {reply, State, State}; - -handle_call(reset, _From, - #{id:=Name,module:=Module,handler_state:=HandlerState}=State) -> - State1 = ?merge_with_stats(State), - {reply, ok, State1#{last_qlen => 0, - last_log_ts => ?timestamp(), - handler_state => Module:reset_state(Name,HandlerState)}}; - -handle_call(stop, _From, State) -> - {stop, {shutdown,stopped}, ok, State}. - -%% This is the asynchronous log event. -handle_cast({log, Bin}, State) -> - {_,State1} = do_log(Bin, cast, State), - {noreply, State1}; + {reply, Result, State#{handler_state=>HandlerState1, last_op=>sync}}. %% If FILESYNC_REPEAT_INTERVAL is set to a millisec value, this %% clause gets called repeatedly by the handler. In order to @@ -319,168 +281,96 @@ handle_cast(repeated_filesync, {_,HS} = Module:filesync(Name, async, HandlerState), State#{handler_state => HS, last_op => sync} end, - {noreply,set_repeated_filesync(State1)}. + {noreply,set_repeated_filesync(State1)}; +handle_cast({config_changed, CommonConfig, HConfig}, + State = #{id := Name, + module := Module, + handler_state := HandlerState, + filesync_repeat_interval := OldFSyncInt}) -> + State1 = + case maps:get(filesync_repeat_interval,CommonConfig) of + OldFSyncInt -> + State; + FSyncInt -> + set_repeated_filesync( + cancel_repeated_filesync( + State#{filesync_repeat_interval=>FSyncInt})) + end, + HS = try Module:config_changed(Name, HConfig, HandlerState) + catch error:undef -> HandlerState + end, + {noreply, State1#{handler_state => HS}}. handle_info(Info, #{id := Name, module := Module, handler_state := HandlerState} = State) -> {noreply,State#{handler_state => Module:handle_info(Name,Info,HandlerState)}}. -terminate(Reason, State = #{id := Name, - module := Module, - handler_state := HandlerState}) -> +terminate(overloaded=Reason, #{id:=Name}=State) -> + _ = log_handler_info(Name,"Handler ~p overloaded and stopping",[Name],State), + do_terminate(Reason,State), + ConfigResult = logger:get_handler_config(Name), + case ConfigResult of + {ok,#{module:=Module}=HConfig0} -> + spawn(fun() -> logger:remove_handler(Name) end), + HConfig = try Module:filter_config(HConfig0) + catch _:_ -> HConfig0 + end, + {ok,fun() -> logger:add_handler(Name,Module,HConfig) end}; + Error -> + error_notify({Name,restart_impossible,Error}), + Error + end; +terminate(Reason, State) -> + do_terminate(Reason, State). + +do_terminate(Reason, State = #{id := Name, + module := Module, + handler_state := HandlerState}) -> _ = cancel_repeated_filesync(State), _ = Module:terminate(Name, Reason, HandlerState), - ok = stop_or_restart(Name, Reason, State), - unregister(?name_to_reg_name(Module, Name)), ok. code_change(_OldVsn, State, _Extra) -> {ok, State}. +reset_state(#{id:=Name, module:=Module, handler_state:=HandlerState} = State) -> + State#{handler_state=>Module:reset_state(Name, HandlerState)}. %%%----------------------------------------------------------------- %%% Internal functions call(Module, Name, Op) when is_atom(Name) -> - call(?name_to_reg_name(Module,Name), Op); + case logger_olp:call(?name_to_reg_name(Module,Name), Op) of + {error,busy} -> {error,handler_busy}; + Other -> Other + end; call(_, Name, Op) -> {error,{badarg,{Op,[Name]}}}. -call(Server, Msg) -> - try - gen_server:call(Server, Msg, ?DEFAULT_CALL_TIMEOUT) - catch - _:{timeout,_} -> {error,handler_busy} - end. - -%% check for overload between every event (and set Mode to async, -%% sync or drop accordingly), but never flush the whole mailbox -%% before LogWindowSize events have been handled -do_log(Bin, CallOrCast, State = #{id:=Name, mode:=Mode0}) -> - T1 = ?timestamp(), - - %% check if the handler is getting overloaded, or if it's - %% recovering from overload (the check must be done for each - %% event to react quickly to large bursts of events and - %% to ensure that the handler can never end up in drop mode - %% with an empty mailbox, which would stop operation) - {Mode1,QLen,Mem,State1} = check_load(State), - - if (Mode1 == drop) andalso (Mode0 =/= drop) -> - log_handler_info(Name, "Handler ~p switched to drop mode", - [Name], State); - (Mode0 == drop) andalso ((Mode1 == async) orelse (Mode1 == sync)) -> - log_handler_info(Name, "Handler ~p switched to ~w mode", - [Name,Mode1], State); - true -> - ok - end, - - %% kill the handler if it can't keep up with the load - kill_if_choked(Name, QLen, Mem, State), - - if Mode1 == flush -> - flush(Name, QLen, T1, State1); - true -> - write(Name, Mode1, T1, Bin, CallOrCast, State1) - end. - -%% this clause is called by do_log/3 after an overload check -%% has been performed, where QLen > FlushQLen -flush(Name, _QLen0, T1, State=#{last_log_ts := _T0, mode_tab := ModeTab}) -> - %% flush messages in the mailbox (a limited number in - %% order to not cause long delays) - NewFlushed = flush_log_events(?FLUSH_MAX_N), - - %% write info in log about flushed messages +notify({mode_change,Mode0,Mode1},#{id:=Name}=State) -> + log_handler_info(Name,"Handler ~p switched from ~p to ~p mode", + [Name,Mode0,Mode1], State); +notify({flushed,Flushed},#{id:=Name}=State) -> log_handler_info(Name, "Handler ~p flushed ~w log events", - [Name,NewFlushed], State), - - %% because of the receive loop when flushing messages, the - %% handler will be scheduled out often and the mailbox could - %% grow very large, so we'd better check the queue again here - {_,_QLen1} = process_info(self(), message_queue_len), - ?observe(Name,{max_qlen,_QLen1}), - - %% Add 1 for the current log event - ?observe(Name,{flushed,NewFlushed+1}), - - State1 = ?update_max_time(?diff_time(T1,_T0),State), - State2 = ?update_max_qlen(_QLen1,State1), - {dropped,?update_other(flushed,FLUSHED,NewFlushed, - State2#{mode => ?set_mode(ModeTab,async), - last_qlen => 0, - last_log_ts => T1})}. - -%% this clause is called to write to file -write(Name, Mode, T1, Bin, _CallOrCast, - State = #{module := Module, - handler_state := HandlerState, - mode_tab := ModeTab, - ctrl_sync_count := CtrlSync, - last_qlen := LastQLen, - last_log_ts := T0}) -> - %% check if we need to limit the number of writes - %% during a burst of log events - {DoWrite,State1} = limit_burst(State), - - %% only log synhrounously every ?CONTROLLER_SYNC_INTERVAL time, to - %% give the handler time between writes so it can keep up with - %% incoming messages - {Result,LastQLen1,HandlerState1} = - if DoWrite, CtrlSync == 0 -> - ?observe(Name,{_CallOrCast,1}), - {_,HS1} = Module:write(Name, sync, Bin, HandlerState), - {ok,element(2, process_info(self(), message_queue_len)),HS1}; - DoWrite -> - ?observe(Name,{_CallOrCast,1}), - {_,HS1} = Module:write(Name, async, Bin, HandlerState), - {ok,LastQLen,HS1}; - not DoWrite -> - ?observe(Name,{flushed,1}), - {dropped,LastQLen,HandlerState} - end, - - %% Check if the time since the previous log event is long enough - - %% and the queue length small enough - to assume the mailbox has - %% been emptied, and if so, do filesync operation and reset mode to - %% async. Note that this is the best we can do to detect an idle - %% handler without setting a timer after each log call/cast. If the - %% time between two consecutive log events is fast and no new - %% event comes in after the last one, idle state won't be detected! - Time = ?diff_time(T1,T0), - State2 = - if (LastQLen1 < ?FILESYNC_OK_QLEN) andalso - (Time > ?IDLE_DETECT_TIME_USEC) -> - {_,HS2} = Module:filesync(Name,async,HandlerState), - State1#{mode => ?change_mode(ModeTab, Mode, async), - burst_msg_count => 0, - handler_state => HS2}; - true -> - State1#{mode => Mode, handler_state => HandlerState1} - end, - State3 = ?update_calls_or_casts(_CallOrCast,1,State2), - State4 = ?update_max_qlen(LastQLen1,State3), - State5 = - ?update_max_time(Time, - State4#{last_qlen := LastQLen1, - last_log_ts => T1, - last_op => write, - ctrl_sync_count => - if CtrlSync==0 -> ?CONTROLLER_SYNC_INTERVAL; - true -> CtrlSync-1 - end}), - {Result,State5}. + [Name,Flushed], State); +notify(restart,#{id:=Name}=State) -> + log_handler_info(Name, "Handler ~p restarted", [Name], State); +notify(idle,#{id:=Name,module:=Module,handler_state:=HandlerState}=State) -> + {_,HS} = Module:filesync(Name,async,HandlerState), + State#{handler_state=>HS, last_op=>sync}. log_handler_info(Name, Format, Args, #{module:=Module, - formatter:=Formatter, - handler_state:=HandlerState}) -> - Config = #{formatter=>Formatter}, - Meta = #{time=>erlang:system_time(microsecond)}, + handler_state:=HandlerState}=State) -> + Config = + case logger:get_handler_config(Name) of + {ok,Conf} -> Conf; + _ -> #{formatter=>{?DEFAULT_FORMATTER,?DEFAULT_FORMAT_CONFIG}} + end, + Meta = #{time=>logger:timestamp()}, Bin = log_to_binary(#{level => notice, msg => {Format,Args}, meta => Meta}, Config), - _ = Module:write(Name, async, Bin, HandlerState), - ok. + {_,HS} = Module:write(Name, async, Bin, HandlerState), + State#{handler_state=>HS, last_op=>write}. %%%----------------------------------------------------------------- %%% Convert log data on any form to binary @@ -540,42 +430,8 @@ string_to_binary(String) -> %%%----------------------------------------------------------------- %%% Check that the configuration term is valid check_config(Config) when is_map(Config) -> - case check_common_config(maps:to_list(Config)) of - ok -> - case overload_levels_ok(Config) of - true -> - ok; - false -> - Faulty = maps:with([sync_mode_qlen, - drop_mode_qlen, - flush_qlen],Config), - {error,{invalid_levels,Faulty}} - end; - Error -> - Error - end. + check_common_config(maps:to_list(Config)). -check_common_config([{sync_mode_qlen,N}|Config]) when is_integer(N) -> - check_common_config(Config); -check_common_config([{drop_mode_qlen,N}|Config]) when is_integer(N) -> - check_common_config(Config); -check_common_config([{flush_qlen,N}|Config]) when is_integer(N) -> - check_common_config(Config); -check_common_config([{burst_limit_enable,Bool}|Config]) when is_boolean(Bool) -> - check_common_config(Config); -check_common_config([{burst_limit_max_count,N}|Config]) when is_integer(N) -> - check_common_config(Config); -check_common_config([{burst_limit_window_time,N}|Config]) when is_integer(N) -> - check_common_config(Config); -check_common_config([{overload_kill_enable,Bool}|Config]) when is_boolean(Bool) -> - check_common_config(Config); -check_common_config([{overload_kill_qlen,N}|Config]) when is_integer(N) -> - check_common_config(Config); -check_common_config([{overload_kill_mem_size,N}|Config]) when is_integer(N) -> - check_common_config(Config); -check_common_config([{overload_kill_restart_after,NorA}|Config]) - when is_integer(NorA); NorA == infinity -> - check_common_config(Config); check_common_config([{filesync_repeat_interval,NorA}|Config]) when is_integer(NorA); NorA == no_repeat -> check_common_config(Config); @@ -585,156 +441,7 @@ check_common_config([]) -> ok. get_default_config() -> - #{sync_mode_qlen => ?SYNC_MODE_QLEN, - drop_mode_qlen => ?DROP_MODE_QLEN, - flush_qlen => ?FLUSH_QLEN, - burst_limit_enable => ?BURST_LIMIT_ENABLE, - burst_limit_max_count => ?BURST_LIMIT_MAX_COUNT, - burst_limit_window_time => ?BURST_LIMIT_WINDOW_TIME, - overload_kill_enable => ?OVERLOAD_KILL_ENABLE, - overload_kill_qlen => ?OVERLOAD_KILL_QLEN, - overload_kill_mem_size => ?OVERLOAD_KILL_MEM_SIZE, - overload_kill_restart_after => ?OVERLOAD_KILL_RESTART_AFTER, - filesync_repeat_interval => ?FILESYNC_REPEAT_INTERVAL}. - -%%%----------------------------------------------------------------- -%%% Overload Protection -call_cast_or_drop(_Name, HandlerPid, ModeTab, Bin) -> - %% If the handler process is getting overloaded, the log event - %% will be synchronous instead of asynchronous (slows down the - %% logging tempo of a process doing lots of logging. If the - %% handler is choked, drop mode is set and no event will be sent. - try ?get_mode(ModeTab) of - async -> - gen_server:cast(HandlerPid, {log,Bin}); - sync -> - case call(HandlerPid, {log,Bin}) of - ok -> - ok; - _Other -> - %% dropped or {error,handler_busy} - ?observe(_Name,{dropped,1}), - ok - end; - drop -> - ?observe(_Name,{dropped,1}) - catch - %% if the ETS table doesn't exist (maybe because of a - %% handler restart), we can only drop the event - _:_ -> ?observe(_Name,{dropped,1}) - end, - ok. - -set_restart_flag(#{id := Name, module := Module} = State) -> - log_handler_info(Name, "Handler ~p overloaded and stopping", [Name], State), - Flag = list_to_atom(lists:concat([Module,"_",Name,"_restarting"])), - spawn(fun() -> - register(Flag, self()), - timer:sleep(infinity) - end), - ok. - -unset_restart_flag(#{id := Name, module := Module} = State) -> - Flag = list_to_atom(lists:concat([Module,"_",Name,"_restarting"])), - case whereis(Flag) of - undefined -> - ok; - Pid -> - exit(Pid, kill), - log_handler_info(Name, "Handler ~p restarted", [Name], State) - end. - -check_load(State = #{id:=_Name, mode_tab := ModeTab, mode := Mode, - sync_mode_qlen := SyncModeQLen, - drop_mode_qlen := DropModeQLen, - flush_qlen := FlushQLen}) -> - {_,Mem} = process_info(self(), memory), - ?observe(_Name,{max_mem,Mem}), - {_,QLen} = process_info(self(), message_queue_len), - ?observe(_Name,{max_qlen,QLen}), - %% When the handler process gets scheduled in, it's impossible - %% to predict the QLen. We could jump "up" arbitrarily from say - %% async to sync, async to drop, sync to flush, etc. However, when - %% the handler process manages the log events (without flushing), - %% one after the other, we will move "down" from drop to sync and - %% from sync to async. This way we don't risk getting stuck in - %% drop or sync mode with an empty mailbox. - {Mode1,_NewDrops,_NewFlushes} = - if - QLen >= FlushQLen -> - {flush, 0,1}; - QLen >= DropModeQLen -> - %% Note that drop mode will force log events to - %% be dropped on the client side (never sent get to - %% the handler). - IncDrops = if Mode == drop -> 0; true -> 1 end, - {?change_mode(ModeTab, Mode, drop), IncDrops,0}; - QLen >= SyncModeQLen -> - {?change_mode(ModeTab, Mode, sync), 0,0}; - true -> - {?change_mode(ModeTab, Mode, async), 0,0} - end, - State1 = ?update_other(drops,DROPS,_NewDrops,State), - {Mode1, QLen, Mem, - ?update_other(flushes,FLUSHES,_NewFlushes, - State1#{last_qlen => QLen})}. - -limit_burst(#{burst_limit_enable := false}=State) -> - {true,State}; -limit_burst(#{burst_win_ts := BurstWinT0, - burst_msg_count := BurstMsgCount, - burst_limit_window_time := BurstLimitWinTime, - burst_limit_max_count := BurstLimitMaxCnt} = State) -> - if (BurstMsgCount >= BurstLimitMaxCnt) -> - %% the limit for allowed messages has been reached - BurstWinT1 = ?timestamp(), - case ?diff_time(BurstWinT1,BurstWinT0) of - BurstCheckTime when BurstCheckTime < (BurstLimitWinTime*1000) -> - %% we're still within the burst time frame - {false,?update_other(burst_drops,BURSTS,1,State)}; - _BurstCheckTime -> - %% burst time frame passed, reset counters - {true,State#{burst_win_ts => BurstWinT1, - burst_msg_count => 0}} - end; - true -> - %% the limit for allowed messages not yet reached - {true,State#{burst_win_ts => BurstWinT0, - burst_msg_count => BurstMsgCount+1}} - end. - -kill_if_choked(Name, QLen, Mem, State = #{overload_kill_enable := KillIfOL, - overload_kill_qlen := OLKillQLen, - overload_kill_mem_size := OLKillMem}) -> - if KillIfOL andalso - ((QLen > OLKillQLen) orelse (Mem > OLKillMem)) -> - set_restart_flag(State), - exit({shutdown,{overloaded,Name,QLen,Mem}}); - true -> - ok - end. - -flush_log_events(Limit) -> - process_flag(priority, high), - Flushed = flush_log_events(0, Limit), - process_flag(priority, normal), - Flushed. - -flush_log_events(Limit, Limit) -> - Limit; -flush_log_events(N, Limit) -> - %% flush log events but leave other events, such as - %% filesync, info and change_config, so that these - %% have a chance to be processed even under heavy load - receive - {'$gen_cast',{log,_}} -> - flush_log_events(N+1, Limit); - {'$gen_call',{Pid,MRef},{log,_}} -> - Pid ! {MRef, dropped}, - flush_log_events(N+1, Limit) - after - 0 -> N - end. + #{filesync_repeat_interval => ?FILESYNC_REPEAT_INTERVAL}. set_repeated_filesync(#{filesync_repeat_interval:=FSyncInt} = State) when is_integer(FSyncInt) -> @@ -752,51 +459,5 @@ cancel_repeated_filesync(State) -> error -> State end. - -stop_or_restart(Name, {shutdown,Reason={overloaded,_Name,_QLen,_Mem}}, - #{overload_kill_restart_after := RestartAfter}) -> - %% If we're terminating because of an overload situation (see - %% kill_if_choked/4), we need to remove the handler and set a - %% restart timer. A separate process must perform this in order to - %% avoid deadlock. - HandlerPid = self(), - ConfigResult = logger:get_handler_config(Name), - RemoveAndRestart = - fun() -> - MRef = erlang:monitor(process, HandlerPid), - receive - {'DOWN',MRef,_,_,_} -> - ok - after 30000 -> - error_notify(Reason), - exit(HandlerPid, kill) - end, - case ConfigResult of - {ok,#{module:=HMod}=HConfig0} when is_integer(RestartAfter) -> - _ = logger:remove_handler(Name), - HConfig = try HMod:filter_config(HConfig0) - catch _:_ -> HConfig0 - end, - _ = timer:apply_after(RestartAfter, logger, add_handler, - [Name,HMod,HConfig]); - {ok,_} -> - _ = logger:remove_handler(Name); - {error,CfgReason} when is_integer(RestartAfter) -> - error_notify({Name,restart_impossible,CfgReason}); - {error,_} -> - ok - end - end, - spawn(RemoveAndRestart), - ok; -stop_or_restart(_Name, _Reason, _State) -> - ok. - -overload_levels_ok(HandlerConfig) -> - SMQL = maps:get(sync_mode_qlen, HandlerConfig, ?SYNC_MODE_QLEN), - DMQL = maps:get(drop_mode_qlen, HandlerConfig, ?DROP_MODE_QLEN), - FQL = maps:get(flush_qlen, HandlerConfig, ?FLUSH_QLEN), - (DMQL > 1) andalso (SMQL =< DMQL) andalso (DMQL =< FQL). - error_notify(Term) -> ?internal_log(error, Term). diff --git a/lib/kernel/src/logger_h_common.hrl b/lib/kernel/src/logger_h_common.hrl index 261b0a6246..004a61d9d9 100644 --- a/lib/kernel/src/logger_h_common.hrl +++ b/lib/kernel/src/logger_h_common.hrl @@ -1,50 +1,22 @@ - -%%%----------------------------------------------------------------- -%%% Overload protection configuration - -%%! *** NOTE *** -%%! It's important that: -%%! SYNC_MODE_QLEN =< DROP_MODE_QLEN =< FLUSH_QLEN -%%! and that DROP_MODE_QLEN >= 2. -%%! Otherwise the handler could end up in drop mode with no new -%%! log requests to process. This would cause all future requests -%%! to be dropped (no switch to async mode would ever take place). - -%% This specifies the message_queue_len value where the log -%% requests switch from asynchronous casts to synchronous calls. --define(SYNC_MODE_QLEN, 10). -%% Above this message_queue_len, log requests will be dropped, -%% i.e. no log requests get sent to the handler process. --define(DROP_MODE_QLEN, 200). -%% Above this message_queue_len, the handler process will flush -%% its mailbox and only leave this number of messages in it. --define(FLUSH_QLEN, 1000). - -%% Never flush more than this number of messages in one go, -%% or the handler will be unresponsive for seconds (keep this -%% number as large as possible or the mailbox could grow large). --define(FLUSH_MAX_N, 5000). - -%% BURST_LIMIT_MAX_COUNT is the max number of log requests allowed -%% to be written within a BURST_LIMIT_WINDOW_TIME time frame. --define(BURST_LIMIT_ENABLE, true). --define(BURST_LIMIT_MAX_COUNT, 500). --define(BURST_LIMIT_WINDOW_TIME, 1000). - -%% This enables/disables the feature to automatically get the -%% handler terminated if it gets too loaded (and can't keep up). --define(OVERLOAD_KILL_ENABLE, false). -%% If the message_queue_len goes above this size even after -%% flushing has been performed, the handler is terminated. --define(OVERLOAD_KILL_QLEN, 20000). -%% If the memory usage exceeds this level --define(OVERLOAD_KILL_MEM_SIZE, 3000000). - -%% This is the default time that the handler will wait before -%% restarting and accepting new requests. The value 'infinity' -%% disables restarts. --define(OVERLOAD_KILL_RESTART_AFTER, 5000). -%%-define(OVERLOAD_KILL_RESTART_AFTER, infinity). +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1997-2015. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% %% The handler sends asynchronous write requests to the process %% controlling the i/o device, but every once in this interval @@ -65,12 +37,6 @@ -define(FILESYNC_REPEAT_INTERVAL, 5000). %%-define(FILESYNC_REPEAT_INTERVAL, no_repeat). -%% This is the time after last message received that we think/hope -%% that the handler has an empty mailbox (no new log request has -%% come in). --define(IDLE_DETECT_TIME_MSEC, 100). --define(IDLE_DETECT_TIME_USEC, 100000). - %% Default disk log option values -define(DISK_LOG_TYPE, wrap). -define(DISK_LOG_MAX_NO_FILES, 10). @@ -83,43 +49,6 @@ list_to_atom(lists:concat([MODULE,"_",Name]))). %%%----------------------------------------------------------------- -%%% Overload protection macros - --define(timestamp(), erlang:monotonic_time(microsecond)). - --define(get_mode(Tid), - case ets:lookup(Tid, mode) of - [{mode,M}] -> M; - _ -> async - end). - --define(set_mode(Tid, M), - begin ets:insert(Tid, {mode,M}), M end). - --define(change_mode(Tid, M0, M1), - if M0 == M1 -> - M0; - true -> - ets:insert(Tid, {mode,M1}), - M1 - end). - --define(min(X1, X2), - if X2 == undefined -> X1; - X2 < X1 -> X2; - true -> X1 - end). - --define(max(X1, X2), - if - X2 == undefined -> X1; - X2 > X1 -> X2; - true -> X1 - end). - --define(diff_time(OS_T1, OS_T0), OS_T1-OS_T0). - -%%%----------------------------------------------------------------- %%% The test hook macros make it possible to observe and manipulate %%% internal handler functionality. When enabled, these macros will %%% slow down execution and therefore should not be include in code @@ -183,7 +112,6 @@ [{_,ERROR}] -> ERROR catch _:_ -> disk_log:sync(LOG) end). - -define(DEFAULT_CALL_TIMEOUT, 5000). -else. % DEFAULTS! -define(TEST_HOOKS_TAB, undefined). @@ -196,68 +124,4 @@ -define(file_datasync(DEVICE), file:datasync(DEVICE)). -define(disk_log_write(LOG, MODE, DATA), disk_log_write(LOG, MODE, DATA)). -define(disk_log_sync(LOG), disk_log:sync(LOG)). - -define(DEFAULT_CALL_TIMEOUT, 10000). --endif. - -%%%----------------------------------------------------------------- -%%% These macros enable statistics counters in the state of the -%%% handler which is useful for analysing the overload protection -%%% behaviour. These counters should not be included in code to be -%%% officially released (as some counters will grow very large -%%% over time). - -%%-define(SAVE_STATS, true). --ifdef(SAVE_STATS). - -define(merge_with_stats(STATE), - STATE#{flushes => 0, flushed => 0, drops => 0, - burst_drops => 0, casts => 0, calls => 0, - max_qlen => 0, max_time => 0}). - - -define(update_max_qlen(QLEN, STATE), - begin #{max_qlen := QLEN0} = STATE, - STATE#{max_qlen => ?max(QLEN0,QLEN)} end). - - -define(update_calls_or_casts(CALL_OR_CAST, INC, STATE), - case CALL_OR_CAST of - cast -> - #{casts := CASTS0} = STATE, - STATE#{casts => CASTS0+INC}; - call -> - #{calls := CALLS0} = STATE, - STATE#{calls => CALLS0+INC} - end). - - -define(update_max_time(TIME, STATE), - begin #{max_time := TIME0} = STATE, - STATE#{max_time => ?max(TIME0,TIME)} end). - - -define(update_other(OTHER, VAR, INCVAL, STATE), - begin #{OTHER := VAR} = STATE, - STATE#{OTHER => VAR+INCVAL} end). - --else. % DEFAULT! - -define(merge_with_stats(STATE), STATE). - -define(update_max_qlen(_QLEN, STATE), STATE). - -define(update_calls_or_casts(_CALL_OR_CAST, _INC, STATE), STATE). - -define(update_max_time(_TIME, STATE), STATE). - -define(update_other(_OTHER, _VAR, _INCVAL, STATE), STATE). --endif. - -%%%----------------------------------------------------------------- -%%% These macros enable callbacks that make it possible to analyse -%%% the overload protection behaviour from outside the handler -%%% process (including dropped requests on the client side). -%%% An external callback module (?OBSERVER_MOD) is required which -%%% is not part of the kernel application. For this reason, these -%%% callbacks should not be included in code to be officially released. - -%%-define(OBSERVER_MOD, logger_test). --ifdef(OBSERVER_MOD). - -define(start_observation(NAME), ?OBSERVER:start_observation(NAME)). - -define(observe(NAME,EVENT), ?OBSERVER:observe(NAME,EVENT)). - --else. % DEFAULT! - -define(start_observation(_NAME), ok). - -define(observe(_NAME,_EVENT), ok). -endif. -%%! <--- diff --git a/lib/kernel/src/logger_internal.hrl b/lib/kernel/src/logger_internal.hrl index d96a4ac78b..e53922e5d3 100644 --- a/lib/kernel/src/logger_internal.hrl +++ b/lib/kernel/src/logger_internal.hrl @@ -19,6 +19,7 @@ %% -include_lib("kernel/include/logger.hrl"). -define(LOGGER_TABLE,logger). +-define(PROXY_KEY,'$proxy_config$'). -define(PRIMARY_KEY,'$primary_config$'). -define(HANDLER_KEY,'$handler_config$'). -define(LOGGER_META_KEY,'$logger_metadata$'). @@ -40,12 +41,14 @@ -define(DEFAULT_LOGGER_CALL_TIMEOUT, infinity). --define(LOG_INTERNAL(Level,Report), +-define(LOG_INTERNAL(Level,Report),?DO_LOG_INTERNAL(Level,[Report])). +-define(LOG_INTERNAL(Level,Format,Args),?DO_LOG_INTERNAL(Level,[Format,Args])). +-define(DO_LOG_INTERNAL(Level,Data), case logger:allow(Level,?MODULE) of true -> %% Spawn this to avoid deadlocks - _ = spawn(logger,macro_log,[?LOCATION,Level,Report, - logger:add_default_metadata(#{})]), + _ = spawn(logger,macro_log,[?LOCATION,Level|Data]++ + [logger:add_default_metadata(#{})]), ok; false -> ok diff --git a/lib/kernel/src/logger_olp.erl b/lib/kernel/src/logger_olp.erl new file mode 100644 index 0000000000..8365383fe2 --- /dev/null +++ b/lib/kernel/src/logger_olp.erl @@ -0,0 +1,627 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(logger_olp). +-behaviour(gen_server). + +-include("logger_olp.hrl"). +-include("logger_internal.hrl"). + +%% API +-export([start_link/4, load/2, info/1, reset/1, stop/1, restart/1, + set_opts/2, get_opts/1, get_default_opts/0, get_pid/1, + call/2, cast/2, get_ref/0, get_ref/1]). + +%% gen_server and proc_lib callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-define(OPT_KEYS,[sync_mode_qlen, + drop_mode_qlen, + flush_qlen, + burst_limit_enable, + burst_limit_max_count, + burst_limit_window_time, + overload_kill_enable, + overload_kill_qlen, + overload_kill_mem_size, + overload_kill_restart_after]). + +-export_type([olp_ref/0, options/0]). + +-opaque olp_ref() :: {atom(),pid(),ets:tid()}. + +-type options() :: logger:olp_config(). + +%%%----------------------------------------------------------------- +%%% API + +-spec start_link(Name,Module,Args,Options) -> {ok,Pid,Olp} | {error,Reason} when + Name :: atom(), + Module :: module(), + Args :: term(), + Options :: options(), + Pid :: pid(), + Olp :: olp_ref(), + Reason :: term(). +start_link(Name,Module,Args,Options0) when is_map(Options0) -> + Options = maps:merge(get_default_opts(),Options0), + case check_opts(Options) of + ok -> + proc_lib:start_link(?MODULE,init,[[Name,Module,Args,Options]]); + Error -> + Error + end. + +-spec load(Olp, Msg) -> ok when + Olp :: olp_ref(), + Msg :: term(). +load({_Name,Pid,ModeRef},Msg) -> + %% If the process is getting overloaded, the message will be + %% synchronous instead of asynchronous (slows down the tempo of a + %% process causing much load). If the process is choked, drop mode + %% is set and no message is sent. + try ?get_mode(ModeRef) of + async -> + gen_server:cast(Pid, {'$olp_load',Msg}); + sync -> + case call(Pid, {'$olp_load',Msg}) of + ok -> + ok; + _Other -> + %% dropped or {error,busy} + ?observe(_Name,{dropped,1}), + ok + end; + drop -> + ?observe(_Name,{dropped,1}) + catch + %% if the ETS table doesn't exist (maybe because of a + %% process restart), we can only drop the event + _:_ -> ?observe(_Name,{dropped,1}) + end, + ok. + +-spec info(Olp) -> map() | {error, busy} when + Olp :: atom() | pid() | olp_ref(). +info(Olp) -> + call(Olp, info). + +-spec reset(Olp) -> ok | {error, busy} when + Olp :: atom() | pid() | olp_ref(). +reset(Olp) -> + call(Olp, reset). + +-spec stop(Olp) -> ok when + Olp :: atom() | pid() | olp_ref(). +stop({_Name,Pid,_ModRef}) -> + stop(Pid); +stop(Pid) -> + _ = gen_server:call(Pid, stop), + ok. + +-spec set_opts(Olp, Opts) -> ok | {error,term()} | {error, busy} when + Olp :: atom() | pid() | olp_ref(), + Opts :: options(). +set_opts(Olp, Opts) -> + call(Olp, {set_opts,Opts}). + +-spec get_opts(Olp) -> options() | {error, busy} when + Olp :: atom() | pid() | olp_ref(). +get_opts(Olp) -> + call(Olp, get_opts). + +-spec get_default_opts() -> options(). +get_default_opts() -> + #{sync_mode_qlen => ?SYNC_MODE_QLEN, + drop_mode_qlen => ?DROP_MODE_QLEN, + flush_qlen => ?FLUSH_QLEN, + burst_limit_enable => ?BURST_LIMIT_ENABLE, + burst_limit_max_count => ?BURST_LIMIT_MAX_COUNT, + burst_limit_window_time => ?BURST_LIMIT_WINDOW_TIME, + overload_kill_enable => ?OVERLOAD_KILL_ENABLE, + overload_kill_qlen => ?OVERLOAD_KILL_QLEN, + overload_kill_mem_size => ?OVERLOAD_KILL_MEM_SIZE, + overload_kill_restart_after => ?OVERLOAD_KILL_RESTART_AFTER}. + +-spec restart(fun(() -> any())) -> ok. +restart(Fun) -> + Result = + try Fun() + catch C:R:S -> + {error,{restart_failed,Fun,C,R,S}} + end, + ?LOG_INTERNAL(debug,[{logger_olp,restart}, + {result,Result}]), + ok. + +-spec get_ref() -> olp_ref(). +get_ref() -> + get(olp_ref). + +-spec get_ref(PidOrName) -> olp_ref() | {error, busy} when + PidOrName :: pid() | atom(). +get_ref(PidOrName) -> + call(PidOrName,get_ref). + +-spec get_pid(olp_ref()) -> pid(). +get_pid({_Name,Pid,_ModeRef}) -> + Pid. + +%%%=================================================================== +%%% gen_server callbacks +%%%=================================================================== + +init([Name,Module,Args,Options]) -> + register(Name, self()), + process_flag(message_queue_data, off_heap), + + ?start_observation(Name), + + try ets:new(Name, [public]) of + ModeRef -> + OlpRef = {Name,self(),ModeRef}, + put(olp_ref,OlpRef), + try Module:init(Args) of + {ok,CBState} -> + ?set_mode(ModeRef, async), + T0 = ?timestamp(), + proc_lib:init_ack({ok,self(),OlpRef}), + %% Storing options in state to avoid copying + %% (sending) the option data with each message + State0 = ?merge_with_stats( + Options#{id => Name, + idle=> true, + module => Module, + mode_ref => ModeRef, + mode => async, + last_qlen => 0, + last_load_ts => T0, + burst_win_ts => T0, + burst_msg_count => 0, + cb_state => CBState}), + State = reset_restart_flag(State0), + gen_server:enter_loop(?MODULE, [], State); + Error -> + _ = ets:delete(ModeRef), + unregister(Name), + proc_lib:init_ack(Error) + catch + _:Error -> + _ = ets:delete(ModeRef), + unregister(Name), + proc_lib:init_ack(Error) + end + catch + _:Error -> + unregister(Name), + proc_lib:init_ack(Error) + end. + +%% This is the synchronous load event. +handle_call({'$olp_load', Msg}, _From, State) -> + {Result,State1} = do_load(Msg, call, State#{idle=>false}), + %% Result == ok | dropped + reply_return(Result,State1); + +handle_call(get_ref,_From,#{id:=Name,mode_ref:=ModeRef}=State) -> + reply_return({Name,self(),ModeRef},State); + +handle_call({set_opts,Opts0},_From,State) -> + Opts = maps:merge(maps:with(?OPT_KEYS,State),Opts0), + case check_opts(Opts) of + ok -> + reply_return(ok, maps:merge(State,Opts)); + Error -> + reply_return(Error, State) + end; + +handle_call(get_opts,_From,State) -> + reply_return(maps:with(?OPT_KEYS,State), State); + +handle_call(info, _From, State) -> + reply_return(State, State); + +handle_call(reset, _From, #{module:=Module,cb_state:=CBState}=State) -> + State1 = ?merge_with_stats(State), + CBState1 = try_callback_call(Module,reset_state,[CBState],CBState), + reply_return(ok, State1#{idle => true, + last_qlen => 0, + last_load_ts => ?timestamp(), + cb_state => CBState1}); + +handle_call(stop, _From, State) -> + {stop, {shutdown,stopped}, ok, State}; + +handle_call(Msg, From, #{module:=Module,cb_state:=CBState}=State) -> + case try_callback_call(Module,handle_call,[Msg, From, CBState]) of + {reply,Reply,CBState1} -> + reply_return(Reply,State#{cb_state=>CBState1}); + {noreply,CBState1} -> + noreply_return(State#{cb_state=>CBState1}); + {stop, Reason, Reply, CBState1} -> + {stop, Reason, Reply, State#{cb_state=>CBState1}}; + {stop, Reason, CBState1} -> + {stop, Reason, State#{cb_state=>CBState1}} + end. + +%% This is the asynchronous load event. +handle_cast({'$olp_load', Msg}, State) -> + {_Result,State1} = do_load(Msg, cast, State#{idle=>false}), + noreply_return(State1); + +handle_cast(Msg, #{module:=Module, cb_state:=CBState} = State) -> + case try_callback_call(Module,handle_cast,[Msg, CBState]) of + {noreply,CBState1} -> + noreply_return(State#{cb_state=>CBState1}); + {stop, Reason, CBState1} -> + {stop, Reason, State#{cb_state=>CBState1}} + end. + +handle_info(timeout, #{mode_ref:=_ModeRef, mode:=Mode} = State) -> + State1 = notify(idle,State), + State2 = maybe_notify_mode_change(async,State1), + {noreply, State2#{idle => true, + mode => ?change_mode(_ModeRef, Mode, async), + burst_msg_count => 0}}; +handle_info(Msg, #{module := Module, cb_state := CBState} = State) -> + case try_callback_call(Module,handle_info,[Msg, CBState]) of + {noreply,CBState1} -> + noreply_return(State#{cb_state=>CBState1}); + {stop, Reason, CBState1} -> + {stop, Reason, State#{cb_state=>CBState1}}; + {load,CBState1} -> + {_,State1} = do_load(Msg, cast, State#{idle=>false, + cb_state=>CBState1}), + noreply_return(State1) + end. + +terminate({shutdown,{overloaded,_QLen,_Mem}}, + #{id:=Name, module := Module, cb_state := CBState, + overload_kill_restart_after := RestartAfter} = State) -> + %% We're terminating because of an overload situation (see + %% kill_if_choked/3). + unregister(Name), %%!!!! to avoid error printout of callback crashed on stop + case try_callback_call(Module,terminate,[overloaded,CBState],ok) of + {ok,Fun} when is_function(Fun,0), is_integer(RestartAfter) -> + set_restart_flag(State), + _ = timer:apply_after(RestartAfter,?MODULE,restart,[Fun]), + ok; + _ -> + ok + end; +terminate(Reason, #{id:=Name, module:=Module, cb_state:=CBState}) -> + _ = try_callback_call(Module,terminate,[Reason,CBState],ok), + unregister(Name), + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + + +%%%----------------------------------------------------------------- +%%% Internal functions +-spec call(Olp, term()) -> term() | {error,busy} when + Olp :: atom() | pid() | olp_ref(). +call({_Name, Pid, _ModeRef},Msg) -> + call(Pid, Msg); +call(Server, Msg) -> + try + gen_server:call(Server, Msg) + catch + _:{timeout,_} -> {error,busy} + end. + +-spec cast(olp_ref(),term()) -> ok. +cast({_Name, Pid, _ModeRef},Msg) -> + gen_server:cast(Pid, Msg). + +%% check for overload between every event (and set Mode to async, +%% sync or drop accordingly), but never flush the whole mailbox +%% before LogWindowSize events have been handled +do_load(Msg, CallOrCast, State) -> + T1 = ?timestamp(), + State1 = ?update_time(T1,State), + + %% check if the process is getting overloaded, or if it's + %% recovering from overload (the check must be done for each + %% event to react quickly to large bursts of events and + %% to ensure that the handler can never end up in drop mode + %% with an empty mailbox, which would stop operation) + {Mode1,QLen,Mem,State2} = check_load(State1), + + %% kill the handler if it can't keep up with the load + kill_if_choked(QLen, Mem, State2), + + if Mode1 == flush -> + flush(T1, State2); + true -> + handle_load(Mode1, T1, Msg, CallOrCast, State2) + end. + +%% this function is called by do_load/3 after an overload check +%% has been performed, where QLen > FlushQLen +flush(T1, State=#{id := _Name, mode := Mode, last_load_ts := _T0, mode_ref := ModeRef}) -> + %% flush load messages in the mailbox (a limited number in order + %% to not cause long delays) + NewFlushed = flush_load(?FLUSH_MAX_N), + + %% write info in log about flushed messages + State1=notify({flushed,NewFlushed},State), + + %% because of the receive loop when flushing messages, the + %% handler will be scheduled out often and the mailbox could + %% grow very large, so we'd better check the queue again here + {_,QLen1} = process_info(self(), message_queue_len), + ?observe(_Name,{max_qlen,QLen1}), + + %% Add 1 for the current log event + ?observe(_Name,{flushed,NewFlushed+1}), + + State2 = ?update_max_time(?diff_time(T1,_T0),State1), + State3 = ?update_max_qlen(QLen1,State2), + State4 = maybe_notify_mode_change(async,State3), + {dropped,?update_other(flushed,FLUSHED,NewFlushed, + State4#{mode => ?change_mode(ModeRef,Mode,async), + last_qlen => QLen1, + last_load_ts => T1})}. + +%% this function is called to actually handle the message +handle_load(Mode, T1, Msg, _CallOrCast, + State = #{id := _Name, + module := Module, + cb_state := CBState, + last_qlen := LastQLen, + last_load_ts := _T0}) -> + %% check if we need to limit the number of writes + %% during a burst of log events + {DoWrite,State1} = limit_burst(State), + + {Result,LastQLen1,CBState1} = + if DoWrite -> + ?observe(_Name,{_CallOrCast,1}), + CBS = try_callback_call(Module,handle_load,[Msg,CBState]), + {ok,element(2, process_info(self(), message_queue_len)),CBS}; + true -> + ?observe(_Name,{flushed,1}), + {dropped,LastQLen,CBState} + end, + State2 = State1#{cb_state=>CBState1}, + + State3 = State2#{mode => Mode}, + State4 = ?update_calls_or_casts(_CallOrCast,1,State3), + State5 = ?update_max_qlen(LastQLen1,State4), + State6 = + ?update_max_time(?diff_time(T1,_T0), + State5#{last_qlen := LastQLen1, + last_load_ts => T1}), + State7 = case Result of + ok -> + S = ?update_freq(T1,State6), + ?update_other(writes,WRITES,1,S); + _ -> + State6 + end, + {Result,State7}. + + +%%%----------------------------------------------------------------- +%%% Check that the options are valid +check_opts(Options) when is_map(Options) -> + case do_check_opts(maps:to_list(Options)) of + ok -> + case overload_levels_ok(Options) of + true -> + ok; + false -> + Faulty = maps:with([sync_mode_qlen, + drop_mode_qlen, + flush_qlen],Options), + {error,{invalid_olp_levels,Faulty}} + end; + {error,Key,Value} -> + {error,{invalid_olp_config,#{Key=>Value}}} + end. + +do_check_opts([{sync_mode_qlen,N}|Options]) when is_integer(N) -> + do_check_opts(Options); +do_check_opts([{drop_mode_qlen,N}|Options]) when is_integer(N) -> + do_check_opts(Options); +do_check_opts([{flush_qlen,N}|Options]) when is_integer(N) -> + do_check_opts(Options); +do_check_opts([{burst_limit_enable,Bool}|Options]) when is_boolean(Bool) -> + do_check_opts(Options); +do_check_opts([{burst_limit_max_count,N}|Options]) when is_integer(N) -> + do_check_opts(Options); +do_check_opts([{burst_limit_window_time,N}|Options]) when is_integer(N) -> + do_check_opts(Options); +do_check_opts([{overload_kill_enable,Bool}|Options]) when is_boolean(Bool) -> + do_check_opts(Options); +do_check_opts([{overload_kill_qlen,N}|Options]) when is_integer(N) -> + do_check_opts(Options); +do_check_opts([{overload_kill_mem_size,N}|Options]) when is_integer(N) -> + do_check_opts(Options); +do_check_opts([{overload_kill_restart_after,NorA}|Options]) + when is_integer(NorA); NorA == infinity -> + do_check_opts(Options); +do_check_opts([{Key,Value}|_]) -> + {error,Key,Value}; +do_check_opts([]) -> + ok. + +set_restart_flag(#{id := Name, module := Module}) -> + Flag = list_to_atom(lists:concat([Module,"_",Name,"_restarting"])), + spawn(fun() -> + register(Flag, self()), + timer:sleep(infinity) + end), + ok. + +reset_restart_flag(#{id := Name, module := Module} = State) -> + Flag = list_to_atom(lists:concat([Module,"_",Name,"_restarting"])), + case whereis(Flag) of + undefined -> + State; + Pid -> + exit(Pid, kill), + notify(restart,State) + end. + +check_load(State = #{id:=_Name, mode_ref := ModeRef, mode := Mode, + sync_mode_qlen := SyncModeQLen, + drop_mode_qlen := DropModeQLen, + flush_qlen := FlushQLen}) -> + {_,Mem} = process_info(self(), memory), + ?observe(_Name,{max_mem,Mem}), + {_,QLen} = process_info(self(), message_queue_len), + ?observe(_Name,{max_qlen,QLen}), + %% When the handler process gets scheduled in, it's impossible + %% to predict the QLen. We could jump "up" arbitrarily from say + %% async to sync, async to drop, sync to flush, etc. However, when + %% the handler process manages the log events (without flushing), + %% one after the other, we will move "down" from drop to sync and + %% from sync to async. This way we don't risk getting stuck in + %% drop or sync mode with an empty mailbox. + {Mode1,_NewDrops,_NewFlushes} = + if + QLen >= FlushQLen -> + {flush, 0,1}; + QLen >= DropModeQLen -> + %% Note that drop mode will force load messages to + %% be dropped on the client side (never sent to + %% the olp process). + IncDrops = if Mode == drop -> 0; true -> 1 end, + {?change_mode(ModeRef, Mode, drop), IncDrops,0}; + QLen >= SyncModeQLen -> + {?change_mode(ModeRef, Mode, sync), 0,0}; + true -> + {?change_mode(ModeRef, Mode, async), 0,0} + end, + State1 = ?update_other(drops,DROPS,_NewDrops,State), + State2 = ?update_max_qlen(QLen,State1), + State3 = ?update_max_mem(Mem,State2), + State4 = maybe_notify_mode_change(Mode1,State3), + {Mode1, QLen, Mem, + ?update_other(flushes,FLUSHES,_NewFlushes, + State4#{last_qlen => QLen})}. + +limit_burst(#{burst_limit_enable := false}=State) -> + {true,State}; +limit_burst(#{burst_win_ts := BurstWinT0, + burst_msg_count := BurstMsgCount, + burst_limit_window_time := BurstLimitWinTime, + burst_limit_max_count := BurstLimitMaxCnt} = State) -> + if (BurstMsgCount >= BurstLimitMaxCnt) -> + %% the limit for allowed messages has been reached + BurstWinT1 = ?timestamp(), + case ?diff_time(BurstWinT1,BurstWinT0) of + BurstCheckTime when BurstCheckTime < (BurstLimitWinTime*1000) -> + %% we're still within the burst time frame + {false,?update_other(burst_drops,BURSTS,1,State)}; + _BurstCheckTime -> + %% burst time frame passed, reset counters + {true,State#{burst_win_ts => BurstWinT1, + burst_msg_count => 0}} + end; + true -> + %% the limit for allowed messages not yet reached + {true,State#{burst_win_ts => BurstWinT0, + burst_msg_count => BurstMsgCount+1}} + end. + +kill_if_choked(QLen, Mem, #{overload_kill_enable := KillIfOL, + overload_kill_qlen := OLKillQLen, + overload_kill_mem_size := OLKillMem}) -> + if KillIfOL andalso + ((QLen > OLKillQLen) orelse (Mem > OLKillMem)) -> + exit({shutdown,{overloaded,QLen,Mem}}); + true -> + ok + end. + +flush_load(Limit) -> + process_flag(priority, high), + Flushed = flush_load(0, Limit), + process_flag(priority, normal), + Flushed. + +flush_load(Limit, Limit) -> + Limit; +flush_load(N, Limit) -> + %% flush log events but leave other events, such as info, reset + %% and stop, so that these have a chance to be processed even + %% under heavy load + receive + {'$gen_cast',{'$olp_load',_}} -> + flush_load(N+1, Limit); + {'$gen_call',{Pid,MRef},{'$olp_load',_}} -> + Pid ! {MRef, dropped}, + flush_load(N+1, Limit); + {log,_,_,_,_} -> + flush_load(N+1, Limit); + {log,_,_,_} -> + flush_load(N+1, Limit) + after + 0 -> N + end. + +overload_levels_ok(Options) -> + SMQL = maps:get(sync_mode_qlen, Options, ?SYNC_MODE_QLEN), + DMQL = maps:get(drop_mode_qlen, Options, ?DROP_MODE_QLEN), + FQL = maps:get(flush_qlen, Options, ?FLUSH_QLEN), + (DMQL > 1) andalso (SMQL =< DMQL) andalso (DMQL =< FQL). + +maybe_notify_mode_change(drop,#{mode:=Mode0}=State) + when Mode0=/=drop -> + notify({mode_change,Mode0,drop},State); +maybe_notify_mode_change(Mode1,#{mode:=drop}=State) + when Mode1==async; Mode1==sync -> + notify({mode_change,drop,Mode1},State); +maybe_notify_mode_change(_,State) -> + State. + +notify(Note,#{module:=Module,cb_state:=CBState}=State) -> + CBState1 = try_callback_call(Module,notify,[Note,CBState],CBState), + State#{cb_state=>CBState1}. + +try_callback_call(Module, Function, Args) -> + try_callback_call(Module, Function, Args, '$no_default_return'). + +try_callback_call(Module, Function, Args, DefRet) -> + try apply(Module, Function, Args) + catch + throw:R -> R; + error:undef:S when DefRet=/='$no_default_return' -> + case S of + [{Module,Function,Args,_}|_] -> + DefRet; + _ -> + erlang:raise(error,undef,S) + end + end. + +noreply_return(#{idle:=true}=State) -> + {noreply,State}; +noreply_return(#{idle:=false}=State) -> + {noreply,State,?IDLE_DETECT_TIME}. + +reply_return(Reply,#{idle:=true}=State) -> + {reply,Reply,State}; +reply_return(Reply,#{idle:=false}=State) -> + {reply,Reply,State,?IDLE_DETECT_TIME}. diff --git a/lib/kernel/src/logger_olp.hrl b/lib/kernel/src/logger_olp.hrl new file mode 100644 index 0000000000..d68b5c048d --- /dev/null +++ b/lib/kernel/src/logger_olp.hrl @@ -0,0 +1,185 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1997-2015. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%%%----------------------------------------------------------------- +%%% Overload protection configuration + +%%! *** NOTE *** +%%! It's important that: +%%! SYNC_MODE_QLEN =< DROP_MODE_QLEN =< FLUSH_QLEN +%%! and that DROP_MODE_QLEN >= 2. +%%! Otherwise the process could end up in drop mode with no new +%%! log requests to process. This would cause all future requests +%%! to be dropped (no switch to async mode would ever take place). + +%% This specifies the message_queue_len value where the log +%% requests switch from asynchronous casts to synchronous calls. +-define(SYNC_MODE_QLEN, 10). +%% Above this message_queue_len, log requests will be dropped, +%% i.e. no log requests get sent to the process. +-define(DROP_MODE_QLEN, 200). +%% Above this message_queue_len, the process will flush its mailbox +%% and only leave this number of messages in it. +-define(FLUSH_QLEN, 1000). + +%% Never flush more than this number of messages in one go, or the +%% process will be unresponsive for seconds (keep this number as large +%% as possible or the mailbox could grow large). +-define(FLUSH_MAX_N, 5000). + +%% BURST_LIMIT_MAX_COUNT is the max number of log requests allowed +%% to be written within a BURST_LIMIT_WINDOW_TIME time frame. +-define(BURST_LIMIT_ENABLE, true). +-define(BURST_LIMIT_MAX_COUNT, 500). +-define(BURST_LIMIT_WINDOW_TIME, 1000). + +%% This enables/disables the feature to automatically terminate the +%% process if it gets too loaded (and can't keep up). +-define(OVERLOAD_KILL_ENABLE, false). +%% If the message_queue_len goes above this size even after +%% flushing has been performed, the process is terminated. +-define(OVERLOAD_KILL_QLEN, 20000). +%% If the memory usage exceeds this level, the process is terminated. +-define(OVERLOAD_KILL_MEM_SIZE, 3000000). + +%% This is the default time to wait before restarting and accepting +%% new requests. The value 'infinity' disables restarts. +-define(OVERLOAD_KILL_RESTART_AFTER, 5000). + +%% This is the time in milliseconds after last load message received +%% that we notify the callback about being idle. +-define(IDLE_DETECT_TIME, 100). + +%%%----------------------------------------------------------------- +%%% Overload protection macros + +-define(timestamp(), erlang:monotonic_time(microsecond)). + +-define(get_mode(Tid), + case ets:lookup(Tid, mode) of + [{mode,M}] -> M; + _ -> async + end). + +-define(set_mode(Tid, M), + begin ets:insert(Tid, {mode,M}), M end). + +-define(change_mode(Tid, M0, M1), + if M0 == M1 -> + M0; + true -> + ets:insert(Tid, {mode,M1}), + M1 + end). + +-define(max(X1, X2), + if + X2 == undefined -> X1; + X2 > X1 -> X2; + true -> X1 + end). + +-define(diff_time(OS_T1, OS_T0), OS_T1-OS_T0). + +%%%----------------------------------------------------------------- +%%% These macros enable statistics counters in the state of the +%%% process, which is useful for analysing the overload protection +%%% behaviour. These counters should not be included in code to be +%%% officially released (as some counters will grow very large over +%%% time). + +%% -define(SAVE_STATS, true). +-ifdef(SAVE_STATS). + -define(merge_with_stats(STATE), + begin + TIME = ?timestamp(), + STATE#{start => TIME, time => {TIME,0}, + flushes => 0, flushed => 0, drops => 0, + burst_drops => 0, casts => 0, calls => 0, + writes => 0, max_qlen => 0, max_time => 0, + max_mem => 0, freq => {TIME,0,0}} end). + + -define(update_max_qlen(QLEN, STATE), + begin #{max_qlen := QLEN0} = STATE, + STATE#{max_qlen => ?max(QLEN0,QLEN)} end). + + -define(update_max_mem(MEM, STATE), + begin #{max_mem := MEM0} = STATE, + STATE#{max_mem => ?max(MEM0,MEM)} end). + + -define(update_calls_or_casts(CALL_OR_CAST, INC, STATE), + case CALL_OR_CAST of + cast -> + #{casts := CASTS0} = STATE, + STATE#{casts => CASTS0+INC}; + call -> + #{calls := CALLS0} = STATE, + STATE#{calls => CALLS0+INC} + end). + + -define(update_max_time(TIME, STATE), + begin #{max_time := TIME0} = STATE, + STATE#{max_time => ?max(TIME0,TIME)} end). + + -define(update_other(OTHER, VAR, INCVAL, STATE), + begin #{OTHER := VAR} = STATE, + STATE#{OTHER => VAR+INCVAL} end). + + -define(update_freq(TIME,STATE), + begin + case STATE of + #{freq := {START, 49, _}} -> + STATE#{freq => {TIME, 0, trunc(1000000*50/(?diff_time(TIME,START)))}}; + #{freq := {START, N, FREQ}} -> + STATE#{freq => {START, N+1, FREQ}} + end end). + + -define(update_time(TIME,STATE), + begin #{start := START} = STATE, + STATE#{time => {TIME,trunc((?diff_time(TIME,START))/1000000)}} end). + +-else. % DEFAULT! + -define(merge_with_stats(STATE), STATE). + -define(update_max_qlen(_QLEN, STATE), STATE). + -define(update_max_mem(_MEM, STATE), STATE). + -define(update_calls_or_casts(_CALL_OR_CAST, _INC, STATE), STATE). + -define(update_max_time(_TIME, STATE), STATE). + -define(update_other(_OTHER, _VAR, _INCVAL, STATE), STATE). + -define(update_freq(_TIME, STATE), STATE). + -define(update_time(_TIME, STATE), STATE). +-endif. + +%%%----------------------------------------------------------------- +%%% These macros enable callbacks that make it possible to analyse the +%%% overload protection behaviour from outside the process (including +%%% dropped requests on the client side). An external callback module +%%% (?OBSERVER_MOD) is required which is not part of the kernel +%%% application. For this reason, these callbacks should not be +%%% included in code to be officially released. + +%%-define(OBSERVER_MOD, logger_test). +-ifdef(OBSERVER_MOD). + -define(start_observation(NAME), ?OBSERVER:start_observation(NAME)). + -define(observe(NAME,EVENT), ?OBSERVER:observe(NAME,EVENT)). + +-else. % DEFAULT! + -define(start_observation(_NAME), ok). + -define(observe(_NAME,_EVENT), ok). +-endif. diff --git a/lib/kernel/src/logger_proxy.erl b/lib/kernel/src/logger_proxy.erl new file mode 100644 index 0000000000..24b293805c --- /dev/null +++ b/lib/kernel/src/logger_proxy.erl @@ -0,0 +1,165 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(logger_proxy). + +%% API +-export([start_link/0, restart/0, log/1, child_spec/0, get_default_config/0]). + +%% logger_olp callbacks +-export([init/1, handle_load/2, handle_info/2, terminate/2, + notify/2]). + +-include("logger_internal.hrl"). + +-define(SERVER,?MODULE). + +%%%----------------------------------------------------------------- +%%% API +-spec log(RemoteLog) -> ok when + RemoteLog :: {remote,node(),LogEvent}, + LogEvent :: {log,Level,Format,Args,Meta} | + {log,Level,StringOrReport,Meta}, + Level :: logger:level(), + Format :: io:format(), + Args :: list(term()), + StringOrReport :: unicode:chardata() | logger:report(), + Meta :: logger:metadata(). +log(RemoteLog) -> + Olp = persistent_term:get(?MODULE), + case logger_olp:get_pid(Olp) =:= self() of + true -> + %% This happens when the log event comes from the + %% emulator, and the group leader is on a remote node. + _ = handle_load(RemoteLog, no_state), + ok; + false -> + logger_olp:load(Olp, RemoteLog) + end. + +%% Called by supervisor +-spec start_link() -> {ok,pid(),logger_olp:olp_ref()} | {error,term()}. +start_link() -> + %% Notice that sync_mode is only used when logging to remote node, + %% i.e. when the log/2 API function is called. + %% + %% When receiving log events from the emulator or from a remote + %% node, the log event is sent as a message to this process, and + %% thus received directly in handle_info/2. This means that the + %% mode (async/sync/drop) is not read before the message is + %% sent. Thus sync mode is never entered, and drop mode is + %% implemented by setting the system_logger flag to undefined (see + %% notify/2) + %% + %% Burst limit is disabled, since this is only a proxy and we + %% don't want to limit bursts twice (here and in the handler). + logger_olp:start_link(?SERVER,?MODULE,[],logger:get_proxy_config()). + +%% Fun used for restarting this process after it has been killed due +%% to overload (must set overload_kill_enable=>true in opts) +restart() -> + case supervisor:start_child(logger_sup, child_spec()) of + {ok,_Pid,Olp} -> + {ok,Olp}; + {error,{Reason,Ch}} when is_tuple(Ch), element(1,Ch)==child -> + {error,Reason}; + Error -> + Error + end. + +%% Called internally and by logger_sup +child_spec() -> + Name = ?SERVER, + #{id => Name, + start => {?MODULE, start_link, []}, + restart => temporary, + shutdown => 2000, + type => worker, + modules => [?MODULE]}. + +get_default_config() -> + OlpDefault = logger_olp:get_default_opts(), + OlpDefault#{sync_mode_qlen=>500, + drop_mode_qlen=>1000, + flush_qlen=>5000, + burst_limit_enable=>false}. + +%%%=================================================================== +%%% gen_server callbacks +%%%=================================================================== + +init([]) -> + process_flag(trap_exit, true), + _ = erlang:system_flag(system_logger,self()), + persistent_term:put(?MODULE,logger_olp:get_ref()), + {ok,no_state}. + +%% Log event to send to the node where the group leader of it's client resides +handle_load({remote,Node,Log},State) -> + %% If the connection is overloaded (send_nosuspend returns false), + %% we drop the message. + _ = erlang:send_nosuspend({?SERVER,Node},Log), + State; +%% Log event to log on this node +handle_load({log,Level,Format,Args,Meta},State) -> + try_log([Level,Format,Args,Meta]), + State; +handle_load({log,Level,Report,Meta},State) -> + try_log([Level,Report,Meta]), + State. + +%% Log event sent to this process e.g. from the emulator - it is really load +handle_info(Log,State) when is_tuple(Log), element(1,Log)==log -> + {load,State}. + +terminate(overloaded, _State) -> + _ = erlang:system_flag(system_logger,undefined), + {ok,fun ?MODULE:restart/0}; +terminate(_Reason, _State) -> + _ = erlang:system_flag(system_logger,whereis(logger)), + ok. + +notify({mode_change,Mode0,Mode1},State) -> + _ = if Mode1=:=drop -> % entering drop mode + erlang:system_flag(system_logger,undefined); + Mode0=:=drop -> % leaving drop mode + erlang:system_flag(system_logger,self()); + true -> + ok + end, + ?LOG_INTERNAL(notice,"~w switched from ~w to ~w mode",[?MODULE,Mode0,Mode1]), + State; +notify({flushed,Flushed},State) -> + ?LOG_INTERNAL(notice, "~w flushed ~w log events",[?MODULE,Flushed]), + State; +notify(restart,State) -> + ?LOG_INTERNAL(notice, "~w restarted", [?MODULE]), + State; +notify(_Note,State) -> + State. + +%%%----------------------------------------------------------------- +%%% Internal functions +try_log(Args) -> + try apply(logger,log,Args) + catch C:R:S -> + ?LOG_INTERNAL(debug,[{?MODULE,log_failed}, + {log,Args}, + {reason,{C,R,S}}]) + end. diff --git a/lib/kernel/src/logger_server.erl b/lib/kernel/src/logger_server.erl index b7735dbcf7..722246e82c 100644 --- a/lib/kernel/src/logger_server.erl +++ b/lib/kernel/src/logger_server.erl @@ -22,8 +22,7 @@ -behaviour(gen_server). %% API --export([start_link/0, - add_handler/3, remove_handler/1, +-export([start_link/0, add_handler/3, remove_handler/1, add_filter/2, remove_filter/2, set_module_level/2, unset_module_level/0, unset_module_level/1, cache_module_level/1, @@ -43,7 +42,7 @@ -define(SERVER, logger). -define(LOGGER_SERVER_TAG, '$logger_cb_process'). --record(state, {tid, async_req, async_req_queue}). +-record(state, {tid, async_req, async_req_queue, remote_logger}). %%%=================================================================== %%% API @@ -155,6 +154,8 @@ init([]) -> process_flag(trap_exit, true), put(?LOGGER_SERVER_TAG,true), Tid = logger_config:new(?LOGGER_TABLE), + %% Store initial proxy config. logger_proxy reads config from here at startup. + logger_config:create(Tid,proxy,logger_proxy:get_default_config()), PrimaryConfig = maps:merge(default_config(primary), #{handlers=>[simple]}), logger_config:create(Tid,primary,PrimaryConfig), @@ -221,6 +222,24 @@ handle_call({add_filter,Id,Filter}, _From,#state{tid=Tid}=State) -> handle_call({remove_filter,Id,FilterId}, _From, #state{tid=Tid}=State) -> Reply = do_remove_filter(Tid,Id,FilterId), {reply,Reply,State}; +handle_call({change_config,SetOrUpd,proxy,Config0},_From,#state{tid=Tid}=State) -> + Default = + case SetOrUpd of + set -> + logger_proxy:get_default_config(); + update -> + {ok,OldConfig} = logger_config:get(Tid,proxy), + OldConfig + end, + Config = maps:merge(Default,Config0), + Reply = + case logger_olp:set_opts(logger_proxy,Config) of + ok -> + logger_config:set(Tid,proxy,Config); + Error -> + Error + end, + {reply,Reply,State}; handle_call({change_config,SetOrUpd,primary,Config0}, _From, #state{tid=Tid}=State) -> {ok,#{handlers:=Handlers}=OldConfig} = logger_config:get(Tid,primary), @@ -357,7 +376,7 @@ terminate(_Reason, _State) -> %%%=================================================================== %%% Internal functions %%%=================================================================== -call(Request) -> +call(Request) when is_tuple(Request) -> Action = element(1,Request), case get(?LOGGER_SERVER_TAG) of true when @@ -369,6 +388,7 @@ call(Request) -> gen_server:call(?SERVER,Request,?DEFAULT_LOGGER_CALL_TIMEOUT) end. + do_add_filter(Tid,Id,{FId,_} = Filter) -> case logger_config:get(Tid,Id) of {ok,Config} -> @@ -413,11 +433,13 @@ default_config(Id,Module) -> sanity_check(Owner,Key,Value) -> sanity_check_1(Owner,[{Key,Value}]). -sanity_check(HandlerId,Config) when is_map(Config) -> - sanity_check_1(HandlerId,maps:to_list(Config)); +sanity_check(Owner,Config) when is_map(Config) -> + sanity_check_1(Owner,maps:to_list(Config)); sanity_check(_,Config) -> {error,{invalid_config,Config}}. +sanity_check_1(proxy,_Config) -> + ok; % Details are checked by logger_olp:set_opts/2 sanity_check_1(Owner,Config) when is_list(Config) -> try Type = get_type(Owner), diff --git a/lib/kernel/src/logger_simple_h.erl b/lib/kernel/src/logger_simple_h.erl index fe181722f3..a0d51dba25 100644 --- a/lib/kernel/src/logger_simple_h.erl +++ b/lib/kernel/src/logger_simple_h.erl @@ -69,7 +69,7 @@ log(#{msg:=_,meta:=#{time:=_}}=Log,_Config) -> do_log( #{level=>error, msg=>{report,{error,simple_handler_process_dead}}, - meta=>#{time=>erlang:system_time(microsecond)}}), + meta=>#{time=>logger:timestamp()}}), do_log(Log); _ -> ?MODULE ! {log,Log} @@ -129,7 +129,7 @@ drop_msg(0) -> drop_msg(N) -> [#{level=>info, msg=>{"Simple handler buffer full, dropped ~w messages",[N]}, - meta=>#{time=>erlang:system_time(microsecond)}}]. + meta=>#{time=>logger:timestamp()}}]. %%%----------------------------------------------------------------- %%% Internal diff --git a/lib/kernel/src/logger_std_h.erl b/lib/kernel/src/logger_std_h.erl index 63d1dbaba2..c8f1acfca4 100644 --- a/lib/kernel/src/logger_std_h.erl +++ b/lib/kernel/src/logger_std_h.erl @@ -26,16 +26,18 @@ -include_lib("kernel/include/file.hrl"). %% API --export([info/1, filesync/1, reset/1]). +-export([filesync/1]). %% logger_h_common callbacks --export([init/2, check_config/4, reset_state/2, +-export([init/2, check_config/4, config_changed/3, reset_state/2, filesync/3, write/4, handle_info/3, terminate/3]). %% logger callbacks -export([log/2, adding_handler/1, removing_handler/1, changing_config/3, filter_config/1]). +-define(DEFAULT_CALL_TIMEOUT, 5000). + %%%=================================================================== %%% API %%%=================================================================== @@ -49,25 +51,6 @@ filesync(Name) -> logger_h_common:filesync(?MODULE,Name). -%%%----------------------------------------------------------------- -%%% --spec info(Name) -> Info | {error,Reason} when - Name :: atom(), - Info :: term(), - Reason :: handler_busy | {badarg,term()}. - -info(Name) -> - logger_h_common:info(?MODULE,Name). - -%%%----------------------------------------------------------------- -%%% --spec reset(Name) -> ok | {error,Reason} when - Name :: atom(), - Reason :: handler_busy | {badarg,term()}. - -reset(Name) -> - logger_h_common:reset(?MODULE,Name). - %%%=================================================================== %%% logger callbacks - just forward to logger_h_common %%%=================================================================== @@ -122,85 +105,169 @@ filter_config(Config) -> %%%=================================================================== %%% logger_h_common callbacks %%%=================================================================== -init(Name, #{type := Type}) -> - case open_log_file(Name, Type) of +init(Name, Config) -> + MyConfig = maps:with([type,file,modes,file_check,max_no_bytes, + max_no_files,compress_on_rotate],Config), + case file_ctrl_start(Name, MyConfig) of {ok,FileCtrlPid} -> - {ok,#{type=>Type,file_ctrl_pid=>FileCtrlPid}}; + {ok,MyConfig#{file_ctrl_pid=>FileCtrlPid}}; Error -> Error end. -check_config(_Name,set,undefined,NewHConfig) -> - check_config(maps:merge(get_default_config(),NewHConfig)); -check_config(_Name,SetOrUpdate,OldHConfig,NewHConfig0) -> - WriteOnce = maps:with([type],OldHConfig), +check_config(Name,set,undefined,NewHConfig) -> + check_h_config(merge_default_config(Name,normalize_config(NewHConfig))); +check_config(Name,SetOrUpdate,OldHConfig,NewHConfig0) -> + WriteOnce = maps:with([type,file,modes],OldHConfig), Default = case SetOrUpdate of set -> %% Do not reset write-once fields to defaults - maps:merge(get_default_config(),WriteOnce); + merge_default_config(Name,WriteOnce); update -> OldHConfig end, - NewHConfig = maps:merge(Default, NewHConfig0), + NewHConfig = maps:merge(Default, normalize_config(NewHConfig0)), %% Fail if write-once fields are changed - case maps:with([type],NewHConfig) of + case maps:with([type,file,modes],NewHConfig) of WriteOnce -> - check_config(NewHConfig); + check_h_config(NewHConfig); Other -> {error,{illegal_config_change,?MODULE,WriteOnce,Other}} end. -check_config(#{type:=Type}=HConfig) -> - case check_h_config(maps:to_list(HConfig)) of - ok when is_atom(Type) -> - {ok,HConfig#{filesync_repeat_interval=>no_repeat}}; +check_h_config(HConfig) -> + case check_h_config(maps:get(type,HConfig),maps:to_list(HConfig)) of ok -> - {ok,HConfig}; + {ok,fix_file_opts(HConfig)}; {error,{Key,Value}} -> {error,{invalid_config,?MODULE,#{Key=>Value}}} end. -check_h_config([{type,Type} | Config]) when Type == standard_io; - Type == standard_error -> - check_h_config(Config); -check_h_config([{type,{file,File}} | Config]) when is_list(File) -> - check_h_config(Config); -check_h_config([{type,{file,File,Modes}} | Config]) when is_list(File), - is_list(Modes) -> - check_h_config(Config); -check_h_config([Other | _]) -> +check_h_config(Type,[{type,Type} | Config]) when Type =:= standard_io; + Type =:= standard_error; + Type =:= file -> + check_h_config(Type,Config); +check_h_config(file,[{file,File} | Config]) when is_list(File) -> + check_h_config(file,Config); +check_h_config(file,[{modes,Modes} | Config]) when is_list(Modes) -> + check_h_config(file,Config); +check_h_config(file,[{max_no_bytes,Size} | Config]) + when (is_integer(Size) andalso Size>0) orelse Size=:=infinity -> + check_h_config(file,Config); +check_h_config(file,[{max_no_files,Num} | Config]) when is_integer(Num), Num>=0 -> + check_h_config(file,Config); +check_h_config(file,[{compress_on_rotate,Bool} | Config]) when is_boolean(Bool) -> + check_h_config(file,Config); +check_h_config(file,[{file_check,FileCheck} | Config]) + when is_integer(FileCheck), FileCheck>=0 -> + check_h_config(file,Config); +check_h_config(_Type,[Other | _]) -> {error,Other}; -check_h_config([]) -> +check_h_config(_Type,[]) -> ok. -get_default_config() -> - #{type => standard_io}. +normalize_config(#{type:={file,File}}=HConfig) -> + HConfig#{type=>file,file=>File}; +normalize_config(#{type:={file,File,Modes}}=HConfig) -> + HConfig#{type=>file,file=>File,modes=>Modes}; +normalize_config(HConfig) -> + HConfig. + +merge_default_config(Name,#{type:=Type}=HConfig) -> + merge_default_config(Name,Type,HConfig); +merge_default_config(Name,#{file:=_}=HConfig) -> + merge_default_config(Name,file,HConfig); +merge_default_config(Name,HConfig) -> + merge_default_config(Name,standard_io,HConfig). + +merge_default_config(Name,Type,HConfig) -> + maps:merge(get_default_config(Name,Type),HConfig). + +get_default_config(Name,file) -> + #{type => file, + file => atom_to_list(Name), + modes => [raw,append], + file_check => 0, + max_no_bytes => infinity, + max_no_files => 0, + compress_on_rotate => false}; +get_default_config(_Name,Type) -> + #{type => Type}. + +fix_file_opts(#{modes:=Modes}=HConfig) -> + HConfig#{modes=>fix_modes(Modes)}; +fix_file_opts(HConfig) -> + HConfig#{filesync_repeat_interval=>no_repeat}. + +fix_modes(Modes) -> + %% Ensure write|append|exclusive + Modes1 = + case [M || M <- Modes, + lists:member(M,[write,append,exclusive])] of + [] -> [append|Modes]; + _ -> Modes + end, + %% Ensure raw + Modes2 = + case lists:member(raw,Modes) of + false -> [raw|Modes1]; + true -> Modes1 + end, + %% Ensure delayed_write + case lists:partition(fun(delayed_write) -> true; + ({delayed_write,_,_}) -> true; + (_) -> false + end, Modes2) of + {[],_} -> + [delayed_write|Modes2]; + _ -> + Modes2 + end. + +config_changed(_Name, + #{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}, + #{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}=State) -> + State; +config_changed(_Name, + #{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}, + #{file_ctrl_pid := FileCtrlPid} = State) -> + FileCtrlPid ! {update_config,#{file_check=>FileCheck, + max_no_bytes=>Size, + max_no_files=>Count, + compress_on_rotate=>Compress}}, + State#{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}; +config_changed(_Name,_NewHConfig,State) -> + State. -filesync(_Name, _Mode, #{type := Type}=State) when is_atom(Type) -> - {ok,State}; -filesync(_Name, async, #{file_ctrl_pid := FileCtrlPid} = State) -> - ok = file_ctrl_filesync_async(FileCtrlPid), - {ok,State}; -filesync(_Name, sync, #{file_ctrl_pid := FileCtrlPid} = State) -> - Result = file_ctrl_filesync_sync(FileCtrlPid), +filesync(_Name, SyncAsync, #{file_ctrl_pid := FileCtrlPid} = State) -> + Result = file_ctrl_filesync(SyncAsync, FileCtrlPid), {Result,State}. -write(_Name, async, Bin, #{file_ctrl_pid:=FileCtrlPid} = State) -> - ok = file_write_async(FileCtrlPid, Bin), - {ok,State}; -write(_Name, sync, Bin, #{file_ctrl_pid:=FileCtrlPid} = State) -> - Result = file_write_sync(FileCtrlPid, Bin), +write(_Name, SyncAsync, Bin, #{file_ctrl_pid:=FileCtrlPid} = State) -> + Result = file_write(SyncAsync, FileCtrlPid, Bin), {Result,State}. reset_state(_Name, State) -> State. -handle_info(_Name, {'EXIT',Pid,Why}, #{type := FileInfo, file_ctrl_pid := Pid}) -> +handle_info(_Name, {'EXIT',Pid,Why}, #{file_ctrl_pid := Pid}=State) -> %% file_ctrl_pid died, file error, terminate handler - exit({error,{write_failed,FileInfo,Why}}); + exit({error,{write_failed,maps:with([type,file,modes],State),Why}}); handle_info(_, _, State) -> State. @@ -228,23 +295,36 @@ terminate(_Name, _Reason, #{file_ctrl_pid:=FWPid}) -> %%%----------------------------------------------------------------- %%% -open_log_file(HandlerName, FileInfo) -> - case file_ctrl_start(HandlerName, FileInfo) of - OK = {ok,_FileCtrlPid} -> OK; - Error -> Error - end. - -do_open_log_file({file,File}) -> - do_open_log_file({file,File,[raw,append,delayed_write]}); - -do_open_log_file({file,File,[]}) -> - do_open_log_file({file,File,[raw,append,delayed_write]}); - -do_open_log_file({file,File,Modes}) -> +open_log_file(HandlerName,#{type:=file, + file:=FileName, + modes:=Modes, + file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}) -> try - case filelib:ensure_dir(File) of + case filelib:ensure_dir(FileName) of ok -> - file:open(File, Modes); + case file:open(FileName, Modes) of + {ok, Fd} -> + {ok,#file_info{inode=INode}} = + file:read_file_info(FileName,[raw]), + UpdateModes = [append | Modes--[write,append,exclusive]], + State0 = #{handler_name=>HandlerName, + file_name=>FileName, + modes=>UpdateModes, + file_check=>FileCheck, + fd=>Fd, + inode=>INode, + last_check=>timestamp(), + synced=>false, + write_res=>ok, + sync_res=>ok}, + State = update_rotation({Size,Count,Compress},State0), + {ok,State}; + Error -> + Error + end; Error -> Error end @@ -252,21 +332,23 @@ do_open_log_file({file,File,Modes}) -> _:Reason -> {error,Reason} end. -close_log_file(Std) when Std == standard_io; Std == standard_error -> - ok; -close_log_file(Fd) -> +close_log_file(#{fd:=Fd}) -> _ = file:datasync(Fd), - _ = file:close(Fd). + _ = file:close(Fd), + ok; +close_log_file(_) -> + ok. + %%%----------------------------------------------------------------- %%% File control process -file_ctrl_start(HandlerName, FileInfo) -> +file_ctrl_start(HandlerName, HConfig) -> Starter = self(), FileCtrlPid = spawn_link(fun() -> - file_ctrl_init(HandlerName, FileInfo, Starter) + file_ctrl_init(HandlerName, HConfig, Starter) end), receive {FileCtrlPid,ok} -> @@ -281,18 +363,16 @@ file_ctrl_start(HandlerName, FileInfo) -> file_ctrl_stop(Pid) -> Pid ! stop. -file_write_async(Pid, Bin) -> +file_write(async, Pid, Bin) -> Pid ! {log,Bin}, - ok. - -file_write_sync(Pid, Bin) -> + ok; +file_write(sync, Pid, Bin) -> file_ctrl_call(Pid, {log,Bin}). -file_ctrl_filesync_async(Pid) -> +file_ctrl_filesync(async, Pid) -> Pid ! filesync, - ok. - -file_ctrl_filesync_sync(Pid) -> + ok; +file_ctrl_filesync(sync, Pid) -> file_ctrl_call(Pid, filesync). file_ctrl_call(Pid, Msg) -> @@ -309,94 +389,255 @@ file_ctrl_call(Pid, Msg) -> {error,{no_response,Pid}} end. -file_ctrl_init(HandlerName, FileInfo, Starter) when is_tuple(FileInfo) -> +file_ctrl_init(HandlerName, + #{type:=file, + file:=FileName} = HConfig, + Starter) -> process_flag(message_queue_data, off_heap), - FileName = element(2, FileInfo), - case do_open_log_file(FileInfo) of - {ok,Fd} -> + case open_log_file(HandlerName,HConfig) of + {ok,State} -> Starter ! {self(),ok}, - file_ctrl_loop(Fd, FileName, false, ok, ok, HandlerName); + file_ctrl_loop(State); {error,Reason} -> Starter ! {self(),{error,{open_failed,FileName,Reason}}} end; -file_ctrl_init(HandlerName, StdDev, Starter) -> +file_ctrl_init(HandlerName, #{type:=StdDev}, Starter) -> Starter ! {self(),ok}, - file_ctrl_loop(StdDev, StdDev, false, ok, ok, HandlerName). + file_ctrl_loop(#{handler_name=>HandlerName,dev=>StdDev}). -file_ctrl_loop(Fd, DevName, Synced, - PrevWriteResult, PrevSyncResult, HandlerName) -> +file_ctrl_loop(State) -> receive %% asynchronous event {log,Bin} -> - Fd1 = ensure(Fd, DevName), - Result = write_to_dev(Fd1, Bin, DevName, PrevWriteResult, HandlerName), - file_ctrl_loop(Fd1, DevName, false, - Result, PrevSyncResult, HandlerName); + State1 = write_to_dev(Bin,State), + file_ctrl_loop(State1); %% synchronous event {{log,Bin},{From,MRef}} -> - Fd1 = ensure(Fd, DevName), - Result = write_to_dev(Fd1, Bin, DevName, PrevWriteResult, HandlerName), + State1 = ensure_file(State), + State2 = write_to_dev(Bin,State1), From ! {MRef,ok}, - file_ctrl_loop(Fd1, DevName, false, - Result, PrevSyncResult, HandlerName); + file_ctrl_loop(State2); filesync -> - Fd1 = ensure(Fd, DevName), - Result = sync_dev(Fd1, DevName, Synced, PrevSyncResult, HandlerName), - file_ctrl_loop(Fd1, DevName, true, - PrevWriteResult, Result, HandlerName); + State1 = sync_dev(State), + file_ctrl_loop(State1); {filesync,{From,MRef}} -> - Fd1 = ensure(Fd, DevName), - Result = sync_dev(Fd1, DevName, Synced, PrevSyncResult, HandlerName), + State1 = ensure_file(State), + State2 = sync_dev(State1), From ! {MRef,ok}, - file_ctrl_loop(Fd1, DevName, true, - PrevWriteResult, Result, HandlerName); + file_ctrl_loop(State2); + + {update_config,#{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}} -> + State1 = update_rotation({Size,Count,Compress},State), + file_ctrl_loop(State1#{file_check=>FileCheck}); stop -> - _ = close_log_file(Fd), + close_log_file(State), stopped end. +maybe_ensure_file(#{file_check:=0}=State) -> + ensure_file(State); +maybe_ensure_file(#{last_check:=T0,file_check:=CheckInt}=State) + when is_integer(CheckInt) -> + T = timestamp(), + if T-T0 > CheckInt -> ensure_file(State); + true -> State + end; +maybe_ensure_file(State) -> + State. + %% In order to play well with tools like logrotate, we need to be able %% to re-create the file if it has disappeared (e.g. if rotated by %% logrotate) -ensure(Fd,DevName) when is_atom(DevName) -> - Fd; -ensure(Fd,FileName) -> - case file:read_file_info(FileName) of - {ok,_} -> - Fd; +ensure_file(#{fd:=Fd0,inode:=INode0,file_name:=FileName,modes:=Modes}=State) -> + case file:read_file_info(FileName,[raw]) of + {ok,#file_info{inode=INode0}} -> + State#{last_check=>timestamp()}; _ -> - _ = file:close(Fd), - _ = file:close(Fd), % delayed_write cause close not to close - case do_open_log_file({file,FileName}) of - {ok,Fd1} -> - Fd1; + close_log_file(Fd0), + case file:open(FileName,Modes) of + {ok,Fd} -> + {ok,#file_info{inode=INode}} = + file:read_file_info(FileName,[raw]), + State#{fd=>Fd,inode=>INode, + last_check=>timestamp(), + synced=>true,sync_res=>ok}; Error -> exit({could_not_reopen_file,Error}) end - end. + end; +ensure_file(State) -> + State. -write_to_dev(DevName, Bin, _DevName, _PrevWriteResult, _HandlerName) - when is_atom(DevName) -> - io:put_chars(DevName, Bin); -write_to_dev(Fd, Bin, FileName, PrevWriteResult, HandlerName) -> +write_to_dev(Bin,#{dev:=DevName}=State) -> + io:put_chars(DevName, Bin), + State; +write_to_dev(Bin, State) -> + State1 = #{fd:=Fd} = maybe_ensure_file(State), Result = ?file_write(Fd, Bin), - maybe_notify_error(write,Result,PrevWriteResult,FileName,HandlerName). + State2 = maybe_rotate_file(Bin,State1), + maybe_notify_error(write,Result,State2), + State2#{synced=>false,write_res=>Result}. -sync_dev(_Fd, _FileName, true, PrevSyncResult, _HandlerName) -> - PrevSyncResult; -sync_dev(Fd, FileName, false, PrevSyncResult, HandlerName) -> +sync_dev(#{synced:=false}=State) -> + State1 = #{fd:=Fd} = maybe_ensure_file(State), Result = ?file_datasync(Fd), - maybe_notify_error(filesync,Result,PrevSyncResult,FileName,HandlerName). + maybe_notify_error(filesync,Result,State1), + State1#{synced=>true,sync_res=>Result}; +sync_dev(State) -> + State. + +update_rotation({infinity,_,_},State) -> + maybe_remove_archives(0,State), + maps:remove(rotation,State); +update_rotation({Size,Count,Compress},#{file_name:=FileName} = State) -> + maybe_remove_archives(Count,State), + {ok,#file_info{size=CurrSize}} = file:read_file_info(FileName,[raw]), + State1 = State#{rotation=>#{size=>Size, + count=>Count, + compress=>Compress, + curr_size=>CurrSize}}, + maybe_update_compress(0,State1), + maybe_rotate_file(0,State1). + +maybe_remove_archives(Count,#{file_name:=FileName}=State) -> + Archive = rot_file_name(FileName,Count,false), + CompressedArchive = rot_file_name(FileName,Count,true), + case {file:read_file_info(Archive,[raw]), + file:read_file_info(CompressedArchive,[raw])} of + {{error,enoent},{error,enoent}} -> + ok; + _ -> + _ = file:delete(Archive), + _ = file:delete(CompressedArchive), + maybe_remove_archives(Count+1,State) + end. + +maybe_update_compress(Count,#{rotation:=#{count:=Count}}) -> + ok; +maybe_update_compress(N,#{file_name:=FileName, + rotation:=#{compress:=Compress}}=State) -> + Archive = rot_file_name(FileName,N,not Compress), + case file:read_file_info(Archive,[raw]) of + {ok,_} when Compress -> + compress_file(Archive); + {ok,_} -> + decompress_file(Archive); + _ -> + ok + end, + maybe_update_compress(N+1,State). + +maybe_rotate_file(Bin,#{rotation:=_}=State) when is_binary(Bin) -> + maybe_rotate_file(byte_size(Bin),State); +maybe_rotate_file(AddSize,#{rotation:=#{size:=RotSize, + curr_size:=CurrSize}=Rotation}=State) -> + NewSize = CurrSize + AddSize, + if NewSize>RotSize -> + rotate_file(State#{rotation=>Rotation#{curr_size=>NewSize}}); + true -> + State#{rotation=>Rotation#{curr_size=>NewSize}} + end; +maybe_rotate_file(_Bin,State) -> + State. + +rotate_file(#{fd:=Fd0,file_name:=FileName,modes:=Modes,rotation:=Rotation}=State) -> + State1 = sync_dev(State), + _ = file:close(Fd0), + _ = file:close(Fd0), + rotate_files(FileName,maps:get(count,Rotation),maps:get(compress,Rotation)), + case file:open(FileName,Modes) of + {ok,Fd} -> + {ok,#file_info{inode=INode}} = file:read_file_info(FileName,[raw]), + State1#{fd=>Fd,inode=>INode,rotation=>Rotation#{curr_size=>0}}; + Error -> + exit({could_not_reopen_file,Error}) + end. + +rotate_files(FileName,0,_Compress) -> + _ = file:delete(FileName), + ok; +rotate_files(FileName,1,Compress) -> + FileName0 = FileName++".0", + _ = file:rename(FileName,FileName0), + if Compress -> compress_file(FileName0); + true -> ok + end, + ok; +rotate_files(FileName,Count,Compress) -> + _ = file:rename(rot_file_name(FileName,Count-2,Compress), + rot_file_name(FileName,Count-1,Compress)), + rotate_files(FileName,Count-1,Compress). + +rot_file_name(FileName,Count,false) -> + FileName ++ "." ++ integer_to_list(Count); +rot_file_name(FileName,Count,true) -> + rot_file_name(FileName,Count,false) ++ ".gz". + +compress_file(FileName) -> + {ok,In} = file:open(FileName,[read,binary]), + {ok,Out} = file:open(FileName++".gz",[write]), + Z = zlib:open(), + zlib:deflateInit(Z, default, deflated, 31, 8, default), + compress_data(Z,In,Out), + zlib:deflateEnd(Z), + zlib:close(Z), + _ = file:close(In), + _ = file:close(Out), + _ = file:delete(FileName), + ok. + +compress_data(Z,In,Out) -> + case file:read(In,100000) of + {ok,Data} -> + Compressed = zlib:deflate(Z, Data), + _ = file:write(Out,Compressed), + compress_data(Z,In,Out); + eof -> + Compressed = zlib:deflate(Z, <<>>, finish), + _ = file:write(Out,Compressed), + ok + end. + +decompress_file(FileName) -> + {ok,In} = file:open(FileName,[read,binary]), + {ok,Out} = file:open(filename:rootname(FileName,".gz"),[write]), + Z = zlib:open(), + zlib:inflateInit(Z, 31), + decompress_data(Z,In,Out), + zlib:inflateEnd(Z), + zlib:close(Z), + _ = file:close(In), + _ = file:close(Out), + _ = file:delete(FileName), + ok. -maybe_notify_error(_Op, ok, _PrevResult, _FileName, _HandlerName) -> +decompress_data(Z,In,Out) -> + case file:read(In,1000) of + {ok,Data} -> + Decompressed = zlib:inflate(Z, Data), + _ = file:write(Out,Decompressed), + decompress_data(Z,In,Out); + eof -> + ok + end. + +maybe_notify_error(_Op, ok, _State) -> ok; -maybe_notify_error(_Op, PrevResult, PrevResult, _FileName, _HandlerName) -> +maybe_notify_error(Op, Result, #{write_res:=WR,sync_res:=SR}) + when (Op==write andalso Result==WR) orelse + (Op==filesync andalso Result==SR) -> %% don't report same error twice - PrevResult; -maybe_notify_error(Op, Error, _PrevResult, FileName, HandlerName) -> + ok; +maybe_notify_error(Op, Error, #{handler_name:=HandlerName,file_name:=FileName}) -> logger_h_common:error_notify({HandlerName,Op,FileName,Error}), - Error. + ok. + +timestamp() -> + erlang:monotonic_time(millisecond). diff --git a/lib/kernel/src/logger_sup.erl b/lib/kernel/src/logger_sup.erl index 3d6f482e20..9ea8558a16 100644 --- a/lib/kernel/src/logger_sup.erl +++ b/lib/kernel/src/logger_sup.erl @@ -50,7 +50,9 @@ init([]) -> start => {logger_handler_watcher, start_link, []}, shutdown => brutal_kill}, - {ok, {SupFlags, [Watcher]}}. + Proxy = logger_proxy:child_spec(), + + {ok, {SupFlags, [Watcher,Proxy]}}. %%%=================================================================== %%% Internal functions diff --git a/lib/kernel/src/net.erl b/lib/kernel/src/net.erl deleted file mode 100644 index 2d0ae2ed0c..0000000000 --- a/lib/kernel/src/net.erl +++ /dev/null @@ -1,40 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% --module(net). - -%% Various network functions, kept here for compatibility - --export([call/4, - cast/4, - broadcast/3, - ping/1, - relay/1, - sleep/1]). - --deprecated(module). - -call(N,M,F,A) -> rpc:call(N,M,F,A). -cast(N,M,F,A) -> rpc:cast(N,M,F,A). -broadcast(M,F,A) -> rpc:eval_everywhere(M,F,A). -ping(Node) -> net_adm:ping(Node). -sleep(T) -> receive after T -> ok end. -relay(X) -> slave:relay(X). - - diff --git a/lib/kernel/src/net_kernel.erl b/lib/kernel/src/net_kernel.erl index a9dc77837e..83d3b4b5e1 100644 --- a/lib/kernel/src/net_kernel.erl +++ b/lib/kernel/src/net_kernel.erl @@ -1126,14 +1126,22 @@ do_disconnect(Node, State) -> {false, State} end. - disconnect_pid(Pid, State) -> exit(Pid, disconnect), + + %% This code used to only use exit + recv 'EXIT' to sync, + %% but since OTP-22 links are no longer broken atomically + %% so the exit message below can arrive before any remaining + %% exit messages have killed the distribution port + Ref = erlang:monitor(process, Pid), %% Sync wait for connection to die!!! receive - {'EXIT',Pid,Reason} -> - {_,State1} = handle_exit(Pid, Reason, State), - {true, State1} + {'DOWN',Ref,_,_,_} -> + receive + {'EXIT',Pid,Reason} -> + {_,State1} = handle_exit(Pid, Reason, State), + {true, State1} + end end. %% @@ -1433,7 +1441,7 @@ validate_hostname([$@|HostPart] = Host) -> end. valid_name_head(Head) -> - {ok, MP} = re:compile("^[0-9A-Za-z_\\-]*$", [unicode]), + {ok, MP} = re:compile("^[0-9A-Za-z_\\-]+$", [unicode]), case re:run(Head, MP) of {match, _} -> true; diff --git a/lib/kernel/src/raw_file_io_inflate.erl b/lib/kernel/src/raw_file_io_inflate.erl index 7e9780310c..d3ed02dd03 100644 --- a/lib/kernel/src/raw_file_io_inflate.erl +++ b/lib/kernel/src/raw_file_io_inflate.erl @@ -26,7 +26,7 @@ -include("file_int.hrl"). --define(INFLATE_CHUNK_SIZE, (1 bsl 10)). +-define(INFLATE_CHUNK_SIZE, (8 bsl 10)). -define(GZIP_WBITS, (16 + 15)). callback_mode() -> state_functions. diff --git a/lib/kernel/src/seq_trace.erl b/lib/kernel/src/seq_trace.erl index 14fe21e9de..4f9d7b3e5c 100644 --- a/lib/kernel/src/seq_trace.erl +++ b/lib/kernel/src/seq_trace.erl @@ -98,7 +98,7 @@ print(Label, Term) -> -spec reset_trace() -> 'true'. reset_trace() -> - erlang:system_flag(1, 0). + erlang:system_flag(reset_seq_trace, true). %% reset_trace(Pid) -> % this might be a useful function too diff --git a/lib/kernel/src/standard_error.erl b/lib/kernel/src/standard_error.erl index 5d649e5f94..ef5b532960 100644 --- a/lib/kernel/src/standard_error.erl +++ b/lib/kernel/src/standard_error.erl @@ -27,7 +27,8 @@ -define(PROCNAME_SUP, standard_error_sup). %% Defines for control ops --define(CTRL_OP_GET_WINSIZE,100). +-define(ERTS_TTYSL_DRV_CONTROL_MAGIC_NUMBER, 16#018b0900). +-define(CTRL_OP_GET_WINSIZE, (100 + ?ERTS_TTYSL_DRV_CONTROL_MAGIC_NUMBER)). %% %% The basic server and start-up. diff --git a/lib/kernel/src/user.erl b/lib/kernel/src/user.erl index 872e63ab53..0c9e1ea303 100644 --- a/lib/kernel/src/user.erl +++ b/lib/kernel/src/user.erl @@ -28,7 +28,8 @@ -define(NAME, user). %% Defines for control ops --define(CTRL_OP_GET_WINSIZE,100). +-define(ERTS_TTYSL_DRV_CONTROL_MAGIC_NUMBER, 16#018b0900). +-define(CTRL_OP_GET_WINSIZE, (100 + ?ERTS_TTYSL_DRV_CONTROL_MAGIC_NUMBER)). %% %% The basic server and start-up. diff --git a/lib/kernel/src/user_drv.erl b/lib/kernel/src/user_drv.erl index 9f914aa222..08286dd476 100644 --- a/lib/kernel/src/user_drv.erl +++ b/lib/kernel/src/user_drv.erl @@ -32,9 +32,10 @@ -define(OP_BEEP,4). -define(OP_PUTC_SYNC,5). % Control op --define(CTRL_OP_GET_WINSIZE,100). --define(CTRL_OP_GET_UNICODE_STATE,101). --define(CTRL_OP_SET_UNICODE_STATE,102). +-define(ERTS_TTYSL_DRV_CONTROL_MAGIC_NUMBER, 16#018b0900). +-define(CTRL_OP_GET_WINSIZE, (100 + ?ERTS_TTYSL_DRV_CONTROL_MAGIC_NUMBER)). +-define(CTRL_OP_GET_UNICODE_STATE, (101 + ?ERTS_TTYSL_DRV_CONTROL_MAGIC_NUMBER)). +-define(CTRL_OP_SET_UNICODE_STATE, (102 + ?ERTS_TTYSL_DRV_CONTROL_MAGIC_NUMBER)). %% start() %% start(ArgumentList) |