From d8ce99ba8dd966e87a61e50b70d01e7f1f381bf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Mon, 15 Feb 2016 15:28:38 +0100 Subject: kernel: Refactor heart code --- lib/kernel/src/heart.erl | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) (limited to 'lib/kernel/src') diff --git a/lib/kernel/src/heart.erl b/lib/kernel/src/heart.erl index 464b6919f1..97f384b1d8 100644 --- a/lib/kernel/src/heart.erl +++ b/lib/kernel/src/heart.erl @@ -49,6 +49,9 @@ -define(CYCLE_TIMEOUT, 10000). -define(HEART_PORT_NAME, heart_port). +-record(state,{port :: port(), + cmd :: [] | binary()}). + %%--------------------------------------------------------------------- -spec start() -> 'ignore' | {'error', term()} | {'ok', pid()}. @@ -85,7 +88,7 @@ init(Starter, Parent) -> case catch start_portprogram() of {ok, Port} -> Starter ! {ok, self()}, - loop(Parent, Port, ""); + loop(Parent, #state{port = Port, cmd = []}); no_heart -> Starter ! {no_heart, self()}; error -> @@ -182,7 +185,7 @@ wait_ack(Port) -> {error, Reason} end. -loop(Parent, Port, Cmd) -> +loop(Parent, #state{port=Port}=S) -> _ = send_heart_beat(Port), receive {From, set_cmd, NewCmd0} -> @@ -192,36 +195,36 @@ loop(Parent, Port, Cmd) -> _ = send_heart_cmd(Port, NewCmd), _ = wait_ack(Port), From ! {heart, ok}, - loop(Parent, Port, NewCmd); + loop(Parent, S#state{cmd=NewCmd}); _ -> From ! {heart, {error, {bad_cmd, NewCmd0}}}, - loop(Parent, Port, Cmd) + loop(Parent, S) end; {From, clear_cmd} -> From ! {heart, ok}, - _ = send_heart_cmd(Port, ""), + _ = send_heart_cmd(Port, []), _ = wait_ack(Port), - loop(Parent, Port, ""); + loop(Parent, S#state{cmd = []}); {From, get_cmd} -> From ! {heart, get_heart_cmd(Port)}, - loop(Parent, Port, Cmd); + loop(Parent, S); {From, cycle} -> %% Calls back to loop - do_cycle_port_program(From, Parent, Port, Cmd); + do_cycle_port_program(From, Parent, S); {'EXIT', Parent, shutdown} -> no_reboot_shutdown(Port); {'EXIT', Parent, Reason} -> exit(Port, Reason), exit(Reason); {'EXIT', Port, badsig} -> % we can ignore badsig-messages! - loop(Parent, Port, Cmd); + loop(Parent, S); {'EXIT', Port, _Reason} -> - exit({port_terminated, {heart, loop, [Parent, Port, Cmd]}}); + exit({port_terminated, {heart, loop, [Parent, S]}}); _ -> - loop(Parent, Port, Cmd) + loop(Parent, S) after ?TIMEOUT -> - loop(Parent, Port, Cmd) + loop(Parent, S) end. -spec no_reboot_shutdown(port()) -> no_return(). @@ -233,31 +236,31 @@ no_reboot_shutdown(Port) -> exit(normal) end. -do_cycle_port_program(Caller, Parent, Port, Cmd) -> +do_cycle_port_program(Caller, Parent, #state{port=Port} = S) -> unregister(?HEART_PORT_NAME), case catch start_portprogram() of {ok, NewPort} -> _ = send_shutdown(Port), receive {'EXIT', Port, _Reason} -> - _ = send_heart_cmd(NewPort, Cmd), + _ = send_heart_cmd(NewPort, S#state.cmd), Caller ! {heart, ok}, - loop(Parent, NewPort, Cmd) + loop(Parent, S#state{port=NewPort}) after ?CYCLE_TIMEOUT -> %% Huh! Two heart port programs running... %% well, the old one has to be sick not to respond %% so we'll settle for the new one... - _ = send_heart_cmd(NewPort, Cmd), + _ = send_heart_cmd(NewPort, S#state.cmd), Caller ! {heart, {error, stop_error}}, - loop(Parent, NewPort, Cmd) + loop(Parent, S#state{port=NewPort}) end; no_heart -> Caller ! {heart, {error, no_heart}}, - loop(Parent, Port, Cmd); + loop(Parent, S); error -> Caller ! {heart, {error, start_error}}, - loop(Parent, Port, Cmd) + loop(Parent, S) end. -- cgit v1.2.3 From f5a6e1c353e2a7fb536dcac4523e4f6e3d99039b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Mon, 15 Feb 2016 16:18:20 +0100 Subject: kernel: Add validation callback for heart * heart:set_callback/2 * heart:get_callback/0 * heart:clear_callback/0 The callback is called before every heartbeat to the heart port. The callback needs to return 'ok' if the validation is correct. --- lib/kernel/src/heart.erl | 97 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 78 insertions(+), 19 deletions(-) (limited to 'lib/kernel/src') diff --git a/lib/kernel/src/heart.erl b/lib/kernel/src/heart.erl index 97f384b1d8..ad3bbf2f7a 100644 --- a/lib/kernel/src/heart.erl +++ b/lib/kernel/src/heart.erl @@ -34,7 +34,10 @@ %%% %%% It recognizes the flag '-heart' %%%-------------------------------------------------------------------- --export([start/0, init/2, set_cmd/1, clear_cmd/0, get_cmd/0, cycle/0]). +-export([start/0, init/2, + set_cmd/1, clear_cmd/0, get_cmd/0, + set_callback/2, clear_callback/0, get_callback/0, + cycle/0]). -define(START_ACK, 1). -define(HEART_BEAT, 2). @@ -50,7 +53,8 @@ -define(HEART_PORT_NAME, heart_port). -record(state,{port :: port(), - cmd :: [] | binary()}). + cmd :: [] | binary(), + callback :: 'undefined' | {module(), function()}}). %%--------------------------------------------------------------------- @@ -84,7 +88,7 @@ wait_for_init_ack(From) -> init(Starter, Parent) -> process_flag(trap_exit, true), process_flag(priority, max), - register(heart, self()), + register(?MODULE, self()), case catch start_portprogram() of {ok, Port} -> Starter ! {ok, self()}, @@ -99,20 +103,41 @@ init(Starter, Parent) -> Cmd :: string(). set_cmd(Cmd) -> - heart ! {self(), set_cmd, Cmd}, + ?MODULE ! {self(), set_cmd, Cmd}, wait(). -spec get_cmd() -> {ok, Cmd} when Cmd :: string(). get_cmd() -> - heart ! {self(), get_cmd}, + ?MODULE ! {self(), get_cmd}, wait(). -spec clear_cmd() -> ok. clear_cmd() -> - heart ! {self(), clear_cmd}, + ?MODULE ! {self(), clear_cmd}, + wait(). + +-spec set_callback(Module,Function) -> 'ok' | {'error', {'bad_callback', {Module, Function}}} when + Module :: module(), + Function :: function(). + +set_callback(Module, Function) -> + ?MODULE ! {self(), set_callback, {Module,Function}}, + wait(). + +-spec get_callback() -> {'ok', Callback} | 'none' when + Callback :: {module(), function()}. + +get_callback() -> + ?MODULE ! {self(), get_callback}, + wait(). + +-spec clear_callback() -> ok. + +clear_callback() -> + ?MODULE ! {self(), clear_callback}, wait(). @@ -120,12 +145,12 @@ clear_cmd() -> -spec cycle() -> 'ok' | {'error', term()}. cycle() -> - heart ! {self(), cycle}, + ?MODULE ! {self(), cycle}, wait(). wait() -> receive - {heart, Res} -> + {?MODULE, Res} -> Res end. @@ -186,7 +211,7 @@ wait_ack(Port) -> end. loop(Parent, #state{port=Port}=S) -> - _ = send_heart_beat(Port), + _ = send_heart_beat(S), receive {From, set_cmd, NewCmd0} -> Enc = file:native_name_encoding(), @@ -194,20 +219,39 @@ loop(Parent, #state{port=Port}=S) -> NewCmd when is_binary(NewCmd), byte_size(NewCmd) < 2047 -> _ = send_heart_cmd(Port, NewCmd), _ = wait_ack(Port), - From ! {heart, ok}, + From ! {?MODULE, ok}, loop(Parent, S#state{cmd=NewCmd}); _ -> - From ! {heart, {error, {bad_cmd, NewCmd0}}}, + From ! {?MODULE, {error, {bad_cmd, NewCmd0}}}, loop(Parent, S) end; {From, clear_cmd} -> - From ! {heart, ok}, + From ! {?MODULE, ok}, _ = send_heart_cmd(Port, []), _ = wait_ack(Port), loop(Parent, S#state{cmd = []}); {From, get_cmd} -> - From ! {heart, get_heart_cmd(Port)}, + From ! {?MODULE, get_heart_cmd(Port)}, loop(Parent, S); + {From, set_callback, Callback} -> + case Callback of + {M,F} when is_atom(M), is_atom(F) -> + From ! {?MODULE, ok}, + loop(Parent, S#state{callback=Callback}); + _ -> + From ! {?MODULE, {error, {bad_callback, Callback}}}, + loop(Parent, S) + end; + {From, get_callback} -> + Res = case S#state.callback of + undefined -> none; + Cb -> {ok, Cb} + end, + From ! {?MODULE, Res}, + loop(Parent, S); + {From, clear_callback} -> + From ! {?MODULE, ok}, + loop(Parent, S#state{callback=undefined}); {From, cycle} -> %% Calls back to loop do_cycle_port_program(From, Parent, S); @@ -219,7 +263,7 @@ loop(Parent, #state{port=Port}=S) -> {'EXIT', Port, badsig} -> % we can ignore badsig-messages! loop(Parent, S); {'EXIT', Port, _Reason} -> - exit({port_terminated, {heart, loop, [Parent, S]}}); + exit({port_terminated, {?MODULE, loop, [Parent, S]}}); _ -> loop(Parent, S) after @@ -244,7 +288,7 @@ do_cycle_port_program(Caller, Parent, #state{port=Port} = S) -> receive {'EXIT', Port, _Reason} -> _ = send_heart_cmd(NewPort, S#state.cmd), - Caller ! {heart, ok}, + Caller ! {?MODULE, ok}, loop(Parent, S#state{port=NewPort}) after ?CYCLE_TIMEOUT -> @@ -252,20 +296,22 @@ do_cycle_port_program(Caller, Parent, #state{port=Port} = S) -> %% well, the old one has to be sick not to respond %% so we'll settle for the new one... _ = send_heart_cmd(NewPort, S#state.cmd), - Caller ! {heart, {error, stop_error}}, + Caller ! {?MODULE, {error, stop_error}}, loop(Parent, S#state{port=NewPort}) end; no_heart -> - Caller ! {heart, {error, no_heart}}, + Caller ! {?MODULE, {error, no_heart}}, loop(Parent, S); error -> - Caller ! {heart, {error, start_error}}, + Caller ! {?MODULE, {error, start_error}}, loop(Parent, S) end. %% "Beates" the heart once. -send_heart_beat(Port) -> Port ! {self(), {command, [?HEART_BEAT]}}. +send_heart_beat(#state{port=Port, callback=Cb}) -> + ok = check_callback(Cb), + Port ! {self(), {command, [?HEART_BEAT]}}. %% Set a new HEART_COMMAND. send_heart_cmd(Port, []) -> @@ -280,6 +326,19 @@ get_heart_cmd(Port) -> {ok, Cmd} end. +%% validate system by performing a check before the heartbeat +%% return 'ok' if everything is alright. +%% Terminate if with reason if something is a miss. +%% It is fine to timeout in the callback, in fact that is the intention +%% if something goes wront -> no heartbeat. + +check_callback(Callback) -> + case Callback of + undefined -> ok; + {M,F} -> + erlang:apply(M,F,[]) + end. + %% Sends shutdown command to the port. send_shutdown(Port) -> Port ! {self(), {command, [?SHUT_DOWN]}}. -- cgit v1.2.3 From f58e74b08380758d97c4d8a1d9ef53217f73fe6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Mon, 15 Feb 2016 18:16:18 +0100 Subject: kernel: Add basic system check of schedulers on heartbeat Before a heartbeat to the port program a responsiveness check of the schedulers is performed. If the responsiveness check fails, stalls, the heartbeat will not be performed (as intended). --- lib/kernel/src/heart.erl | 1 + 1 file changed, 1 insertion(+) (limited to 'lib/kernel/src') diff --git a/lib/kernel/src/heart.erl b/lib/kernel/src/heart.erl index ad3bbf2f7a..617ae2f91b 100644 --- a/lib/kernel/src/heart.erl +++ b/lib/kernel/src/heart.erl @@ -333,6 +333,7 @@ get_heart_cmd(Port) -> %% if something goes wront -> no heartbeat. check_callback(Callback) -> + ok = erts_internal:system_check(schedulers), case Callback of undefined -> ok; {M,F} -> -- cgit v1.2.3 From a3ed00b212fedb7dbe9dc5daf4c3beaf2c4dbe9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Tue, 16 Feb 2016 16:02:34 +0100 Subject: kernel: Fix heart dialyzer types --- lib/kernel/src/heart.erl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'lib/kernel/src') diff --git a/lib/kernel/src/heart.erl b/lib/kernel/src/heart.erl index 617ae2f91b..686e2e03ff 100644 --- a/lib/kernel/src/heart.erl +++ b/lib/kernel/src/heart.erl @@ -54,7 +54,7 @@ -record(state,{port :: port(), cmd :: [] | binary(), - callback :: 'undefined' | {module(), function()}}). + callback :: 'undefined' | {atom(), atom()}}). %%--------------------------------------------------------------------- @@ -120,15 +120,16 @@ clear_cmd() -> wait(). -spec set_callback(Module,Function) -> 'ok' | {'error', {'bad_callback', {Module, Function}}} when - Module :: module(), - Function :: function(). + Module :: atom(), + Function :: atom(). set_callback(Module, Function) -> ?MODULE ! {self(), set_callback, {Module,Function}}, wait(). --spec get_callback() -> {'ok', Callback} | 'none' when - Callback :: {module(), function()}. +-spec get_callback() -> {'ok', {Module, Function}} | 'none' when + Module :: atom(), + Function :: atom(). get_callback() -> ?MODULE ! {self(), get_callback}, -- cgit v1.2.3 From f623b7430ec8f7bc48770953b32a486816ad6eb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Mon, 22 Feb 2016 16:59:49 +0100 Subject: kernel: Add builtin scheduler check for heart In addition, the heart API is extended with the following functions: * heart:set_options/1 * heart:get_options/0 If heart:set_options([scheduler]) is set, heart will check scheduler responsiveness before every heartbeat to the heart port. --- lib/kernel/src/heart.erl | 47 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) (limited to 'lib/kernel/src') diff --git a/lib/kernel/src/heart.erl b/lib/kernel/src/heart.erl index 686e2e03ff..273bdcb352 100644 --- a/lib/kernel/src/heart.erl +++ b/lib/kernel/src/heart.erl @@ -37,6 +37,7 @@ -export([start/0, init/2, set_cmd/1, clear_cmd/0, get_cmd/0, set_callback/2, clear_callback/0, get_callback/0, + set_options/1, get_options/0, cycle/0]). -define(START_ACK, 1). @@ -54,6 +55,7 @@ -record(state,{port :: port(), cmd :: [] | binary(), + options :: [atom()], callback :: 'undefined' | {atom(), atom()}}). %%--------------------------------------------------------------------- @@ -92,7 +94,7 @@ init(Starter, Parent) -> case catch start_portprogram() of {ok, Port} -> Starter ! {ok, self()}, - loop(Parent, #state{port = Port, cmd = []}); + loop(Parent, #state{port=Port, cmd=[], options=[]}); no_heart -> Starter ! {no_heart, self()}; error -> @@ -141,6 +143,19 @@ clear_callback() -> ?MODULE ! {self(), clear_callback}, wait(). +-spec set_options(Options) -> 'ok' | {'error', {'bad_options', Options}} when + Options :: [atom()]. + +set_options(Options) -> + ?MODULE ! {self(), set_options, Options}, + wait(). + +-spec get_options() -> {'ok', Options} | 'none' when + Options :: [atom()]. + +get_options() -> + ?MODULE ! {self(), get_options}, + wait(). %%% Should be used solely by the release handler!!!!!!! -spec cycle() -> 'ok' | {'error', term()}. @@ -253,6 +268,22 @@ loop(Parent, #state{port=Port}=S) -> {From, clear_callback} -> From ! {?MODULE, ok}, loop(Parent, S#state{callback=undefined}); + {From, set_options, Options} -> + case validate_options(Options) of + Validated when is_list(Validated) -> + From ! {?MODULE, ok}, + loop(Parent, S#state{options=Validated}); + _ -> + From ! {?MODULE, {error, {bad_options, Options}}}, + loop(Parent, S) + end; + {From, get_options} -> + Res = case S#state.options of + [] -> none; + Cb -> {ok, Cb} + end, + From ! {?MODULE, Res}, + loop(Parent, S); {From, cycle} -> %% Calls back to loop do_cycle_port_program(From, Parent, S); @@ -281,6 +312,11 @@ no_reboot_shutdown(Port) -> exit(normal) end. +validate_options(Opts) -> validate_options(Opts,[]). +validate_options([],Res) -> Res; +validate_options([scheduler=Opt|Opts],Res) -> validate_options(Opts,[Opt|Res]); +validate_options(_,_) -> error. + do_cycle_port_program(Caller, Parent, #state{port=Port} = S) -> unregister(?HEART_PORT_NAME), case catch start_portprogram() of @@ -310,7 +346,8 @@ do_cycle_port_program(Caller, Parent, #state{port=Port} = S) -> %% "Beates" the heart once. -send_heart_beat(#state{port=Port, callback=Cb}) -> +send_heart_beat(#state{port=Port, callback=Cb, options=Opts}) -> + ok = check_system(Opts), ok = check_callback(Cb), Port ! {self(), {command, [?HEART_BEAT]}}. @@ -327,6 +364,11 @@ get_heart_cmd(Port) -> {ok, Cmd} end. +check_system([]) -> ok; +check_system([scheduler|Opts]) -> + ok = erts_internal:system_check(schedulers), + check_system(Opts). + %% validate system by performing a check before the heartbeat %% return 'ok' if everything is alright. %% Terminate if with reason if something is a miss. @@ -334,7 +376,6 @@ get_heart_cmd(Port) -> %% if something goes wront -> no heartbeat. check_callback(Callback) -> - ok = erts_internal:system_check(schedulers), case Callback of undefined -> ok; {M,F} -> -- cgit v1.2.3 From 0ca09ee90d6384e74d1b18ab0e05f2c05fc03905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= Date: Thu, 25 Feb 2016 15:13:32 +0100 Subject: kernel: Clarify heart option Change scheduler responsiveness to 'check_schedulers'. --- lib/kernel/src/heart.erl | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'lib/kernel/src') diff --git a/lib/kernel/src/heart.erl b/lib/kernel/src/heart.erl index 273bdcb352..d14598cd05 100644 --- a/lib/kernel/src/heart.erl +++ b/lib/kernel/src/heart.erl @@ -53,9 +53,14 @@ -define(CYCLE_TIMEOUT, 10000). -define(HEART_PORT_NAME, heart_port). +%% valid heart options +-define(SCHEDULER_CHECK_OPT, check_schedulers). + +-type heart_option() :: ?SCHEDULER_CHECK_OPT. + -record(state,{port :: port(), cmd :: [] | binary(), - options :: [atom()], + options :: [heart_option()], callback :: 'undefined' | {atom(), atom()}}). %%--------------------------------------------------------------------- @@ -144,7 +149,7 @@ clear_callback() -> wait(). -spec set_options(Options) -> 'ok' | {'error', {'bad_options', Options}} when - Options :: [atom()]. + Options :: [heart_option()]. set_options(Options) -> ?MODULE ! {self(), set_options, Options}, @@ -314,7 +319,7 @@ no_reboot_shutdown(Port) -> validate_options(Opts) -> validate_options(Opts,[]). validate_options([],Res) -> Res; -validate_options([scheduler=Opt|Opts],Res) -> validate_options(Opts,[Opt|Res]); +validate_options([?SCHEDULER_CHECK_OPT=Opt|Opts],Res) -> validate_options(Opts,[Opt|Res]); validate_options(_,_) -> error. do_cycle_port_program(Caller, Parent, #state{port=Port} = S) -> @@ -365,7 +370,7 @@ get_heart_cmd(Port) -> end. check_system([]) -> ok; -check_system([scheduler|Opts]) -> +check_system([?SCHEDULER_CHECK_OPT|Opts]) -> ok = erts_internal:system_check(schedulers), check_system(Opts). -- cgit v1.2.3