Merge branch 'siri/kernel/application-shutdown-timeout/OTP-9540'

* siri/kernel/application-shutdown-timeout/OTP-9540:
  Add documentation and test for kernel env var shutdown_timeout
  Add env var shutdown_timeout to kernel to avoid deadlock on node shutdown

6 files changed, 137 insertions, 5 deletions

diff --git a/lib/kernel/doc/src/kernel_app.xml b/lib/kernel/doc/src/kernel_app.xml
index bf513b7815..0f71a4f0f2 100644
--- a/lib/kernel/doc/src/kernel_app.xml
+++ b/lib/kernel/doc/src/kernel_app.xml
@@ -4,7 +4,7 @@
 <appref>
   <header>
     <copyright>
-      <year>1996</year><year>2009</year>
+      <year>1996</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -231,6 +231,15 @@ MaxT = TickTime + TickTime / 4</code>
         <p><em>Note:</em> Normally, a terminating node is detected
           immediately.</p>
       </item>
+      <tag><c>shutdown_timeout = integer() | infinity</c></tag>
+      <item>
+        <p>Specifies the time <c>application_controller</c> will wait
+          for an application to terminate during node shutdown. If the
+          timer expires, <c>application_controller</c> will brutally
+          kill <c>application_master</c> of the hanging
+          application. If this parameter is undefined, it defaults
+          to <c>infinity</c>.</p>
+      </item>
       <tag><c>sync_nodes_mandatory = [NodeName]</c></tag>
       <item>
         <p>Specifies which other nodes <em>must</em> be alive in order
diff --git a/lib/kernel/src/application_controller.erl b/lib/kernel/src/application_controller.erl
index 42f527f400..ebfe84463a 100644
--- a/lib/kernel/src/application_controller.erl
+++ b/lib/kernel/src/application_controller.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1996-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1996-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -1180,10 +1180,27 @@ terminate(Reason, S) ->
 	_ ->
 	    ok
     end,
+    ShutdownTimeout =
+	case application:get_env(kernel, shutdown_timeout) of
+	    undefined -> infinity;
+	    {ok,T} -> T
+	end,
     foreach(fun({_AppName, Id}) when is_pid(Id) -> 
+		    Ref = erlang:monitor(process, Id),
+		    unlink(Id),
 		    exit(Id, shutdown),
 		    receive
+			%% Proc died before link
 			{'EXIT', Id, _} -> ok
+		    after 0 ->
+			    receive
+				{'DOWN', Ref, process, Id, _} -> ok
+			    after ShutdownTimeout ->
+				    exit(Id, kill),
+				    receive
+					{'DOWN', Ref, process, Id, _} -> ok
+				    end
+			    end
 		    end;
 	       (_) -> ok
 	    end,
diff --git a/lib/kernel/test/application_SUITE.erl b/lib/kernel/test/application_SUITE.erl
index 2c5b8ccb66..f469a0af98 100644
--- a/lib/kernel/test/application_SUITE.erl
+++ b/lib/kernel/test/application_SUITE.erl
@@ -33,7 +33,7 @@
 
 -export([config_change/1,
 	 distr_changed_tc1/1, distr_changed_tc2/1,
-	 shutdown_func/1, do_shutdown/1]).
+	 shutdown_func/1, do_shutdown/1, shutdown_timeout/1]).
 
 -define(TESTCASE, testcase_name).
 -define(testcase, ?config(?TESTCASE, Config)).
@@ -50,7 +50,7 @@ all() ->
      load_use_cache, {group, reported_bugs}, start_phases,
      script_start, nodedown_start, permit_false_start_local,
      permit_false_start_dist, get_key,
-     {group, distr_changed}, config_change, shutdown_func].
+     {group, distr_changed}, config_change, shutdown_func, shutdown_timeout].
 
 groups() -> 
     [{reported_bugs, [],
@@ -1915,6 +1915,32 @@ do_shutdown(Reason) ->
 
 
 
+%%%-----------------------------------------------------------------
+%%% Tests the 'shutdown_timeout' kernel config parameter
+%%%-----------------------------------------------------------------
+shutdown_timeout(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir,Config),
+    {ok,Cp1} = start_node(?MODULE_STRING++"_shutdown_timeout"),
+    wait_for_ready_net(),
+    ok = rpc:call(Cp1, application, set_env, [kernel, shutdown_timeout, 1000]),
+    rpc:call(Cp1, code, add_path, [filename:join([DataDir,deadlock])]),
+    ok = rpc:call(Cp1, application, start, [sasl]),
+    ok = rpc:call(Cp1, application, start, [deadlock]),
+    rpc:call(Cp1, deadlock, restart_and_fail, []),
+
+    ok = net_kernel:monitor_nodes(true),
+    _ = rpc:call(Cp1, init, stop, []),
+    receive
+	{nodedown,Cp1} ->
+	    ok
+    after 10000 ->
+	    ct:fail("timeout 10 sec: node termination hangs")
+    end,
+    ok.
+
+
+
+
 %%-----------------------------------------------------------------
 %% Utility functions
 %%-----------------------------------------------------------------
diff --git a/lib/kernel/test/application_SUITE_data/Makefile.src b/lib/kernel/test/application_SUITE_data/Makefile.src
index a237f6badb..abc3c82907 100644
--- a/lib/kernel/test/application_SUITE_data/Makefile.src
+++ b/lib/kernel/test/application_SUITE_data/Makefile.src
@@ -2,7 +2,8 @@ EFLAGS=+debug_info
            
 all: app_start_error.@EMULATOR@ trans_abnormal_sup.@EMULATOR@ \
      trans_normal_sup.@EMULATOR@ transient.@EMULATOR@ \
-     group_leader_sup.@EMULATOR@ group_leader.@EMULATOR@
+     group_leader_sup.@EMULATOR@ group_leader.@EMULATOR@ \
+     deadlock/deadlock.@EMULATOR@
 
 app_start_error.@EMULATOR@: app_start_error.erl
 	erlc $(EFLAGS) app_start_error.erl
@@ -22,3 +23,5 @@ group_leader.@EMULATOR@: group_leader.erl
 group_leader_sup.@EMULATOR@: group_leader_sup.erl
 	erlc $(EFLAGS) group_leader_sup.erl
 
+deadlock/deadlock.@EMULATOR@: deadlock/deadlock.erl
+	erlc $(EFLAGS) -o deadlock deadlock/deadlock.erl
\ No newline at end of file
diff --git a/lib/kernel/test/application_SUITE_data/deadlock/deadlock.app b/lib/kernel/test/application_SUITE_data/deadlock/deadlock.app
new file mode 100644
index 0000000000..0c1001bed6
--- /dev/null
+++ b/lib/kernel/test/application_SUITE_data/deadlock/deadlock.app
@@ -0,0 +1,8 @@
+{application, deadlock, [
+  {vsn, "1"},
+  {registered, []},
+  {applications, [kernel, stdlib, sasl]},
+  {modules, [deadlock]},
+  {mod, {deadlock, []}},
+  {env, [{fail_start, false}]}
+]}.
diff --git a/lib/kernel/test/application_SUITE_data/deadlock/deadlock.erl b/lib/kernel/test/application_SUITE_data/deadlock/deadlock.erl
new file mode 100644
index 0000000000..5f68bf9078
--- /dev/null
+++ b/lib/kernel/test/application_SUITE_data/deadlock/deadlock.erl
@@ -0,0 +1,69 @@
+-module(deadlock).
+-behaviour(application).
+-compile(export_all).
+-define(SUP,deadlock_sup).
+-define(CHILD,deadlock_child).
+
+
+%%%-----------------------------------------------------------------
+%%% application callbacks
+start(_StartType, _StartArgs) ->
+    supervisor:start_link({local, ?SUP}, ?MODULE, [sup]).
+
+stop(_State) ->
+    ok.
+
+
+
+%%%-----------------------------------------------------------------
+%%% supervisor callbacks
+init([sup]) ->
+    {ok, {{one_for_one, 5, 10}, [
+        {
+            sasl_syslog_dm, {?MODULE, start_link, []},
+            permanent, brutal_kill, worker,
+            [deadlock]
+        }
+    ]}};
+
+
+%%%-----------------------------------------------------------------
+%%% gen_server callbacks
+init([child]) ->
+    case application:get_env(deadlock, fail_start) of
+        {ok, false} ->
+            %% we must not fail on the first init, otherwise supervisor
+            %% terminates immediately
+            {ok, []};
+        {ok, true}  ->
+	    timer:sleep(infinity), % init hangs!!!!
+	    {ok, []}
+    end.
+
+handle_call(_Req, _From, State) ->
+    {reply, ok, State}.
+
+handle_cast(restart, State) ->
+    {stop, error, State}.
+
+handle_info(_Msg, State) ->
+    {noreply, State}.
+
+terminate(_Reason, _State) ->
+    ok.
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+
+%%%-----------------------------------------------------------------
+%%% Start child
+start_link() ->
+    gen_server:start_link({local, ?CHILD}, ?MODULE, [child], []).
+
+
+%%%-----------------------------------------------------------------
+%%% Provoke hanging
+restart_and_fail() ->
+    application:set_env(deadlock, fail_start, true), % next init will hang
+    gen_server:cast(?CHILD, restart).