Repair broken force_stop functionality

3 files changed, 68 insertions, 29 deletions

diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl
index 5924930072..b7eeace609 100644
--- a/lib/common_test/src/ct_logs.erl
+++ b/lib/common_test/src/ct_logs.erl
@@ -1475,8 +1475,9 @@ count_cases(Dir) ->
 			    write_summary(SumFile, Summary),
 			    Summary
 		    end;
-		{error, _Reason} ->
-		    io:format("\nFailed to read ~p (skipped)\n", [LogFile]),
+		{error, Reason} ->
+		    io:format("\nFailed to read ~p: ~p (skipped)\n",
+			      [LogFile,Reason]),
 		    error
 	    end
     end.
diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index c0bdbb2a09..9f247fdb9e 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -2291,8 +2291,12 @@ add_jobs([{TestDir,all,_}|Tests], Skip, Opts, CleanUp) ->
 	{'EXIT',_} ->
 	    CleanUp;
 	_ ->
-	    wait_for_idle(),
-	    add_jobs(Tests, Skip, Opts, CleanUp)
+	    case wait_for_idle() of
+		ok ->
+		    add_jobs(Tests, Skip, Opts, CleanUp);
+		_ ->
+		    CleanUp
+	    end
     end;
 add_jobs([{TestDir,[Suite],all}|Tests], Skip,
 	 Opts, CleanUp) when is_atom(Suite) ->
@@ -2305,8 +2309,12 @@ add_jobs([{TestDir,Suites,all}|Tests], Skip,
 	{'EXIT',_} ->
 	    CleanUp;
 	_ ->
-	    wait_for_idle(),
-	    add_jobs(Tests, Skip, Opts, CleanUp)
+	    case wait_for_idle() of
+		ok ->
+		    add_jobs(Tests, Skip, Opts, CleanUp);
+		_ ->
+		    CleanUp
+	    end
     end;
 add_jobs([{TestDir,Suite,all}|Tests], Skip, Opts, CleanUp) ->
     case maybe_interpret(Suite, all, Opts) of
@@ -2318,8 +2326,12 @@ add_jobs([{TestDir,Suite,all}|Tests], Skip, Opts, CleanUp) ->
 		{'EXIT',_} ->
 		    CleanUp;
 		_ ->
-		    wait_for_idle(),
-		    add_jobs(Tests, Skip, Opts, [Suite|CleanUp])
+		    case wait_for_idle() of
+			ok ->
+			    add_jobs(Tests, Skip, Opts, [Suite|CleanUp]);
+			_ ->
+			    CleanUp
+		    end
 	    end;
 	Error ->
 	    Error
@@ -2358,8 +2370,12 @@ add_jobs([{TestDir,Suite,Confs}|Tests], Skip, Opts, CleanUp) when
 		{'EXIT',_} ->
 		    CleanUp;
 		_ ->
-		    wait_for_idle(),
-		    add_jobs(Tests, Skip, Opts, [Suite|CleanUp])
+		    case wait_for_idle() of
+			ok ->
+			    add_jobs(Tests, Skip, Opts, [Suite|CleanUp]);
+			_ ->
+			    CleanUp
+		    end
 	    end;
 	Error ->
 	    Error
@@ -2384,8 +2400,12 @@ add_jobs([{TestDir,Suite,Cases}|Tests],
 		{'EXIT',_} ->
 		    CleanUp;
 		_ ->
-		    wait_for_idle(),
-		    add_jobs(Tests, Skip, Opts, [Suite|CleanUp])
+		    case wait_for_idle() of
+			ok ->
+			    add_jobs(Tests, Skip, Opts, [Suite|CleanUp]);
+			_ ->
+			    CleanUp
+		    end
 	    end;
 	Error ->
 	    Error
@@ -2401,8 +2421,12 @@ add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opts, CleanUp) when is_atom(Case) -
 		{'EXIT',_} ->
 		    CleanUp;
 		_ ->
-		    wait_for_idle(),
-		    add_jobs(Tests, Skip, Opts, [Suite|CleanUp])
+		    case wait_for_idle() of
+			ok ->
+			    add_jobs(Tests, Skip, Opts, [Suite|CleanUp]);
+			_ ->
+			    CleanUp
+		    end
 	    end;
 	Error ->
 	    Error
@@ -2412,7 +2436,13 @@ add_jobs([], _, _, CleanUp) ->
 
 wait_for_idle() ->
     ct_util:update_last_run_index(),
-    Notify = fun(Me) -> Me ! idle end,
+    Notify = fun(Me,IdleState) -> Me ! {idle,IdleState},
+				  receive
+				      {Me,proceed} -> ok
+				  after
+				      30000 -> ok
+				  end
+	     end,
     case catch test_server_ctrl:idle_notify(Notify) of
 	{'EXIT',_} ->
 	    error;
@@ -2420,11 +2450,14 @@ wait_for_idle() ->
 	    %% so we don't hang forever if test_server dies
 	    Ref = erlang:monitor(process, TSPid),
 	    Result = receive
-			 idle -> ok;
+			 {idle,abort}           -> aborted;
+			 {idle,_}               -> ok;
 			 {'DOWN', Ref, _, _, _} -> error
 		     end,
 	    erlang:demonitor(Ref, [flush]),
 	    ct_util:update_last_run_index(),
+	    %% let test_server_ctrl proceed (and possibly shut down) now
+	    TSPid ! {self(),proceed},
 	    Result
     end.
 
diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl
index 70cb6fa220..5d4d392166 100644
--- a/lib/test_server/src/test_server_ctrl.erl
+++ b/lib/test_server/src/test_server_ctrl.erl
@@ -681,7 +681,7 @@ handle_call({abort_current_testcase,Reason}, _From, State) ->
 handle_call({finish,Fini}, _From, State) ->
     case State#state.jobs of
 	[] ->
-	    lists:foreach(fun({Cli,Fun}) -> Fun(Cli) end,
+	    lists:foreach(fun({Cli,Fun}) -> Fun(Cli,Fini) end,
 			  State#state.idle_notify),
 	    State2 = State#state{finish=false},
 	    {stop,shutdown,{ok,self()}, State2};
@@ -699,14 +699,11 @@ handle_call({finish,Fini}, _From, State) ->
 
 handle_call({idle_notify,Fun}, {Cli,_Ref}, State) ->
     case State#state.jobs of
-	[] ->
-	    Fun(Cli),
-	    {reply, {ok,self()}, State};
-	_ ->
-	    Subscribed = State#state.idle_notify,
-	    {reply, {ok,self()},
-	     State#state{idle_notify=[{Cli,Fun}|Subscribed]}}
-    end;
+	[] -> self() ! report_idle;
+	_  -> ok
+    end,
+    Subscribed = State#state.idle_notify,
+    {reply, {ok,self()}, State#state{idle_notify=[{Cli,Fun}|Subscribed]}};
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %% handle_call(start_get_totals, From, State) -> {ok,Pid}
@@ -1000,6 +997,13 @@ handle_cast({node_started,Node}, State) ->
 %% lost contact with target. The test_server_ctrl process is
 %% terminated, and teminate/2 will do the cleanup
 
+handle_info(report_idle, State) ->
+    Finish = State#state.finish,
+    lists:foreach(fun({Cli,Fun}) -> Fun(Cli,Finish) end,
+		  State#state.idle_notify),
+    {noreply,State#state{idle_notify=[]}};
+
+
 handle_info({'EXIT',Pid,Reason}, State) ->
     case lists:keysearch(Pid,2,State#state.jobs) of
 	false ->
@@ -1017,11 +1021,12 @@ handle_info({'EXIT',Pid,Reason}, State) ->
 			      [Name,Reason])
 	    end,
 	    State2 = State#state{jobs=NewJobs},
+	    Finish = State2#state.finish,
 	    case NewJobs of
 		[] ->
-		    lists:foreach(fun({Cli,Fun}) -> Fun(Cli) end,
+		    lists:foreach(fun({Cli,Fun}) -> Fun(Cli,Finish) end,
 				  State2#state.idle_notify),
-		    case State2#state.finish of
+		    case Finish of
 			false ->
 			    {noreply,State2#state{idle_notify=[]}};
 			_ ->			% true | abort
@@ -1031,9 +1036,9 @@ handle_info({'EXIT',Pid,Reason}, State) ->
 			    {stop,shutdown,State2#state{finish=false}}
 		    end;
 		_ ->				% pending jobs
-		    case State2#state.finish of
+		    case Finish of
 			abort ->		% abort test now!
-			    lists:foreach(fun({Cli,Fun}) -> Fun(Cli) end,
+			    lists:foreach(fun({Cli,Fun}) -> Fun(Cli,Finish) end,
 					  State2#state.idle_notify),
 			    {stop,shutdown,State2#state{finish=false}};
 			_ ->			% true | false