Merge remote branch 'origin/peppe/common_test/break_and_continue' into maint

* origin/peppe/common_test/break_and_continue:
  Fix error in documentation
  Implement support for test case execution break/continue

Conflicts:
	lib/common_test/src/ct.erl
	lib/common_test/src/ct_run.erl

OTP-10127
OTP-10172

5 files changed, 235 insertions, 86 deletions

diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl
index d5938c5db9..8d4721ab63 100644
--- a/lib/common_test/src/ct.erl
+++ b/lib/common_test/src/ct.erl
@@ -69,7 +69,8 @@
 	 fail/1, fail/2, comment/1, comment/2, make_priv_dir/0,
 	 testcases/2, userdata/2, userdata/3,
 	 timetrap/1, get_timetrap_info/0, sleep/1,
-	 notify/2, sync_notify/2]).
+	 notify/2, sync_notify/2,
+	 break/1, break/2, continue/0, continue/1]).
 
 %% New API for manipulating with config handlers
 -export([add_config/2, remove_config/2]).
@@ -155,7 +156,8 @@ run(TestDirs) ->
 %%%               {force_stop,Bool} | {decrypt,DecryptKeyOrFile} |
 %%%               {refresh_logs,LogDir} | {logopts,LogOpts} | 
 %%%               {verbosity,VLevels} | {basic_html,Bool} | 
-%%%               {ct_hooks, CTHs} | {enable_builtin_hooks,Bool}
+%%%               {ct_hooks, CTHs} | {enable_builtin_hooks,Bool} |
+%%%               {noinput,Bool}
 %%%   TestDirs = [string()] | string()
 %%%   Suites = [string()] | [atom()] | string() | atom()
 %%%   Cases = [atom()] | atom()
@@ -907,8 +909,9 @@ userdata(TestDir, Suite, Case) when is_atom(Case) ->
 %%%-----------------------------------------------------------------
 %%% @spec get_status() -> TestStatus | {error,Reason} | no_tests_running
 %%%       TestStatus = [StatusElem]
-%%%       StatusElem = {current,{Suite,TestCase}} | {successful,Successful} |
+%%%       StatusElem = {current,TestCaseInfo} | {successful,Successful} |
 %%%                    {failed,Failed} | {skipped,Skipped} | {total,Total}
+%%%       TestCaseInfo = {Suite,TestCase} | [{Suite,TestCase}]
 %%%       Suite = atom()
 %%%       TestCase = atom()
 %%%       Successful = integer()
@@ -920,7 +923,8 @@ userdata(TestDir, Suite, Case) when is_atom(Case) ->
 %%%       Reason = term()
 %%%
 %%% @doc Returns status of ongoing test. The returned list contains info about
-%%%      which test case is currently executing, as well as counters for 
+%%%      which test case is currently executing (a list of cases when a
+%%%      parallel test case group is executing), as well as counters for 
 %%%      successful, failed, skipped, and total test cases so far.
 get_status() ->
     case get_testdata(curr_tc) of
@@ -945,6 +949,8 @@ get_testdata(Key) ->
 	    Error;
 	{'EXIT',_Reason} ->
 	    no_tests_running;
+	[CurrTC] when Key == curr_tc ->
+	    {ok,CurrTC};
 	Data ->
 	    {ok,Data}
     end.
@@ -1138,3 +1144,74 @@ notify(Name,Data) ->
 %%% @see //stdlib/gen_event
 sync_notify(Name,Data) ->
     ct_event:sync_notify(Name, Data).
+
+%%%-----------------------------------------------------------------
+%%% @spec break(Comment) -> ok | {error,Reason}
+%%%       Comment = string()
+%%%       Reason = {multiple_cases_running,TestCases}
+%%%       TestCases = [atom()]
+%%%
+%%% @doc <p>This function will cancel all timetraps and pause the
+%%%       execution of the current test case until the user calls the
+%%%       <c>continue/0</c> function. It gives the user the opportunity
+%%%       to interact with the erlang node running the tests, e.g. for
+%%%       debugging purposes or for manually executing a part of the
+%%%       test case. If a parallel group is executing, <c>break/2</c>
+%%%       should be called instead.</p>
+break(Comment) ->
+    case get_testdata(curr_tc) of
+	{ok,{_,TestCase}} ->
+	    test_server:break(?MODULE, Comment);
+	{ok,Cases} when is_list(Cases) ->
+	    {error,{multiple_cases_running,
+		    [TC || {_,TC} <- Cases]}};
+	Error -> 
+	    {error,Error}
+    end.
+
+%%%-----------------------------------------------------------------
+%%% @spec break(TestCase, Comment) -> ok | {error,Reason}
+%%%       TestCase = atom()
+%%%       Comment = string()
+%%%       Reason = test_case_not_running
+%%%
+%%% @doc <p>This function works the same way as <c>break/1</c>,
+%%%       only the <c>TestCase</c> argument makes it possible to
+%%%       pause a test case executing in a parallel group. The
+%%%       <c>continue/1</c> function should be used to resume
+%%%       execution of <c>TestCase</c>.</p>
+break(TestCase, Comment) ->
+    case get_testdata(curr_tc) of
+	{ok,Cases} when is_list(Cases) ->
+	    case lists:keymember(TestCase, 2, Cases) of
+		true ->
+		    test_server:break(?MODULE, TestCase, Comment);
+		false ->
+		    {error,test_case_not_running}
+	    end;
+	{ok,{_,TestCase}} ->
+	    test_server:break(?MODULE, TestCase, Comment);
+	Error -> 
+	    {error,Error}
+    end.
+
+%%%-----------------------------------------------------------------
+%%% @spec continue() -> ok
+%%%
+%%% @doc <p>This function must be called in order to continue after a
+%%%      test case (not executing in a parallel group) has called
+%%%      <c>break/1</c>.</p>
+continue() -> 
+    test_server:continue().
+
+%%%-----------------------------------------------------------------
+%%% @spec continue(TestCase) -> ok
+%%%       TestCase = atom()
+%%%
+%%% @doc <p>This function must be called in order to continue after a
+%%%      test case has called <c>break/2</c>. If the paused test case,
+%%%      <c>TestCase</c>, executes in a parallel group, this
+%%%      function - rather than <c>continue/0</c> - must be used
+%%%      in order to let the test case proceed.</p>
+continue(TestCase) -> 
+    test_server:continue(TestCase).
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index 19a26872ee..d698ebb5dc 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -71,8 +71,13 @@ init_tc(Mod,Func,Config) ->
 	    {skip,{require_failed_in_suite0,Reason}};
 	{Suite,{suite0_failed,_}=Failure} ->
 	    {skip,Failure};
-	_ ->
-	    ct_util:set_testdata({curr_tc,{Suite,Func}}),
+	CurrTC ->
+	    case CurrTC of
+		undefined ->
+		    ct_util:set_testdata({curr_tc,[{Suite,Func}]});
+		Running when is_list(Running) ->
+		    ct_util:set_testdata({curr_tc,[{Suite,Func}|Running]})
+	    end,
 	    case ct_util:read_suite_data({seq,Suite,Func}) of
 		undefined ->
 		    init_tc1(Mod,Suite,Func,Config);
@@ -206,7 +211,8 @@ init_tc2(Mod,Suite,Func,SuiteInfo,MergeResult,Config) ->
     case catch configure(MergedInfo,MergedInfo,SuiteInfo,
 			 FuncSpec,[],Config) of
 	{suite0_failed,Reason} ->
-	    ct_util:set_testdata({curr_tc,{Mod,{suite0_failed,{require,Reason}}}}),
+	    ct_util:set_testdata({curr_tc,{Mod,{suite0_failed,
+						{require,Reason}}}}),
 	    {skip,{require_failed_in_suite0,Reason}};
 	{error,Reason} ->
 	    {auto_skip,{require_failed,Reason}};
@@ -656,7 +662,16 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) ->
     end,
 
     ct_util:reset_silent_connections(),
-    
+
+    %% reset the curr_tc state, or delete this TC from the list of
+    %% executing cases (if in a parallel group)
+    case ct_util:get_testdata(curr_tc) of
+	Running = [_,_|_] ->
+	    ct_util:set_testdata({curr_tc,lists:delete({Mod,Func}, Running)});
+	[_] ->
+	    ct_util:set_testdata({curr_tc,undefined})
+    end,
+
     case FinalResult of
 	{skip,{sequence_failed,_,_}} ->
 	    %% ct_logs:init_tc is never called for a skipped test case
diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl
index f942ce6f28..4b01532863 100644
--- a/lib/common_test/src/ct_logs.erl
+++ b/lib/common_test/src/ct_logs.erl
@@ -409,11 +409,11 @@ tc_print(Category,Importance,Format,Args) ->
     end.
 
 get_heading(default) ->
-    io_lib:format("-----------------------------"
+    io_lib:format("\n-----------------------------"
 		  "-----------------------\n~s\n",
 		  [log_timestamp(now())]);
 get_heading(Category) ->
-    io_lib:format("-----------------------------"
+    io_lib:format("\n-----------------------------"
 		  "-----------------------\n~s  ~w\n",
 		  [log_timestamp(now()),Category]).    
     
diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index 7c31f88f1f..1dbd9973cf 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -1619,69 +1619,111 @@ do_run(Tests, Skip, Opts, Args) when is_record(Opts, opts) ->
 			      "run ct:start_interactive()\n\n",[]),
 		    {error,interactive_mode};
 		_Pid ->
-		    %% save stylesheet info
-		    ct_util:set_testdata({stylesheet,Opts#opts.stylesheet}),
-		    %% save logopts
-		    ct_util:set_testdata({logopts,Opts#opts.logopts}),
-		    %% enable silent connections
-		    case Opts#opts.silent_connections of
-			[] ->
-			    Conns = ct_util:override_silence_all_connections(),
-			    ct_logs:log("Silent connections", "~p", [Conns]);
-			Conns when is_list(Conns) ->
-			    ct_util:override_silence_connections(Conns),
-			    ct_logs:log("Silent connections", "~p", [Conns]);
-			_ ->
-			    ok
-		    end,
-		    log_ts_names(Opts1#opts.testspecs),
-		    TestSuites = suite_tuples(Tests),
-
-		    {_TestSuites1,SuiteMakeErrors,AllMakeErrors} =
-			case application:get_env(common_test, auto_compile) of
-			    {ok,false} ->
-				{TestSuites1,SuitesNotFound} =
-				    verify_suites(TestSuites),
-				{TestSuites1,SuitesNotFound,SuitesNotFound};
-			    _ ->
-				{SuiteErrs,HelpErrs} = auto_compile(TestSuites),
-				{TestSuites,SuiteErrs,SuiteErrs++HelpErrs}
-			end,
+		    compile_and_run(Tests, Skip, Opts1, Args)
+	    end
+    end.
 
-		    case continue(AllMakeErrors) of
-			true ->
-			    SavedErrors = save_make_errors(SuiteMakeErrors),
-			    ct_repeat:log_loop_info(Args),
+compile_and_run(Tests, Skip, Opts, Args) ->
+    %% save stylesheet info
+    ct_util:set_testdata({stylesheet,Opts#opts.stylesheet}),
+    %% save logopts
+    ct_util:set_testdata({logopts,Opts#opts.logopts}),
+    %% enable silent connections
+    case Opts#opts.silent_connections of
+	[] ->
+	    Conns = ct_util:override_silence_all_connections(),
+	    ct_logs:log("Silent connections", "~p", [Conns]);
+	Conns when is_list(Conns) ->
+	    ct_util:override_silence_connections(Conns),
+	    ct_logs:log("Silent connections", "~p", [Conns]);
+	_ ->
+	    ok
+    end,
+    log_ts_names(Opts#opts.testspecs),
+    TestSuites = suite_tuples(Tests),
+    
+    {_TestSuites1,SuiteMakeErrors,AllMakeErrors} =
+	case application:get_env(common_test, auto_compile) of
+	    {ok,false} ->
+		{TestSuites1,SuitesNotFound} =
+		    verify_suites(TestSuites),
+		{TestSuites1,SuitesNotFound,SuitesNotFound};
+	    _ ->
+		{SuiteErrs,HelpErrs} = auto_compile(TestSuites),
+		{TestSuites,SuiteErrs,SuiteErrs++HelpErrs}
+	end,
+    
+    case continue(AllMakeErrors) of
+	true ->
+	    SavedErrors = save_make_errors(SuiteMakeErrors),
+	    ct_repeat:log_loop_info(Args),
+	    
+	    {Tests1,Skip1} = final_tests(Tests,Skip,SavedErrors),
+	    
+	    possibly_spawn(true == proplists:get_value(noinput, Args),
+			   Tests1, Skip1, Opts);
+	false ->
+	    io:nl(),
+	    ct_util:stop(clean),
+	    BadMods =
+		lists:foldr(
+		  fun({{_,_},Ms}, Acc) ->
+			  Ms ++ lists:foldl(
+				  fun(M, Acc1) ->
+					  lists:delete(M, Acc1)
+				  end, Acc, Ms)
+		  end, [], AllMakeErrors),
+	    {error,{make_failed,BadMods}}
+    end.
 
-			    {Tests1,Skip1} = final_tests(Tests,Skip,SavedErrors),
+%% keep the shell as the top controlling process
+possibly_spawn(false, Tests, Skip, Opts) ->	
+    TestResult = (catch do_run_test(Tests, Skip, Opts)),
+    case TestResult of
+	{EType,_} = Error when EType == user_error;
+			       EType == error ->
+	    ct_util:stop(clean),
+	    exit(Error);
+	_ ->
+	    ct_util:stop(normal),
+	    TestResult
+    end;
 
-			    R = (catch do_run_test(Tests1, Skip1,
-						   Opts1#opts{
-						     verbosity=Verbosity})),
-			    case R of
-				{EType,_} = Error when EType == user_error ;
+%% we must return control to the shell now, so we spawn
+%% a test supervisor process to keep an eye on the test run 
+possibly_spawn(true, Tests, Skip, Opts) ->
+    CTUtilSrv = whereis(ct_util_server),
+    Supervisor = 
+	fun() ->
+		process_flag(trap_exit, true),
+		link(CTUtilSrv),
+		TestRun =
+		    fun() ->
+			    TestResult = (catch do_run_test(Tests, Skip, Opts)),
+			    case TestResult of
+				{EType,_} = Error when EType == user_error;
 						       EType == error ->
 				    ct_util:stop(clean),
 				    exit(Error);
 				_ ->
 				    ct_util:stop(normal),
-				    TestResult
-			    end;
-			false ->
-			    io:nl(),
-			    ct_util:stop(clean),
-			    BadMods =
-				lists:foldr(
-				  fun({{_,_},Ms}, Acc) ->
-					  Ms ++ lists:foldl(
-						  fun(M, Acc1) ->
-							  lists:delete(M, Acc1)
-						  end, Acc, Ms)
-				  end, [], AllMakeErrors),
-			    {error,{make_failed,BadMods}}
-		    end
-	    end
-    end.
+				    exit({ok,TestResult})
+			    end
+		    end,
+		TestRunPid = spawn_link(TestRun),
+		receive
+		    {'EXIT',TestRunPid,{ok,TestResult}} ->
+			io:format(user, "~nCommon Test returned ~p~n~n",
+				  [TestResult]);
+		    {'EXIT',TestRunPid,Error} ->
+			exit(Error)				
+		end
+	end,
+    unlink(CTUtilSrv),
+    SupPid = spawn(Supervisor),
+    io:format(user, "~nTest control handed over to process ~p~n~n",
+	      [SupPid]),
+    SupPid.
 
 %% attempt to compile the modules specified in TestSuites
 auto_compile(TestSuites) ->
@@ -2128,7 +2170,7 @@ do_run_test(Tests, Skip, Opts) ->
 		    {error,test_result_unknown}
 	    end;
 	Error ->
-	    Error
+	    exit(Error)
     end.
 
 delete_dups([S | Suites]) ->
diff --git a/lib/test_server/src/test_server.erl b/lib/test_server/src/test_server.erl
index 1b94ada6b4..0dc3ce48ab 100644
--- a/lib/test_server/src/test_server.erl
+++ b/lib/test_server/src/test_server.erl
@@ -50,7 +50,7 @@
 -export([run_on_shielded_node/2]).
 -export([is_cover/0,is_debug/0,is_commercial/0]).
 
--export([break/1,continue/0]).
+-export([break/1,break/2,break/3,continue/0,continue/1]).
 
 %%% DEBUGGER INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 -export([purify_new_leaks/0, purify_format/2, purify_new_fds_inuse/0,
@@ -2102,31 +2102,40 @@ fail() ->
 %% Break a test case so part of the test can be done manually.
 %% Use continue/0 to continue.
 break(Comment) ->
-    case erase(test_server_timetraps) of
-	undefined -> ok;
-	List -> lists:foreach(fun({Ref,_,_}) -> 
-				      timetrap_cancel(Ref)
-			      end, List)
-    end,
+    break(?MODULE, Comment).
+
+break(CBM, Comment) ->
+    break(CBM, '', Comment).
+
+break(CBM, TestCase, Comment) ->
+    timetrap_cancel(),
+    {TCName,CntArg,PName} =
+	if TestCase == '' -> 
+		{"", "", test_server_break_process};
+	   true ->
+		Str = atom_to_list(TestCase),
+		{[32 | Str], Str,
+		 list_to_atom("test_server_break_process_" ++ Str)}
+	end,
     io:format(user,
 	      "\n\n\n--- SEMIAUTOMATIC TESTING ---"
-	      "\nThe test case executes on process ~w"
+	      "\nThe test case~s executes on process ~w"
 	      "\n\n\n~s"
 	      "\n\n\n-----------------------------\n\n"
-	      "Continue with --> test_server:continue().\n",
-	      [self(),Comment]),
-    case whereis(test_server_break_process) of
+	      "Continue with --> ~w:continue(~s).\n",
+	      [TCName,self(),Comment,CBM,CntArg]),
+    case whereis(PName) of
 	undefined ->
-	    spawn_break_process(self());
+	    spawn_break_process(self(), PName);
 	OldBreakProcess ->
 	    OldBreakProcess ! cancel,
-	    spawn_break_process(self())
+	    spawn_break_process(self(), PName)
     end,
     receive continue -> ok end.
 
-spawn_break_process(Pid) ->
+spawn_break_process(Pid, PName) ->
     spawn(fun() ->
-		  register(test_server_break_process,self()),
+		  register(PName, self()),
 		  receive
 		      continue -> continue(Pid);
 		      cancel -> ok
@@ -2135,13 +2144,19 @@ spawn_break_process(Pid) ->
 
 continue() ->
     case whereis(test_server_break_process) of
-	undefined ->
-	     ok;
-	BreakProcess ->
-	    BreakProcess ! continue
+	undefined    -> ok;
+	BreakProcess -> BreakProcess ! continue
     end.
 
-continue(Pid) ->
+continue(TestCase) when is_atom(TestCase) ->
+    PName = list_to_atom("test_server_break_process_" ++
+			 atom_to_list(TestCase)),
+    case whereis(PName) of
+	undefined    -> ok;
+	BreakProcess -> BreakProcess ! continue
+    end;	
+
+continue(Pid) when is_pid(Pid) ->
     Pid ! continue.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%