Make it possible to execute one test run per test specification

OTP-9881

2 files changed, 194 insertions, 145 deletions

diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index eb05c90ba8..50723a763c 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -378,7 +378,8 @@ run_or_refresh(StartOpts = #opts{logdir = LogDir}, Args) ->
 			    {error,{all_suites_index,ASReason}};
 			_ ->
 			    file:set_cwd(Cwd),
-			    io:format("Logs in ~s refreshed!~n~n", [LogDir1]),
+			    io:format("Logs in ~s refreshed!~n~n",
+				      [LogDir1]),
 			    timer:sleep(500), % time to flush io before quitting
 			    ok
 		    end
@@ -387,145 +388,29 @@ run_or_refresh(StartOpts = #opts{logdir = LogDir}, Args) ->
 
 script_start2(StartOpts = #opts{vts = undefined,
 				shell = undefined}, Args) ->
-    TestSpec = proplists:get_value(spec, Args),
-    {Terms,Opts} =
-	case TestSpec of
-	    Specs when Specs =/= [], Specs =/= undefined ->
-		%% using testspec as input for test
-		Relaxed = get_start_opt(allow_user_terms, true, false, Args),
-		case catch ct_testspec:collect_tests_from_file(Specs, Relaxed) of
-		    {E,Reason} when E == error ; E == 'EXIT' ->
-			{{error,Reason},StartOpts};
-		    TS ->
-			SpecStartOpts = get_data_for_node(TS, node()),
-
-			Label = choose_val(StartOpts#opts.label,
-					   SpecStartOpts#opts.label),
-
-			Profile = choose_val(StartOpts#opts.profile,
-					     SpecStartOpts#opts.profile),
-
-			LogDir = choose_val(StartOpts#opts.logdir,
-					    SpecStartOpts#opts.logdir),
-
-			AllLogOpts = merge_vals([StartOpts#opts.logopts,
-						 SpecStartOpts#opts.logopts]),
-			AllVerbosity =
-			    merge_keyvals([StartOpts#opts.verbosity,
-					   SpecStartOpts#opts.verbosity]),
-			AllSilentConns =
-			    merge_vals([StartOpts#opts.silent_connections,
-					SpecStartOpts#opts.silent_connections]),
-			Cover =
-			    choose_val(StartOpts#opts.cover,
-				       SpecStartOpts#opts.cover),
-			CoverStop =
-			    choose_val(StartOpts#opts.cover_stop,
-				       SpecStartOpts#opts.cover_stop),
-			MultTT =
-			    choose_val(StartOpts#opts.multiply_timetraps,
-				       SpecStartOpts#opts.multiply_timetraps),
-			ScaleTT =
-			    choose_val(StartOpts#opts.scale_timetraps,
-				       SpecStartOpts#opts.scale_timetraps),
-
-			CreatePrivDir =
-			    choose_val(StartOpts#opts.create_priv_dir,
-				       SpecStartOpts#opts.create_priv_dir),
-
-			AllEvHs = 
-			    merge_vals([StartOpts#opts.event_handlers,
-					SpecStartOpts#opts.event_handlers]),
-
-			AllCTHooks = merge_vals(
-					[StartOpts#opts.ct_hooks,
-					 SpecStartOpts#opts.ct_hooks]),
-
-			EnableBuiltinHooks =
-			    choose_val(
-			      StartOpts#opts.enable_builtin_hooks,
-			      SpecStartOpts#opts.enable_builtin_hooks),
-			
-			Stylesheet =
-			    choose_val(StartOpts#opts.stylesheet,
-				       SpecStartOpts#opts.stylesheet),
-			    
-			AllInclude = merge_vals([StartOpts#opts.include,
-						 SpecStartOpts#opts.include]),
-			application:set_env(common_test, include, AllInclude),
-			
-			AutoCompile =
-			    case choose_val(StartOpts#opts.auto_compile,
-					    SpecStartOpts#opts.auto_compile) of
-				undefined ->
-				    true;
-				ACBool ->
-				    application:set_env(common_test,
-							auto_compile,
-							ACBool),
-				    ACBool
-			    end,
-
-			BasicHtml =
-			    case choose_val(StartOpts#opts.basic_html,
-					    SpecStartOpts#opts.basic_html) of
-				undefined ->
-				    false;
-				BHBool ->
-				    application:set_env(common_test, basic_html, 
-							BHBool),
-				    BHBool
-			    end,
-
-			{TS,StartOpts#opts{label = Label,
-					   profile = Profile,
-					   testspecs = Specs,
-					   cover = Cover,
-					   cover_stop = CoverStop,
-					   logdir = LogDir,
-					   logopts = AllLogOpts,
-					   basic_html = BasicHtml,
-					   verbosity = AllVerbosity,
-					   silent_connections = AllSilentConns,
-					   config = SpecStartOpts#opts.config,
-					   event_handlers = AllEvHs,
-					   ct_hooks = AllCTHooks,
-					   enable_builtin_hooks =
-					       EnableBuiltinHooks,
-					   stylesheet = Stylesheet,
-					   auto_compile = AutoCompile,
-					   include = AllInclude,
-					   multiply_timetraps = MultTT,
-					   scale_timetraps = ScaleTT,
-					   create_priv_dir = CreatePrivDir}}
-		end;
-	    _ ->
-		{undefined,StartOpts}
-	end,
-    %% read config/userconfig from start flags
-    InitConfig = ct_config:prepare_config_list(Args),
-    TheLogDir = which(logdir, Opts#opts.logdir),
-    case {TestSpec,Terms} of
-	{_,{error,_}=Error} ->
-	    Error;
-	{[],_} ->
+    case proplists:get_value(spec, Args) of
+	Specs when Specs =/= [], Specs =/= undefined ->
+	    Specs1 = get_start_opt(join_specs, [Specs], Specs, Args),
+	    %% using testspec as input for test
+	    Relaxed = get_start_opt(allow_user_terms, true, false, Args),
+	    case catch ct_testspec:collect_tests_from_file(Specs1, Relaxed) of
+		{E,Reason} when E == error ; E == 'EXIT' ->
+		    {error,Reason};
+		TestSpecData ->
+		    execute_testspecs(TestSpecData, StartOpts, Args, [])
+	    end;
+	[] ->
 	    {error,no_testspec_specified};
-	{undefined,_} ->   % no testspec used
-	    case check_and_install_configfiles(InitConfig, TheLogDir, Opts) of
+	_ ->	    % no testspec used
+	    %% read config/userconfig from start flags
+	    InitConfig = ct_config:prepare_config_list(Args),
+	    TheLogDir = which(logdir, StartOpts#opts.logdir),
+	    case check_and_install_configfiles(InitConfig,
+					       TheLogDir,
+					       StartOpts) of
 		ok ->      % go on read tests from start flags
-		    script_start3(Opts#opts{config=InitConfig,
-					    logdir=TheLogDir}, Args);
-		Error ->
-		    Error
-	    end;
-	{_,_} ->           % testspec used
-	    %% merge config from start flags with config from testspec
-	    AllConfig = merge_vals([InitConfig, Opts#opts.config]),
-	    case check_and_install_configfiles(AllConfig, TheLogDir, Opts) of
-		ok ->      % read tests from spec
-		    {Run,Skip} = ct_testspec:prepare_tests(Terms, node()),
-		    do_run(Run, Skip, Opts#opts{config=AllConfig,
-						logdir=TheLogDir}, Args);
+		    script_start3(StartOpts#opts{config=InitConfig,
+						 logdir=TheLogDir}, Args);
 		Error ->
 		    Error
 	    end
@@ -543,6 +428,152 @@ script_start2(StartOpts, Args) ->
 	    Error
     end.
 
+execute_testspecs([], _, _, Result) ->
+    Result1 = lists:reverse(Result),
+    case lists:keysearch('EXIT', 1, Result1) of
+	{value,{_,_,ExitReason}} ->
+	    exit(ExitReason);
+	false ->
+	    case lists:keysearch(error, 1, Result1) of
+		{value,Error} ->
+		    Error;
+		false ->
+		    lists:foldl(fun({Ok,Fail,{UserSkip,AutoSkip}},
+				    {Ok1,Fail1,{UserSkip1,AutoSkip1}}) ->
+					{Ok1+Ok,Fail1+Fail,
+					 {UserSkip1+UserSkip,
+					  AutoSkip1+AutoSkip}}
+				end, {0,0,{0,0}}, Result1)
+	    end
+    end;
+
+execute_testspecs([{Specs,TS} | TSs], StartOpts, Args, Result) ->    
+    SpecStartOpts = get_data_for_node(TS, node()),
+    
+    Label = choose_val(StartOpts#opts.label,
+		       SpecStartOpts#opts.label),
+    
+    Profile = choose_val(StartOpts#opts.profile,
+			 SpecStartOpts#opts.profile),
+    
+    LogDir = choose_val(StartOpts#opts.logdir,
+			SpecStartOpts#opts.logdir),
+    
+    AllLogOpts = merge_vals([StartOpts#opts.logopts,
+			     SpecStartOpts#opts.logopts]),
+    AllVerbosity =
+	merge_keyvals([StartOpts#opts.verbosity,
+		       SpecStartOpts#opts.verbosity]),
+    AllSilentConns =
+	merge_vals([StartOpts#opts.silent_connections,
+		    SpecStartOpts#opts.silent_connections]),
+    Cover =
+	choose_val(StartOpts#opts.cover,
+		   SpecStartOpts#opts.cover),
+    CoverStop =
+	choose_val(StartOpts#opts.cover_stop,
+		   SpecStartOpts#opts.cover_stop),
+    MultTT =
+	choose_val(StartOpts#opts.multiply_timetraps,
+		   SpecStartOpts#opts.multiply_timetraps),
+    ScaleTT =
+	choose_val(StartOpts#opts.scale_timetraps,
+		   SpecStartOpts#opts.scale_timetraps),
+    
+    CreatePrivDir =
+	choose_val(StartOpts#opts.create_priv_dir,
+		   SpecStartOpts#opts.create_priv_dir),
+    
+    AllEvHs = 
+	merge_vals([StartOpts#opts.event_handlers,
+		    SpecStartOpts#opts.event_handlers]),
+    
+    AllCTHooks = merge_vals(
+		   [StartOpts#opts.ct_hooks,
+		    SpecStartOpts#opts.ct_hooks]),
+    
+    EnableBuiltinHooks =
+	choose_val(
+	  StartOpts#opts.enable_builtin_hooks,
+	  SpecStartOpts#opts.enable_builtin_hooks),
+    
+    Stylesheet =
+	choose_val(StartOpts#opts.stylesheet,
+		   SpecStartOpts#opts.stylesheet),
+    
+    AllInclude = merge_vals([StartOpts#opts.include,
+			     SpecStartOpts#opts.include]),
+    application:set_env(common_test, include, AllInclude),
+    
+    AutoCompile =
+	case choose_val(StartOpts#opts.auto_compile,
+			SpecStartOpts#opts.auto_compile) of
+	    undefined ->
+		true;
+	    ACBool ->
+		application:set_env(common_test,
+				    auto_compile,
+				    ACBool),
+		ACBool
+	end,
+
+    BasicHtml =
+	case choose_val(StartOpts#opts.basic_html,
+			SpecStartOpts#opts.basic_html) of
+	    undefined ->
+		false;
+	    BHBool ->
+		application:set_env(common_test, basic_html, 
+				    BHBool),
+		BHBool
+	end,
+    
+    Opts = StartOpts#opts{label = Label,
+			  profile = Profile,
+			  testspecs = Specs,
+			  cover = Cover,
+			  cover_stop = CoverStop,
+			  logdir = LogDir,
+			  logopts = AllLogOpts,
+			  basic_html = BasicHtml,
+			  verbosity = AllVerbosity,
+			  silent_connections = AllSilentConns,
+			  config = SpecStartOpts#opts.config,
+			  event_handlers = AllEvHs,
+			  ct_hooks = AllCTHooks,
+			  enable_builtin_hooks =
+			      EnableBuiltinHooks,
+			  stylesheet = Stylesheet,
+			  auto_compile = AutoCompile,
+			  include = AllInclude,
+			  multiply_timetraps = MultTT,
+			  scale_timetraps = ScaleTT,
+			  create_priv_dir = CreatePrivDir},
+
+    try execute_testspec(TS, Opts, Args) of
+	ExecResult ->
+	    execute_testspecs(TSs, StartOpts, Args, [ExecResult | Result])
+    catch
+	_ : ExitReason ->
+	    execute_testspecs(TSs, StartOpts, Args,
+			      [{'EXIT',self(),ExitReason} | Result])
+    end.
+	
+execute_testspec(TS, Opts, Args) ->
+    %% read config/userconfig from start flags
+    InitConfig = ct_config:prepare_config_list(Args),
+    TheLogDir = which(logdir, Opts#opts.logdir),
+    %% merge config from start flags with config from testspec
+    AllConfig = merge_vals([InitConfig, Opts#opts.config]),
+    case check_and_install_configfiles(AllConfig, TheLogDir, Opts) of
+	ok ->      % read tests from spec
+	    {Run,Skip} = ct_testspec:prepare_tests(TS, node()),
+	    do_run(Run, Skip, Opts#opts{config=AllConfig,
+					logdir=TheLogDir}, Args);
+	Error ->
+	    Error
+    end.
+
 check_and_install_configfiles(
   Configs, LogDir, #opts{
 	     event_handlers = EvHandlers,
diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl
index 202d8f9373..9a7a384ebb 100644
--- a/lib/common_test/src/ct_testspec.erl
+++ b/lib/common_test/src/ct_testspec.erl
@@ -246,22 +246,40 @@ collect_tests_from_file(Specs, Relaxed) ->
 
 collect_tests_from_file(Specs,Nodes,Relaxed) when is_list(Nodes) ->
     NodeRefs = lists:map(fun(N) -> {undefined,N} end, Nodes),
-    catch collect_tests_from_file1(Specs,#testspec{nodes=NodeRefs},Relaxed).
-
-collect_tests_from_file1([Spec|Specs],TestSpec,Relaxed) ->
+    %% [Spec1,Spec2,...] means create one testpec record per Spec file
+    %% [[Spec1,Spec2,...]] means merge all specs into one testspec record
+    {MergeSpecs,Specs1} = if is_list(hd(hd(Specs))) -> {true,hd(Specs)};
+		    true -> {false,Specs}
+		 end,
+    catch create_specs(Specs1,Specs1,#testspec{nodes=NodeRefs},
+		       Relaxed,MergeSpecs,[]).
+
+create_specs([Spec|Ss],Specs,TestSpec,Relaxed,MergeSpecs,Saved) ->
     SpecDir = filename:dirname(filename:absname(Spec)),
     case file:consult(Spec) of
 	{ok,Terms} ->	    
 	    case collect_tests(Terms,
 			       TestSpec#testspec{spec_dir=SpecDir},
 			       Relaxed) of
-		TS = #testspec{tests=Tests, logdir=LogDirs} when Specs == [] ->
+		TS = #testspec{tests=Tests, logdir=LogDirs} when
+		      Ss == [], MergeSpecs == true ->
+		    LogDirs1 = lists:delete(".",LogDirs) ++ ["."],
+		    [{Specs,TS#testspec{tests=lists:flatten(Tests),
+					logdir=LogDirs1}}];
+		TS = #testspec{tests=Tests, logdir=LogDirs} when
+		      Ss == [], MergeSpecs == false ->
 		    LogDirs1 = lists:delete(".",LogDirs) ++ ["."],
-		    TS#testspec{tests=lists:flatten(Tests), logdir=LogDirs1};  
-		TS = #testspec{alias = As, nodes = Ns} ->
+		    TSRet = {[Spec],TS#testspec{tests=lists:flatten(Tests),
+						logdir=LogDirs1}},
+		    lists:reverse([TSRet|Saved]);
+		TS = #testspec{alias = As, nodes = Ns} when
+		      MergeSpecs == true ->
 		    TS1 = TS#testspec{alias = lists:reverse(As),
 				      nodes = lists:reverse(Ns)},
-		    collect_tests_from_file1(Specs,TS1,Relaxed)
+		    create_specs(Ss,Specs,TS1,Relaxed,MergeSpecs,[]);
+		TS when MergeSpecs == false ->
+		    create_specs(Ss,Specs,TestSpec,Relaxed,MergeSpecs,
+				[{[Spec],TS}|Saved])
 	    end;
 	{error,Reason} ->
 	    ReasonStr =