Fix problem with tests running in wrong order

OTP-9881

2 files changed, 98 insertions, 69 deletions

diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl
index abd82a3176..4f3597dffb 100644
--- a/lib/common_test/src/ct_testspec.erl
+++ b/lib/common_test/src/ct_testspec.erl
@@ -28,7 +28,6 @@
 	 collect_tests_from_file/2, collect_tests_from_file/3]).
 
 -include("ct_util.hrl").
-
 -define(testspec_fields, record_info(fields, testspec)).
 
 %%%------------------------------------------------------------------
@@ -295,7 +294,7 @@ delete_dups1([],Keep) ->
     Keep.
 
 create_specs(Specs,TestSpec,Relaxed,Join) ->
-    SpecsTree = create_spec_tree(Specs,TestSpec,Join,[]),    
+    SpecsTree = create_spec_tree(Specs,TestSpec,Join,[]),   
     create_specs(SpecsTree,TestSpec,Relaxed).
 
 create_spec_tree([Spec|Specs],TS,JoinWithNext,Known) ->
@@ -328,9 +327,11 @@ create_spec_tree([],_TS,_JoinWithNext,_Known) ->
 create_specs({Spec,Terms,InclJoin,InclSep,JoinWithNext,NextSpec},
 	     TestSpec,Relaxed) ->
     SpecDir = filename:dirname(filename:absname(Spec)),
-    TestSpec1 = create_spec(Terms,TestSpec#testspec{spec_dir=SpecDir},Relaxed),
+    TestSpec1 = create_spec(Terms,TestSpec#testspec{spec_dir=SpecDir},
+			    JoinWithNext,Relaxed),
 
     {{JoinSpecs1,JoinTS1},Separate1} = create_specs(InclJoin,TestSpec1,Relaxed),
+
     {{JoinSpecs2,JoinTS2},Separate2} =
 	case JoinWithNext of
 	    true ->
@@ -351,7 +352,7 @@ create_specs({Spec,Terms,InclJoin,InclSep,JoinWithNext,NextSpec},
     AllSeparate =
 	[TSData || TSData = {Ss,_TS} <- Separate3++Separate1++
 		                        [SepJoinSpecs]++Separate2++
-		                        Separate4++[SepJoinSpecs1],
+		                        [SepJoinSpecs1]++Separate4,
 		   Ss /= []],
 
     case {JoinWithNext,JoinSpecs1} of
@@ -368,9 +369,15 @@ create_specs({Spec,Terms,InclJoin,InclSep,JoinWithNext,NextSpec},
 create_specs([],TestSpec,_Relaxed) ->
     {{[],TestSpec},[]}.
 		
-create_spec(Terms,TestSpec,Relaxed) ->
+create_spec(Terms,TestSpec,JoinedByPrev,Relaxed) ->
+    %% it's the "includer" that decides the value of merge_tests
+    Terms1 = if not JoinedByPrev ->
+		     [{set_merge_tests,true}|Terms];
+		true ->
+		     [{set_merge_tests,false}|Terms]
+	     end,
     TS = #testspec{tests=Tests, logdir=LogDirs} =
-	collect_tests({false,Terms},TestSpec,Relaxed),
+	collect_tests({false,Terms1},TestSpec,Relaxed),
     LogDirs1 = lists:delete(".",LogDirs) ++ ["."],
     TS#testspec{tests=lists:flatten(Tests),
 		logdir=LogDirs1}.
@@ -397,14 +404,23 @@ collect_tests({Replace,Terms},TestSpec=#testspec{alias=As,nodes=Ns},Relaxed) ->
     Terms1 = if Replace -> replace_names(Terms);
 		true    -> Terms
 	     end,
+    {MergeTestsDef,Terms2} =
+	case proplists:get_value(set_merge_tests,Terms1,true) of
+	    false -> 
+		%% disable merge_tests
+		{TestSpec#testspec.merge_tests,
+		 proplists:delete(merge_tests,Terms1)};
+	    true ->
+		{true,Terms1}
+	end,
     %% reverse nodes and aliases initially to get the order of them right
     %% in case this spec is being joined with a previous one
-    TestSpec1 = get_global(Terms1,TestSpec#testspec{alias = lists:reverse(As),
+    TestSpec1 = get_global(Terms2,TestSpec#testspec{alias = lists:reverse(As),
 						    nodes = lists:reverse(Ns),
-						    merge_tests = true}),
-    TestSpec2 = get_all_nodes(Terms1,TestSpec1),
-    {Terms2, TestSpec3} = filter_init_terms(Terms1, [], TestSpec2),
-    add_tests(Terms2,TestSpec3).
+						    merge_tests = MergeTestsDef}),
+    TestSpec2 = get_all_nodes(Terms2,TestSpec1),
+    {Terms3, TestSpec3} = filter_init_terms(Terms2, [], TestSpec2),
+    add_tests(Terms3,TestSpec3).
 
 %% replace names (atoms) in the testspec matching those in 'define' terms by
 %% searching recursively through tuples and lists
@@ -915,6 +931,9 @@ add_tests([{release_shell,Bool}|Ts],Spec) ->
     add_tests(Ts, Spec#testspec{release_shell = Bool});
 
 %% --- handled/errors ---
+add_tests([{set_merge_tests,_}|Ts],Spec) ->	% internal
+    add_tests(Ts,Spec);
+
 add_tests([{define,_,_}|Ts],Spec) ->		% handled
     add_tests(Ts,Spec);
 
@@ -924,10 +943,10 @@ add_tests([{alias,_,_}|Ts],Spec) ->		% handled
 add_tests([{node,_,_}|Ts],Spec) ->		% handled
     add_tests(Ts,Spec);
 
-add_tests([{merge_tests,_} | Ts], Spec) ->     % handled
+add_tests([{merge_tests,_} | Ts], Spec) ->      % handled
     add_tests(Ts,Spec);
 
-add_tests([{specs,_,_} | Ts], Spec) ->     % handled
+add_tests([{specs,_,_} | Ts], Spec) ->          % handled
     add_tests(Ts,Spec);
 
 %%     --------------------------------------------------
@@ -1001,7 +1020,18 @@ add_tests([],Spec) ->				% done
 %% check if it's a CT term that has bad format or if the user seems to
 %% have added something of his/her own, which we'll let pass if relaxed
 %% mode is enabled.
-check_term(Term) ->
+check_term(Atom) when is_atom(Atom) ->
+    Valid = valid_terms(),
+    case lists:member(Atom,Valid) of
+	true ->
+	    valid;
+	false ->				% ignore
+	    case get(relaxed) of
+		true -> invalid;
+		false -> throw({error,{undefined_term_in_spec,Atom}})
+	    end
+    end;
+check_term(Term) when is_tuple(Term) ->
     Size = size(Term),
     [Name|_] = tuple_to_list(Term),
     Valid = valid_terms(),
@@ -1029,7 +1059,9 @@ check_term(Term) ->
 			    throw({error,{undefined_term_in_spec,Term}})
 		    end
 	    end
-    end.
+    end;
+check_term(Other) ->
+    throw({error,{undefined_term_in_spec,Other}}).
 
 %% specific data handling before saving in testspec record, e.g.
 %% converting relative paths to absolute for directories and files
@@ -1421,6 +1453,7 @@ is_node([],_) ->
 
 valid_terms() ->
     [
+     {set_merge_tests,2},
      {define,3},
      {specs,3},
      {node,3},
diff --git a/lib/common_test/test/ct_testspec_3_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE.erl
index 8b84b563ab..e2f57c2c64 100644
--- a/lib/common_test/test/ct_testspec_3_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_3_SUITE.erl
@@ -331,16 +331,16 @@ test_events(start_join) ->
      {?eh,test_stats,{1,2,{1,1}}},
      {?eh,tc_start,{t11_SUITE,end_per_suite}},
      {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
-     {?eh,tc_start,{t21_SUITE,init_per_suite}},
-     {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
-     {?eh,test_stats,{2,4,{2,2}}},
-     {?eh,tc_start,{t21_SUITE,end_per_suite}},
-     {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
      {?eh,tc_start,{t12_SUITE,init_per_suite}},
      {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
-     {?eh,test_stats,{3,6,{3,3}}},
+     {?eh,test_stats,{2,4,{2,2}}},
      {?eh,tc_start,{t12_SUITE,end_per_suite}},
      {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+     {?eh,tc_start,{t21_SUITE,init_per_suite}},
+     {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+     {?eh,test_stats,{3,6,{3,3}}},
+     {?eh,tc_start,{t21_SUITE,end_per_suite}},
+     {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
      {?eh,tc_start,{t22_SUITE,init_per_suite}},
      {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
      {?eh,test_stats,{4,8,{4,4}}},
@@ -520,8 +520,7 @@ test_events(incl_separate2) ->
      {?eh,stop_logging,[]}];
 
 test_events(incl_join1) ->
-    [
-     {?eh,start_logging,{'DEF','RUNDIR'}},
+    [{?eh,start_logging,{'DEF','RUNDIR'}},
      {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
      {?eh,start_info,{4,4,20}},
      {?eh,tc_start,{t12_SUITE,init_per_suite}},
@@ -549,61 +548,58 @@ test_events(incl_join1) ->
      {?eh,start_logging,{'DEF','RUNDIR'}},
      {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
      {?eh,start_info,{4,4,20}},
+     {?eh,tc_start,{t22_SUITE,init_per_suite}},
+     {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+     {?eh,test_stats,{1,2,{1,1}}},
+     {?eh,tc_start,{t22_SUITE,end_per_suite}},
+     {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
      {?eh,tc_start,{t21_SUITE,init_per_suite}},
      {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
-     {?eh,test_stats,{1,2,{1,1}}},
+     {?eh,test_stats,{2,4,{2,2}}},
      {?eh,tc_start,{t21_SUITE,end_per_suite}},
      {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
      {?eh,tc_start,{t11_SUITE,init_per_suite}},
      {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
-     {?eh,test_stats,{2,4,{2,2}}},
+     {?eh,test_stats,{3,6,{3,3}}},
      {?eh,tc_start,{t11_SUITE,end_per_suite}},
      {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
      {?eh,tc_start,{t12_SUITE,init_per_suite}},
      {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
-     {?eh,test_stats,{3,6,{3,3}}},
+     {?eh,test_stats,{4,8,{4,4}}},
      {?eh,tc_start,{t12_SUITE,end_per_suite}},
      {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
-     {?eh,tc_start,{t22_SUITE,init_per_suite}},
-     {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
-     {?eh,test_stats,{4,8,{4,4}}},
-     {?eh,tc_start,{t22_SUITE,end_per_suite}},
-     {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
      {?eh,test_done,{'DEF','STOP_TIME'}},
      {?eh,stop_logging,[]}];
 
 test_events(incl_join2) ->
-    [
-     {?eh,start_logging,{'DEF','RUNDIR'}},
+    [{?eh,start_logging,{'DEF','RUNDIR'}},
      {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
      {?eh,start_info,{4,4,20}},
-     {?eh,tc_start,{t11_SUITE,init_per_suite}},
-     {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
-     {?eh,tc_start,{t11_SUITE,ok_tc}},
-     {?eh,test_stats,{1,2,{1,1}}},
-     {?eh,tc_start,{t11_SUITE,end_per_suite}},
-     {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
-     {?eh,tc_start,{t21_SUITE,init_per_suite}},
-     {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
-     {?eh,test_stats,{2,4,{2,2}}},
-     {?eh,tc_start,{t21_SUITE,end_per_suite}},
-     {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
      {?eh,tc_start,{t12_SUITE,init_per_suite}},
      {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
-     {?eh,test_stats,{3,6,{3,3}}},
+     {?eh,test_stats,{1,2,{1,1}}},
      {?eh,tc_start,{t12_SUITE,end_per_suite}},
      {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+     {?eh,tc_start,{t11_SUITE,init_per_suite}},
+     {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+     {?eh,test_stats,{2,4,{2,2}}},
+     {?eh,tc_start,{t11_SUITE,end_per_suite}},
+     {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
      {?eh,tc_start,{t22_SUITE,init_per_suite}},
      {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
-     {?eh,test_stats,{4,8,{4,4}}},
+     {?eh,test_stats,{3,6,{3,3}}},
      {?eh,tc_start,{t22_SUITE,end_per_suite}},
      {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+     {?eh,tc_start,{t21_SUITE,init_per_suite}},
+     {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+     {?eh,test_stats,{4,8,{4,4}}},
+     {?eh,tc_start,{t21_SUITE,end_per_suite}},
+     {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
      {?eh,test_done,{'DEF','STOP_TIME'}},
      {?eh,stop_logging,[]}];
 
 test_events(incl_both1) ->
-    [
-     {?eh,start_logging,{'DEF','RUNDIR'}},
+    [{?eh,start_logging,{'DEF','RUNDIR'}},
      {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
      {?eh,start_info,{2,2,10}},
      {?eh,tc_start,{t11_SUITE,init_per_suite}},
@@ -641,21 +637,6 @@ test_events(incl_both1) ->
      {?eh,start_logging,{'DEF','RUNDIR'}},
      {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
      {?eh,start_info,{2,2,10}},
-     {?eh,tc_start,{t11_SUITE,init_per_suite}},
-     {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
-     {?eh,test_stats,{1,2,{1,1}}},
-     {?eh,tc_start,{t11_SUITE,end_per_suite}},
-     {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
-     {?eh,tc_start,{t21_SUITE,init_per_suite}},
-     {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
-     {?eh,test_stats,{2,4,{2,2}}},
-     {?eh,tc_start,{t21_SUITE,end_per_suite}},
-     {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
-     {?eh,test_done,{'DEF','STOP_TIME'}},
-     {?eh,stop_logging,[]},
-     {?eh,start_logging,{'DEF','RUNDIR'}},
-     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
-     {?eh,start_info,{2,2,10}},
      {?eh,tc_start,{t12_SUITE,init_per_suite}},
      {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
      {?eh,test_stats,{1,2,{1,1}}},
@@ -667,13 +648,10 @@ test_events(incl_both1) ->
      {?eh,tc_start,{t22_SUITE,end_per_suite}},
      {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
      {?eh,test_done,{'DEF','STOP_TIME'}},
-     {?eh,stop_logging,[]}];
-
-test_events(incl_both2) ->
-    [
+     {?eh,stop_logging,[]},
      {?eh,start_logging,{'DEF','RUNDIR'}},
      {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
-     {?eh,start_info,{4,4,20}},
+     {?eh,start_info,{2,2,10}},
      {?eh,tc_start,{t11_SUITE,init_per_suite}},
      {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
      {?eh,test_stats,{1,2,{1,1}}},
@@ -684,11 +662,29 @@ test_events(incl_both2) ->
      {?eh,test_stats,{2,4,{2,2}}},
      {?eh,tc_start,{t21_SUITE,end_per_suite}},
      {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+     {?eh,test_done,{'DEF','STOP_TIME'}},
+     {?eh,stop_logging,[]}];
+
+test_events(incl_both2) ->
+    [{?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+     {?eh,start_info,{4,4,20}},
+     {?eh,tc_start,{t11_SUITE,init_per_suite}},
+     {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+     {?eh,test_stats,{1,2,{1,1}}},
+     {?eh,tc_start,{t11_SUITE,end_per_suite}},
+     {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
      {?eh,tc_start,{t12_SUITE,init_per_suite}},
      {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
-     {?eh,test_stats,{3,6,{3,3}}},
+     {?eh,test_stats,{2,4,{2,2}}},
      {?eh,tc_start,{t12_SUITE,end_per_suite}},
      {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+     {?eh,tc_start,{t21_SUITE,init_per_suite}},
+     {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+     {?eh,tc_start,{t21_SUITE,ok_tc}},
+     {?eh,test_stats,{3,6,{3,3}}},
+     {?eh,tc_start,{t21_SUITE,end_per_suite}},
+     {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
      {?eh,tc_start,{t22_SUITE,init_per_suite}},
      {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
      {?eh,test_stats,{4,8,{4,4}}},