From dd1a44e059a5de918de7067d49df6eb00ca563b2 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Sun, 27 Jan 2013 15:03:23 +0100 Subject: Fix problem with tests running in wrong order OTP-9881 --- lib/common_test/src/ct_testspec.erl | 63 ++++++++++++---- lib/common_test/test/ct_testspec_3_SUITE.erl | 104 +++++++++++++-------------- 2 files changed, 98 insertions(+), 69 deletions(-) (limited to 'lib') diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index abd82a3176..4f3597dffb 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -28,7 +28,6 @@ collect_tests_from_file/2, collect_tests_from_file/3]). -include("ct_util.hrl"). - -define(testspec_fields, record_info(fields, testspec)). %%%------------------------------------------------------------------ @@ -295,7 +294,7 @@ delete_dups1([],Keep) -> Keep. create_specs(Specs,TestSpec,Relaxed,Join) -> - SpecsTree = create_spec_tree(Specs,TestSpec,Join,[]), + SpecsTree = create_spec_tree(Specs,TestSpec,Join,[]), create_specs(SpecsTree,TestSpec,Relaxed). create_spec_tree([Spec|Specs],TS,JoinWithNext,Known) -> @@ -328,9 +327,11 @@ create_spec_tree([],_TS,_JoinWithNext,_Known) -> create_specs({Spec,Terms,InclJoin,InclSep,JoinWithNext,NextSpec}, TestSpec,Relaxed) -> SpecDir = filename:dirname(filename:absname(Spec)), - TestSpec1 = create_spec(Terms,TestSpec#testspec{spec_dir=SpecDir},Relaxed), + TestSpec1 = create_spec(Terms,TestSpec#testspec{spec_dir=SpecDir}, + JoinWithNext,Relaxed), {{JoinSpecs1,JoinTS1},Separate1} = create_specs(InclJoin,TestSpec1,Relaxed), + {{JoinSpecs2,JoinTS2},Separate2} = case JoinWithNext of true -> @@ -351,7 +352,7 @@ create_specs({Spec,Terms,InclJoin,InclSep,JoinWithNext,NextSpec}, AllSeparate = [TSData || TSData = {Ss,_TS} <- Separate3++Separate1++ [SepJoinSpecs]++Separate2++ - Separate4++[SepJoinSpecs1], + [SepJoinSpecs1]++Separate4, Ss /= []], case {JoinWithNext,JoinSpecs1} of @@ -368,9 +369,15 @@ create_specs({Spec,Terms,InclJoin,InclSep,JoinWithNext,NextSpec}, create_specs([],TestSpec,_Relaxed) -> {{[],TestSpec},[]}. -create_spec(Terms,TestSpec,Relaxed) -> +create_spec(Terms,TestSpec,JoinedByPrev,Relaxed) -> + %% it's the "includer" that decides the value of merge_tests + Terms1 = if not JoinedByPrev -> + [{set_merge_tests,true}|Terms]; + true -> + [{set_merge_tests,false}|Terms] + end, TS = #testspec{tests=Tests, logdir=LogDirs} = - collect_tests({false,Terms},TestSpec,Relaxed), + collect_tests({false,Terms1},TestSpec,Relaxed), LogDirs1 = lists:delete(".",LogDirs) ++ ["."], TS#testspec{tests=lists:flatten(Tests), logdir=LogDirs1}. @@ -397,14 +404,23 @@ collect_tests({Replace,Terms},TestSpec=#testspec{alias=As,nodes=Ns},Relaxed) -> Terms1 = if Replace -> replace_names(Terms); true -> Terms end, + {MergeTestsDef,Terms2} = + case proplists:get_value(set_merge_tests,Terms1,true) of + false -> + %% disable merge_tests + {TestSpec#testspec.merge_tests, + proplists:delete(merge_tests,Terms1)}; + true -> + {true,Terms1} + end, %% reverse nodes and aliases initially to get the order of them right %% in case this spec is being joined with a previous one - TestSpec1 = get_global(Terms1,TestSpec#testspec{alias = lists:reverse(As), + TestSpec1 = get_global(Terms2,TestSpec#testspec{alias = lists:reverse(As), nodes = lists:reverse(Ns), - merge_tests = true}), - TestSpec2 = get_all_nodes(Terms1,TestSpec1), - {Terms2, TestSpec3} = filter_init_terms(Terms1, [], TestSpec2), - add_tests(Terms2,TestSpec3). + merge_tests = MergeTestsDef}), + TestSpec2 = get_all_nodes(Terms2,TestSpec1), + {Terms3, TestSpec3} = filter_init_terms(Terms2, [], TestSpec2), + add_tests(Terms3,TestSpec3). %% replace names (atoms) in the testspec matching those in 'define' terms by %% searching recursively through tuples and lists @@ -915,6 +931,9 @@ add_tests([{release_shell,Bool}|Ts],Spec) -> add_tests(Ts, Spec#testspec{release_shell = Bool}); %% --- handled/errors --- +add_tests([{set_merge_tests,_}|Ts],Spec) -> % internal + add_tests(Ts,Spec); + add_tests([{define,_,_}|Ts],Spec) -> % handled add_tests(Ts,Spec); @@ -924,10 +943,10 @@ add_tests([{alias,_,_}|Ts],Spec) -> % handled add_tests([{node,_,_}|Ts],Spec) -> % handled add_tests(Ts,Spec); -add_tests([{merge_tests,_} | Ts], Spec) -> % handled +add_tests([{merge_tests,_} | Ts], Spec) -> % handled add_tests(Ts,Spec); -add_tests([{specs,_,_} | Ts], Spec) -> % handled +add_tests([{specs,_,_} | Ts], Spec) -> % handled add_tests(Ts,Spec); %% -------------------------------------------------- @@ -1001,7 +1020,18 @@ add_tests([],Spec) -> % done %% check if it's a CT term that has bad format or if the user seems to %% have added something of his/her own, which we'll let pass if relaxed %% mode is enabled. -check_term(Term) -> +check_term(Atom) when is_atom(Atom) -> + Valid = valid_terms(), + case lists:member(Atom,Valid) of + true -> + valid; + false -> % ignore + case get(relaxed) of + true -> invalid; + false -> throw({error,{undefined_term_in_spec,Atom}}) + end + end; +check_term(Term) when is_tuple(Term) -> Size = size(Term), [Name|_] = tuple_to_list(Term), Valid = valid_terms(), @@ -1029,7 +1059,9 @@ check_term(Term) -> throw({error,{undefined_term_in_spec,Term}}) end end - end. + end; +check_term(Other) -> + throw({error,{undefined_term_in_spec,Other}}). %% specific data handling before saving in testspec record, e.g. %% converting relative paths to absolute for directories and files @@ -1421,6 +1453,7 @@ is_node([],_) -> valid_terms() -> [ + {set_merge_tests,2}, {define,3}, {specs,3}, {node,3}, diff --git a/lib/common_test/test/ct_testspec_3_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE.erl index 8b84b563ab..e2f57c2c64 100644 --- a/lib/common_test/test/ct_testspec_3_SUITE.erl +++ b/lib/common_test/test/ct_testspec_3_SUITE.erl @@ -331,16 +331,16 @@ test_events(start_join) -> {?eh,test_stats,{1,2,{1,1}}}, {?eh,tc_start,{t11_SUITE,end_per_suite}}, {?eh,tc_done,{t11_SUITE,end_per_suite,ok}}, - {?eh,tc_start,{t21_SUITE,init_per_suite}}, - {?eh,tc_done,{t21_SUITE,init_per_suite,ok}}, - {?eh,test_stats,{2,4,{2,2}}}, - {?eh,tc_start,{t21_SUITE,end_per_suite}}, - {?eh,tc_done,{t21_SUITE,end_per_suite,ok}}, {?eh,tc_start,{t12_SUITE,init_per_suite}}, {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, - {?eh,test_stats,{3,6,{3,3}}}, + {?eh,test_stats,{2,4,{2,2}}}, {?eh,tc_start,{t12_SUITE,end_per_suite}}, {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t21_SUITE,init_per_suite}}, + {?eh,tc_done,{t21_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{3,6,{3,3}}}, + {?eh,tc_start,{t21_SUITE,end_per_suite}}, + {?eh,tc_done,{t21_SUITE,end_per_suite,ok}}, {?eh,tc_start,{t22_SUITE,init_per_suite}}, {?eh,tc_done,{t22_SUITE,init_per_suite,ok}}, {?eh,test_stats,{4,8,{4,4}}}, @@ -520,8 +520,7 @@ test_events(incl_separate2) -> {?eh,stop_logging,[]}]; test_events(incl_join1) -> - [ - {?eh,start_logging,{'DEF','RUNDIR'}}, + [{?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, {?eh,start_info,{4,4,20}}, {?eh,tc_start,{t12_SUITE,init_per_suite}}, @@ -549,61 +548,58 @@ test_events(incl_join1) -> {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, {?eh,start_info,{4,4,20}}, + {?eh,tc_start,{t22_SUITE,init_per_suite}}, + {?eh,tc_done,{t22_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t22_SUITE,end_per_suite}}, + {?eh,tc_done,{t22_SUITE,end_per_suite,ok}}, {?eh,tc_start,{t21_SUITE,init_per_suite}}, {?eh,tc_done,{t21_SUITE,init_per_suite,ok}}, - {?eh,test_stats,{1,2,{1,1}}}, + {?eh,test_stats,{2,4,{2,2}}}, {?eh,tc_start,{t21_SUITE,end_per_suite}}, {?eh,tc_done,{t21_SUITE,end_per_suite,ok}}, {?eh,tc_start,{t11_SUITE,init_per_suite}}, {?eh,tc_done,{t11_SUITE,init_per_suite,ok}}, - {?eh,test_stats,{2,4,{2,2}}}, + {?eh,test_stats,{3,6,{3,3}}}, {?eh,tc_start,{t11_SUITE,end_per_suite}}, {?eh,tc_done,{t11_SUITE,end_per_suite,ok}}, {?eh,tc_start,{t12_SUITE,init_per_suite}}, {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, - {?eh,test_stats,{3,6,{3,3}}}, + {?eh,test_stats,{4,8,{4,4}}}, {?eh,tc_start,{t12_SUITE,end_per_suite}}, {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, - {?eh,tc_start,{t22_SUITE,init_per_suite}}, - {?eh,tc_done,{t22_SUITE,init_per_suite,ok}}, - {?eh,test_stats,{4,8,{4,4}}}, - {?eh,tc_start,{t22_SUITE,end_per_suite}}, - {?eh,tc_done,{t22_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]}]; test_events(incl_join2) -> - [ - {?eh,start_logging,{'DEF','RUNDIR'}}, + [{?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, {?eh,start_info,{4,4,20}}, - {?eh,tc_start,{t11_SUITE,init_per_suite}}, - {?eh,tc_done,{t11_SUITE,init_per_suite,ok}}, - {?eh,tc_start,{t11_SUITE,ok_tc}}, - {?eh,test_stats,{1,2,{1,1}}}, - {?eh,tc_start,{t11_SUITE,end_per_suite}}, - {?eh,tc_done,{t11_SUITE,end_per_suite,ok}}, - {?eh,tc_start,{t21_SUITE,init_per_suite}}, - {?eh,tc_done,{t21_SUITE,init_per_suite,ok}}, - {?eh,test_stats,{2,4,{2,2}}}, - {?eh,tc_start,{t21_SUITE,end_per_suite}}, - {?eh,tc_done,{t21_SUITE,end_per_suite,ok}}, {?eh,tc_start,{t12_SUITE,init_per_suite}}, {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, - {?eh,test_stats,{3,6,{3,3}}}, + {?eh,test_stats,{1,2,{1,1}}}, {?eh,tc_start,{t12_SUITE,end_per_suite}}, {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t11_SUITE,init_per_suite}}, + {?eh,tc_done,{t11_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{2,4,{2,2}}}, + {?eh,tc_start,{t11_SUITE,end_per_suite}}, + {?eh,tc_done,{t11_SUITE,end_per_suite,ok}}, {?eh,tc_start,{t22_SUITE,init_per_suite}}, {?eh,tc_done,{t22_SUITE,init_per_suite,ok}}, - {?eh,test_stats,{4,8,{4,4}}}, + {?eh,test_stats,{3,6,{3,3}}}, {?eh,tc_start,{t22_SUITE,end_per_suite}}, {?eh,tc_done,{t22_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t21_SUITE,init_per_suite}}, + {?eh,tc_done,{t21_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{4,8,{4,4}}}, + {?eh,tc_start,{t21_SUITE,end_per_suite}}, + {?eh,tc_done,{t21_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]}]; test_events(incl_both1) -> - [ - {?eh,start_logging,{'DEF','RUNDIR'}}, + [{?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, {?eh,start_info,{2,2,10}}, {?eh,tc_start,{t11_SUITE,init_per_suite}}, @@ -641,21 +637,6 @@ test_events(incl_both1) -> {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, {?eh,start_info,{2,2,10}}, - {?eh,tc_start,{t11_SUITE,init_per_suite}}, - {?eh,tc_done,{t11_SUITE,init_per_suite,ok}}, - {?eh,test_stats,{1,2,{1,1}}}, - {?eh,tc_start,{t11_SUITE,end_per_suite}}, - {?eh,tc_done,{t11_SUITE,end_per_suite,ok}}, - {?eh,tc_start,{t21_SUITE,init_per_suite}}, - {?eh,tc_done,{t21_SUITE,init_per_suite,ok}}, - {?eh,test_stats,{2,4,{2,2}}}, - {?eh,tc_start,{t21_SUITE,end_per_suite}}, - {?eh,tc_done,{t21_SUITE,end_per_suite,ok}}, - {?eh,test_done,{'DEF','STOP_TIME'}}, - {?eh,stop_logging,[]}, - {?eh,start_logging,{'DEF','RUNDIR'}}, - {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, - {?eh,start_info,{2,2,10}}, {?eh,tc_start,{t12_SUITE,init_per_suite}}, {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, {?eh,test_stats,{1,2,{1,1}}}, @@ -667,13 +648,10 @@ test_events(incl_both1) -> {?eh,tc_start,{t22_SUITE,end_per_suite}}, {?eh,tc_done,{t22_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, - {?eh,stop_logging,[]}]; - -test_events(incl_both2) -> - [ + {?eh,stop_logging,[]}, {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, - {?eh,start_info,{4,4,20}}, + {?eh,start_info,{2,2,10}}, {?eh,tc_start,{t11_SUITE,init_per_suite}}, {?eh,tc_done,{t11_SUITE,init_per_suite,ok}}, {?eh,test_stats,{1,2,{1,1}}}, @@ -684,11 +662,29 @@ test_events(incl_both2) -> {?eh,test_stats,{2,4,{2,2}}}, {?eh,tc_start,{t21_SUITE,end_per_suite}}, {?eh,tc_done,{t21_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}]; + +test_events(incl_both2) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{4,4,20}}, + {?eh,tc_start,{t11_SUITE,init_per_suite}}, + {?eh,tc_done,{t11_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t11_SUITE,end_per_suite}}, + {?eh,tc_done,{t11_SUITE,end_per_suite,ok}}, {?eh,tc_start,{t12_SUITE,init_per_suite}}, {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, - {?eh,test_stats,{3,6,{3,3}}}, + {?eh,test_stats,{2,4,{2,2}}}, {?eh,tc_start,{t12_SUITE,end_per_suite}}, {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t21_SUITE,init_per_suite}}, + {?eh,tc_done,{t21_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{t21_SUITE,ok_tc}}, + {?eh,test_stats,{3,6,{3,3}}}, + {?eh,tc_start,{t21_SUITE,end_per_suite}}, + {?eh,tc_done,{t21_SUITE,end_per_suite,ok}}, {?eh,tc_start,{t22_SUITE,init_per_suite}}, {?eh,tc_done,{t22_SUITE,init_per_suite,ok}}, {?eh,test_stats,{4,8,{4,4}}}, -- cgit v1.2.3