From 18ca34b08868d5c045a854b022a4d9fb3fa727a0 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Fri, 14 Jan 2011 17:58:42 +0100 Subject: Add test cases for test spec ordering of testcases in spec --- lib/common_test/test/ct_testspec_1_SUITE.erl | 154 ++++++++++++++++++++++++++- 1 file changed, 153 insertions(+), 1 deletion(-) diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl index dc399bfb4c..f592d697bd 100644 --- a/lib/common_test/test/ct_testspec_1_SUITE.erl +++ b/lib/common_test/test/ct_testspec_1_SUITE.erl @@ -73,7 +73,13 @@ all(suite) -> subgroup_all_testcases, skip_subgroup_all_testcases, subgroup_testcase, skip_subgroup_testcase, sub_skipped_by_top, - testcase_in_multiple_groups]. + testcase_in_multiple_groups, + order_of_tests_in_multiple_dirs, + order_of_tests_in_multiple_suites, + order_of_suites_in_multiple_dirs, + order_of_groups_in_multiple_dirs, + order_of_groups_in_multiple_suites + ]. %%-------------------------------------------------------------------- %% TEST CASES @@ -365,6 +371,74 @@ testcase_in_multiple_groups(Config) when is_list(Config) -> setup_and_execute(testcase_in_multiple_groups, TestSpec, Config). +%%%----------------------------------------------------------------- +%%% + +order_of_tests_in_multiple_dirs(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestDir2 = filename:join(DataDir, "groups_2"), + TestSpec = [{cases,TestDir1,groups_12_SUITE,[testcase_1a]}, + {cases,TestDir2,groups_22_SUITE,[testcase_1]}, + {cases,TestDir1,groups_12_SUITE,[testcase_1b]}], + + setup_and_execute(order_of_tests_in_multiple_dirs, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_tests_in_multiple_suites(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{cases,TestDir1,groups_12_SUITE,[testcase_1a]}, + {cases,TestDir1,groups_11_SUITE,[testcase_1]}, + {cases,TestDir1,groups_12_SUITE,[testcase_1b]}], + + setup_and_execute(order_of_tests_in_multiple_suites, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_suites_in_multiple_dirs(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestDir2 = filename:join(DataDir, "groups_2"), + TestSpec = [{suites,TestDir1,groups_12_SUITE}, + {suites,TestDir2,groups_22_SUITE}, + {suites,TestDir1,groups_11_SUITE}], + + setup_and_execute(order_of_suites_in_multiple_dirs, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_groups_in_multiple_dirs(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestDir2 = filename:join(DataDir, "groups_2"), + TestSpec = [{groups,TestDir1,groups_12_SUITE,test_group_1a}, + {groups,TestDir2,groups_22_SUITE,test_group_1a}, + {groups,TestDir1,groups_12_SUITE,test_group_1b}], + + setup_and_execute(order_of_groups_in_multiple_dirs, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_groups_in_multiple_suites(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir1,groups_12_SUITE,test_group_1a}, + {groups,TestDir1,groups_11_SUITE,test_group_1a}, + {groups,TestDir1,groups_12_SUITE,test_group_1b}], + + setup_and_execute(order_of_groups_in_multiple_suites, TestSpec, Config). + %%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- @@ -428,6 +502,84 @@ events_to_check(_, 0) -> events_to_check(Test, N) -> test_events(Test) ++ events_to_check(Test, N-1). + +test_events(order_of_tests_in_multiple_dirs) -> + [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, + {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {ct_test_support_eh,tc_done, + {groups_12_SUITE,testcase_1a, + {failed,{error,{test_case_failed,no_group_data}}}}}, + {ct_test_support_eh,tc_start,{groups_22_SUITE,testcase_1}}, + {ct_test_support_eh,tc_done,{groups_22_SUITE,testcase_1,ok}}, + {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1b}}, + {ct_test_support_eh,tc_done, + {groups_12_SUITE,testcase_1b, + {failed,{error,{test_case_failed,no_group_data}}}}}, + {ct_test_support_eh,stop_logging,[]} + ]; +test_events(order_of_tests_in_multiple_suites) -> + [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, + {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {ct_test_support_eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, + {ct_test_support_eh,tc_start,{groups_11_SUITE,testcase_1}}, + {ct_test_support_eh,tc_done,{groups_11_SUITE,testcase_1,ok}}, + {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1b}}, + {ct_test_support_eh,tc_done,{groups_12_SUITE,testcase_1b,'_'}}, + {ct_test_support_eh,stop_logging,[]} + ]; +test_events(order_of_suites_in_multiple_dirs) -> + [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, + {ct_test_support_eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {ct_test_support_eh,tc_done,{groups_12_SUITE,init_per_suite,'_'}}, + {ct_test_support_eh,tc_start,{groups_12_SUITE,end_per_suite}}, + {ct_test_support_eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {ct_test_support_eh,tc_start,{groups_22_SUITE,init_per_suite}}, + {ct_test_support_eh,tc_done,{groups_22_SUITE,init_per_suite,'_'}}, + {ct_test_support_eh,tc_start,{groups_22_SUITE,end_per_suite}}, + {ct_test_support_eh,tc_done,{groups_22_SUITE,end_per_suite,'_'}}, + {ct_test_support_eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {ct_test_support_eh,tc_done,{groups_11_SUITE,init_per_suite,'_'}}, + {ct_test_support_eh,tc_start,{groups_11_SUITE,end_per_suite}}, + {ct_test_support_eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {ct_test_support_eh,stop_logging,[]}]; +test_events(order_of_groups_in_multiple_dirs) -> + [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, + + {ct_test_support_eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, + {ct_test_support_eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {ct_test_support_eh,tc_start, + {groups_22_SUITE,{init_per_group,test_group_1a,'_'}}}, + {ct_test_support_eh,tc_done, + {groups_22_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {ct_test_support_eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}}, + {ct_test_support_eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, + + {ct_test_support_eh,stop_logging,[]}]; +test_events(order_of_groups_in_multiple_suites) -> + [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, + + {ct_test_support_eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, + {ct_test_support_eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {ct_test_support_eh,tc_start, + {groups_11_SUITE,{init_per_group,test_group_1a,'_'}}}, + {ct_test_support_eh,tc_done, + {groups_11_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {ct_test_support_eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}}, + {ct_test_support_eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, + + {ct_test_support_eh,stop_logging,[]}]; test_events(_) -> [ ]. -- cgit v1.2.3 From 05a18c6877dde2e7d0d64edb384607098cf8af90 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Thu, 20 Jan 2011 17:38:30 +0100 Subject: Update test spec parser to not group suites, groups and cases which are hierarchically related if merge_tests config i set to false --- lib/common_test/doc/src/run_test_chapter.xml | 10 +- lib/common_test/src/ct_testspec.erl | 133 +++++++---- lib/common_test/src/ct_util.hrl | 3 +- lib/common_test/test/ct_testspec_1_SUITE.erl | 342 ++++++++++++++++++++++++--- 4 files changed, 408 insertions(+), 80 deletions(-) diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index 94fcf6bf01..1662923a69 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -365,15 +365,17 @@ either one or more suites, one or more test case groups, or one or more test cases in a group or suite.

An arbitrary number of test terms may be declared in sequence. - Common Test will compile the terms into one or more tests to be - performed in one resulting test run. Note that a term that + Common Test will by default compile the terms into one or more tests + to be performed in one resulting test run. Note that a term that specifies a set of test cases will "swallow" one that only specifies a subset of these cases. E.g. the result of merging one term that specifies that all cases in suite S should be executed, with another term specifying only test case X and Y in S, is a test of all cases in S. However, if a term specifying test case X and Y in S is merged with a term specifying case Z - in S, the result is a test of X, Y and Z in S.

+ in S, the result is a test of X, Y and Z in S. To disable this + behaviour, it is possible in test specification to set the + merge_tests term to false.

A test term can also specify one or more test suites, groups, or test cases to be skipped. Skipped suites, groups and cases are not executed and show up in the HTML test log files as @@ -432,6 +434,8 @@ {userconfig, NodeRefs, {CallbackModule, ConfigStrings}}. {alias, DirAlias, Dir}. + + {merge_tests, Bool}. {logdir, LogDir}. {logdir, NodeRefs, LogDir}. diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index f5069427a2..bf9282900b 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -68,7 +68,8 @@ prepare_tests(TestSpec) when is_record(TestSpec,testspec) -> %% Create initial list of {Node,{Run,Skip}} tuples NodeList = lists:map(fun(N) -> {N,{[],[]}} end, list_nodes(TestSpec)), %% Get all Run tests sorted per node basis. - NodeList1 = run_per_node(Run,NodeList), + NodeList1 = run_per_node(Run,NodeList, + TestSpec#testspec.merge_tests), %% Get all Skip entries sorted per node basis. NodeList2 = skip_per_node(Skip,NodeList1), %% Change representation. @@ -89,11 +90,17 @@ prepare_tests(TestSpec) when is_record(TestSpec,testspec) -> %% run_per_node/2 takes the Run list as input and returns a list %% of {Node,RunPerNode,[]} tuples where the tests have been sorted %% on a per node basis. -run_per_node([{{Node,Dir},Test}|Ts],Result) -> +run_per_node([{{Node,Dir},Test}|Ts],Result, MergeTests) -> {value,{Node,{Run,Skip}}} = lists:keysearch(Node,1,Result), - Run1 = merge_tests(Dir,Test,Run), - run_per_node(Ts,insert_in_order({Node,{Run1,Skip}},Result)); -run_per_node([],Result) -> + Run1 = case MergeTests of + false -> + append({Dir, Test}, Run); + true -> + merge_tests(Dir,Test,Run) + end, + run_per_node(Ts,insert_in_order({Node,{Run1,Skip}},Result), + MergeTests); +run_per_node([],Result,_) -> Result. merge_tests(Dir,Test={all,_},TestDirs) -> @@ -281,6 +288,8 @@ collect_tests(Terms,TestSpec,Relaxed) -> {Terms2, TestSpec3} = filter_init_terms(Terms, [], TestSpec2), add_tests(Terms2,TestSpec3). +get_global([{merge_tests, Bool} | Ts], Spec) -> + get_global(Ts,Spec#testspec{ merge_tests = Bool }); get_global([{alias,Ref,Dir}|Ts],Spec=#testspec{alias=Refs}) -> get_global(Ts,Spec#testspec{alias=[{Ref,get_absdir(Dir,Spec)}|Refs]}); get_global([{node,Ref,Node}|Ts],Spec=#testspec{nodes=Refs}) -> @@ -656,7 +665,7 @@ add_tests([{suites,Node,Dir,Ss}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = insert_suites(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Ss,Tests), + Ss,Tests, Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); %% --- groups --- @@ -682,13 +691,15 @@ add_tests([{groups,Node,Dir,Suite,Gs}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = insert_groups(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Suite,Gs,all,Tests), + Suite,Gs,all,Tests, + Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); add_tests([{groups,Node,Dir,Suite,Gs,{cases,TCs}}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = insert_groups(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Suite,Gs,TCs,Tests), + Suite,Gs,TCs,Tests, + Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); %% --- cases --- @@ -703,7 +714,7 @@ add_tests([{cases,Node,Dir,Suite,Cs}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = insert_cases(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Suite,Cs,Tests), + Suite,Cs,Tests, Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); %% --- skip_suites --- @@ -718,7 +729,8 @@ add_tests([{skip_suites,Node,Dir,Ss,Cmt}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = skip_suites(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Ss,Cmt,Tests), + Ss,Cmt,Tests, + Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); %% --- skip_groups --- @@ -740,13 +752,15 @@ add_tests([{skip_groups,Node,Dir,Suite,Gs,Cmt}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = skip_groups(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Suite,Gs,all,Cmt,Tests), + Suite,Gs,all,Cmt,Tests, + Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); add_tests([{skip_groups,Node,Dir,Suite,Gs,{cases,TCs},Cmt}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = skip_groups(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Suite,Gs,TCs,Cmt,Tests), + Suite,Gs,TCs,Cmt,Tests, + Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); %% --- skip_cases --- @@ -761,7 +775,7 @@ add_tests([{skip_cases,Node,Dir,Suite,Cs,Cmt}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = skip_cases(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Suite,Cs,Cmt,Tests), + Suite,Cs,Cmt,Tests,Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); %% --- handled/errors --- @@ -771,6 +785,9 @@ add_tests([{alias,_,_}|Ts],Spec) -> % handled add_tests([{node,_,_}|Ts],Spec) -> % handled add_tests(Ts,Spec); +add_tests([{merge_tests, _} | Ts], Spec) -> % handled + add_tests(Ts,Spec); + %% check if it's a CT term that has bad format or if the user seems to %% have added something of his/her own, which we'll let pass if relaxed %% mode is enabled. @@ -823,17 +840,22 @@ separate([],_,_,_) -> %% {Suite2,[{GrOrCase21,{skip,Cmt}},GrOrCase22,...]},...]} %% GrOrCase = {GroupName,[Case1,Case2,...]} | Case -insert_suites(Node,Dir,[S|Ss],Tests) -> - Tests1 = insert_cases(Node,Dir,S,all,Tests), - insert_suites(Node,Dir,Ss,Tests1); -insert_suites(_Node,_Dir,[],Tests) -> +insert_suites(Node,Dir,[S|Ss],Tests, MergeTests) -> + Tests1 = insert_cases(Node,Dir,S,all,Tests,MergeTests), + insert_suites(Node,Dir,Ss,Tests1,MergeTests); +insert_suites(_Node,_Dir,[],Tests,_MergeTests) -> Tests; -insert_suites(Node,Dir,S,Tests) -> - insert_suites(Node,Dir,[S],Tests). +insert_suites(Node,Dir,S,Tests,MergeTests) -> + insert_suites(Node,Dir,[S],Tests,MergeTests). -insert_groups(Node,Dir,Suite,Group,Cases,Tests) when is_atom(Group) -> - insert_groups(Node,Dir,Suite,[Group],Cases,Tests); -insert_groups(Node,Dir,Suite,Groups,Cases,Tests) when +insert_groups(Node,Dir,Suite,Group,Cases,Tests,MergeTests) + when is_atom(Group) -> + insert_groups(Node,Dir,Suite,[Group],Cases,Tests,MergeTests); +insert_groups(Node,Dir,Suite,Groups,Cases,Tests,false) when + ((Cases == all) or is_list(Cases)) and is_list(Groups) -> + Groups1 = [{Gr,Cases} || Gr <- Groups], + append({{Node,Dir},[{Suite,Groups1}]},Tests); +insert_groups(Node,Dir,Suite,Groups,Cases,Tests,true) when ((Cases == all) or is_list(Cases)) and is_list(Groups) -> case lists:keysearch({Node,Dir},1,Tests) of {value,{{Node,Dir},[{all,_}]}} -> @@ -847,9 +869,10 @@ insert_groups(Node,Dir,Suite,Groups,Cases,Tests) when Groups1 = [{Gr,Cases} || Gr <- Groups], insert_in_order({{Node,Dir},[{Suite,Groups1}]},Tests) end; -insert_groups(Node,Dir,Suite,Groups,Case,Tests) when is_atom(Case) -> +insert_groups(Node,Dir,Suite,Groups,Case,Tests, MergeTests) + when is_atom(Case) -> Cases = if Case == all -> all; true -> [Case] end, - insert_groups(Node,Dir,Suite,Groups,Cases,Tests). + insert_groups(Node,Dir,Suite,Groups,Cases,Tests, MergeTests). insert_groups1(_Suite,_Groups,all) -> all; @@ -879,7 +902,9 @@ insert_groups2([Group={GrName,Cases}|Groups],GrAndCases) -> insert_groups2([],GrAndCases) -> GrAndCases. -insert_cases(Node,Dir,Suite,Cases,Tests) when is_list(Cases) -> +insert_cases(Node,Dir,Suite,Cases,Tests,false) when is_list(Cases) -> + append({{Node,Dir},[{Suite,Cases}]},Tests); +insert_cases(Node,Dir,Suite,Cases,Tests,true) when is_list(Cases) -> case lists:keysearch({Node,Dir},1,Tests) of {value,{{Node,Dir},[{all,_}]}} -> Tests; @@ -889,8 +914,8 @@ insert_cases(Node,Dir,Suite,Cases,Tests) when is_list(Cases) -> false -> insert_in_order({{Node,Dir},[{Suite,Cases}]},Tests) end; -insert_cases(Node,Dir,Suite,Case,Tests) when is_atom(Case) -> - insert_cases(Node,Dir,Suite,[Case],Tests). +insert_cases(Node,Dir,Suite,Case,Tests,MergeTests) when is_atom(Case) -> + insert_cases(Node,Dir,Suite,[Case],Tests,MergeTests). insert_cases1(_Suite,_Cases,all) -> all; @@ -905,22 +930,28 @@ insert_cases1(Suite,Cases,Suites0) -> insert_in_order({Suite,Cases},Suites0) end. -skip_suites(Node,Dir,[S|Ss],Cmt,Tests) -> - Tests1 = skip_cases(Node,Dir,S,all,Cmt,Tests), - skip_suites(Node,Dir,Ss,Cmt,Tests1); -skip_suites(_Node,_Dir,[],_Cmt,Tests) -> +skip_suites(Node,Dir,[S|Ss],Cmt,Tests,MergeTests) -> + Tests1 = skip_cases(Node,Dir,S,all,Cmt,Tests,MergeTests), + skip_suites(Node,Dir,Ss,Cmt,Tests1,MergeTests); +skip_suites(_Node,_Dir,[],_Cmt,Tests,_MergeTests) -> Tests; -skip_suites(Node,Dir,S,Cmt,Tests) -> - skip_suites(Node,Dir,[S],Cmt,Tests). - -skip_groups(Node,Dir,Suite,Group,all,Cmt,Tests) when is_atom(Group) -> - skip_groups(Node,Dir,Suite,[Group],all,Cmt,Tests); -skip_groups(Node,Dir,Suite,Group,Cases,Cmt,Tests) when is_atom(Group) -> - skip_groups(Node,Dir,Suite,[Group],Cases,Cmt,Tests); -skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests) when is_atom(Case), - Case =/= all -> - skip_groups(Node,Dir,Suite,Groups,[Case],Cmt,Tests); -skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests) when +skip_suites(Node,Dir,S,Cmt,Tests,MergeTests) -> + skip_suites(Node,Dir,[S],Cmt,Tests,MergeTests). + +skip_groups(Node,Dir,Suite,Group,all,Cmt,Tests,MergeTests) + when is_atom(Group) -> + skip_groups(Node,Dir,Suite,[Group],all,Cmt,Tests,MergeTests); +skip_groups(Node,Dir,Suite,Group,Cases,Cmt,Tests,MergeTests) + when is_atom(Group) -> + skip_groups(Node,Dir,Suite,[Group],Cases,Cmt,Tests,MergeTests); +skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests,MergeTests) + when is_atom(Case),Case =/= all -> + skip_groups(Node,Dir,Suite,Groups,[Case],Cmt,Tests,MergeTests); +skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests,false) when + ((Cases == all) or is_list(Cases)) and is_list(Groups) -> + Suites1 = skip_groups1(Suite,[{Gr,Cases} || Gr <- Groups],Cmt,[]), + append({{Node,Dir},Suites1},Tests); +skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests,true) when ((Cases == all) or is_list(Cases)) and is_list(Groups) -> Suites = case lists:keysearch({Node,Dir},1,Tests) of @@ -931,9 +962,10 @@ skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests) when end, Suites1 = skip_groups1(Suite,[{Gr,Cases} || Gr <- Groups],Cmt,Suites), insert_in_order({{Node,Dir},Suites1},Tests); -skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests) when is_atom(Case) -> +skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests,MergeTests) + when is_atom(Case) -> Cases = if Case == all -> all; true -> [Case] end, - skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests). + skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests,MergeTests). skip_groups1(Suite,Groups,Cmt,Suites0) -> SkipGroups = lists:map(fun(Group) -> @@ -947,7 +979,10 @@ skip_groups1(Suite,Groups,Cmt,Suites0) -> insert_in_order({Suite,SkipGroups},Suites0) end. -skip_cases(Node,Dir,Suite,Cases,Cmt,Tests) when is_list(Cases) -> +skip_cases(Node,Dir,Suite,Cases,Cmt,Tests,false) when is_list(Cases) -> + Suites1 = skip_cases1(Suite,Cases,Cmt,[]), + append({{Node,Dir},Suites1},Tests); +skip_cases(Node,Dir,Suite,Cases,Cmt,Tests,true) when is_list(Cases) -> Suites = case lists:keysearch({Node,Dir},1,Tests) of {value,{{Node,Dir},Suites0}} -> @@ -957,8 +992,8 @@ skip_cases(Node,Dir,Suite,Cases,Cmt,Tests) when is_list(Cases) -> end, Suites1 = skip_cases1(Suite,Cases,Cmt,Suites), insert_in_order({{Node,Dir},Suites1},Tests); -skip_cases(Node,Dir,Suite,Case,Cmt,Tests) when is_atom(Case) -> - skip_cases(Node,Dir,Suite,[Case],Cmt,Tests). +skip_cases(Node,Dir,Suite,Case,Cmt,Tests,MergeTests) when is_atom(Case) -> + skip_cases(Node,Dir,Suite,[Case],Cmt,Tests,MergeTests). skip_cases1(Suite,Cases,Cmt,Suites0) -> SkipCases = lists:map(fun(C) -> @@ -972,6 +1007,9 @@ skip_cases1(Suite,Cases,Cmt,Suites0) -> insert_in_order({Suite,SkipCases},Suites0) end. +append(Elem, List) -> + List ++ [Elem]. + insert_in_order([E|Es],List) -> List1 = insert_elem(E,List,[]), insert_in_order(Es,List1); @@ -1044,6 +1082,7 @@ valid_terms() -> {userconfig,2}, {userconfig,3}, {alias,3}, + {merge_tests,1}, {logdir,2}, {logdir,3}, {label,2}, diff --git a/lib/common_test/src/ct_util.hrl b/lib/common_test/src/ct_util.hrl index ee973f6220..66021ae99b 100644 --- a/lib/common_test/src/ct_util.hrl +++ b/lib/common_test/src/ct_util.hrl @@ -40,7 +40,8 @@ multiply_timetraps=[], scale_timetraps=[], alias=[], - tests=[]}). + tests=[], + merge_tests = true }). -record(cover, {app=none, level=details, diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl index f592d697bd..7b09310e2e 100644 --- a/lib/common_test/test/ct_testspec_1_SUITE.erl +++ b/lib/common_test/test/ct_testspec_1_SUITE.erl @@ -60,25 +60,35 @@ all(doc) -> ["Run smoke tests of Common Test."]; all(suite) -> - [all_suites, skip_all_suites, - suite, skip_suite, - all_testcases, skip_all_testcases, - testcase, skip_testcase, - all_groups, skip_all_groups, - group, skip_group, - group_all_testcases, skip_group_all_testcases, - group_testcase, skip_group_testcase, - topgroup, - subgroup, skip_subgroup, - subgroup_all_testcases, skip_subgroup_all_testcases, - subgroup_testcase, skip_subgroup_testcase, - sub_skipped_by_top, - testcase_in_multiple_groups, - order_of_tests_in_multiple_dirs, - order_of_tests_in_multiple_suites, - order_of_suites_in_multiple_dirs, - order_of_groups_in_multiple_dirs, - order_of_groups_in_multiple_suites + [ + %% all_suites, skip_all_suites, + %% suite, skip_suite, + %% all_testcases, skip_all_testcases, + %% testcase, skip_testcase, + %% all_groups, skip_all_groups, + %% group, skip_group, + %% group_all_testcases, skip_group_all_testcases, + %% group_testcase, skip_group_testcase, + %% topgroup, + %% subgroup, skip_subgroup, + %% subgroup_all_testcases, skip_subgroup_all_testcases, + %% subgroup_testcase, skip_subgroup_testcase, + %% sub_skipped_by_top, + %% testcase_in_multiple_groups, + %% order_of_tests_in_multiple_dirs_no_merge_tests, + %% order_of_tests_in_multiple_suites_no_merge_tests, + %% order_of_suites_in_multiple_dirs_no_merge_tests, + %% order_of_groups_in_multiple_dirs_no_merge_tests, + %% order_of_groups_in_multiple_suites_no_merge_tests, + %% order_of_tests_in_multiple_dirs, + %% order_of_tests_in_multiple_suites, + %% order_of_suites_in_multiple_dirs, + %% order_of_groups_in_multiple_dirs, + %% order_of_groups_in_multiple_suites, + order_of_tests_in_multiple_suites_with_skip_no_merge_tests, + order_of_tests_in_multiple_suites_with_skip, + all_plus_one_tc_no_merge_tests, + all_plus_one_tc ]. %%-------------------------------------------------------------------- @@ -374,6 +384,105 @@ testcase_in_multiple_groups(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% +order_of_tests_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestDir2 = filename:join(DataDir, "groups_2"), + TestSpec = [{merge_tests, false}, + {cases,TestDir1,groups_12_SUITE,[testcase_1a]}, + {cases,TestDir2,groups_22_SUITE,[testcase_1]}, + {cases,TestDir1,groups_12_SUITE,[testcase_1b]}], + + setup_and_execute(order_of_tests_in_multiple_dirs_no_merge_tests, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_tests_in_multiple_suites_no_merge_tests(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{merge_tests, false}, + {cases,TestDir1,groups_12_SUITE,[testcase_1a]}, + {cases,TestDir1,groups_11_SUITE,[testcase_1]}, + {cases,TestDir1,groups_12_SUITE,[testcase_1b]}], + + setup_and_execute(order_of_tests_in_multiple_suites_no_merge_tests, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_suites_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestDir2 = filename:join(DataDir, "groups_2"), + TestSpec = [{merge_tests, false}, + {suites,TestDir1,groups_12_SUITE}, + {suites,TestDir2,groups_22_SUITE}, + {suites,TestDir1,groups_11_SUITE}], + + setup_and_execute(order_of_suites_in_multiple_dirs_no_merge_tests, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_groups_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestDir2 = filename:join(DataDir, "groups_2"), + TestSpec = [{merge_tests, false}, + {groups,TestDir1,groups_12_SUITE,test_group_1a}, + {groups,TestDir2,groups_22_SUITE,test_group_1a}, + {groups,TestDir1,groups_12_SUITE,test_group_1b}], + + setup_and_execute(order_of_groups_in_multiple_dirs_no_merge_tests, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_groups_in_multiple_suites_no_merge_tests(Config) + when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{merge_tests, false}, + {groups,TestDir1,groups_12_SUITE,test_group_1a}, + {groups,TestDir1,groups_11_SUITE,test_group_1a}, + {groups,TestDir1,groups_12_SUITE,test_group_1b}], + + setup_and_execute(order_of_groups_in_multiple_suites_no_merge_tests, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_tests_in_multiple_suites_with_skip_no_merge_tests(Config) + when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{merge_tests, false}, + {cases,TestDir1,groups_12_SUITE,[testcase_1a]}, + {cases,TestDir1,groups_11_SUITE,[testcase_1]}, + {cases,TestDir1,groups_12_SUITE,[testcase_1b]}, + {cases,TestDir1,groups_11_SUITE,[testcase_2]}, + {skip_cases,TestDir1,groups_12_SUITE,[testcase_1b],"Skip it"}], + + setup_and_execute( + order_of_tests_in_multiple_suites_with_skip_no_merge_tests, + TestSpec, Config). + + +%%%----------------------------------------------------------------- +%%% + order_of_tests_in_multiple_dirs(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -383,7 +492,8 @@ order_of_tests_in_multiple_dirs(Config) when is_list(Config) -> {cases,TestDir2,groups_22_SUITE,[testcase_1]}, {cases,TestDir1,groups_12_SUITE,[testcase_1b]}], - setup_and_execute(order_of_tests_in_multiple_dirs, TestSpec, Config). + setup_and_execute(order_of_tests_in_multiple_dirs, + TestSpec, Config). %%%----------------------------------------------------------------- %%% @@ -396,7 +506,8 @@ order_of_tests_in_multiple_suites(Config) when is_list(Config) -> {cases,TestDir1,groups_11_SUITE,[testcase_1]}, {cases,TestDir1,groups_12_SUITE,[testcase_1b]}], - setup_and_execute(order_of_tests_in_multiple_suites, TestSpec, Config). + setup_and_execute(order_of_tests_in_multiple_suites, + TestSpec, Config). %%%----------------------------------------------------------------- %%% @@ -410,7 +521,8 @@ order_of_suites_in_multiple_dirs(Config) when is_list(Config) -> {suites,TestDir2,groups_22_SUITE}, {suites,TestDir1,groups_11_SUITE}], - setup_and_execute(order_of_suites_in_multiple_dirs, TestSpec, Config). + setup_and_execute(order_of_suites_in_multiple_dirs, + TestSpec, Config). %%%----------------------------------------------------------------- %%% @@ -424,7 +536,8 @@ order_of_groups_in_multiple_dirs(Config) when is_list(Config) -> {groups,TestDir2,groups_22_SUITE,test_group_1a}, {groups,TestDir1,groups_12_SUITE,test_group_1b}], - setup_and_execute(order_of_groups_in_multiple_dirs, TestSpec, Config). + setup_and_execute(order_of_groups_in_multiple_dirs, + TestSpec, Config). %%%----------------------------------------------------------------- %%% @@ -437,7 +550,53 @@ order_of_groups_in_multiple_suites(Config) when is_list(Config) -> {groups,TestDir1,groups_11_SUITE,test_group_1a}, {groups,TestDir1,groups_12_SUITE,test_group_1b}], - setup_and_execute(order_of_groups_in_multiple_suites, TestSpec, Config). + setup_and_execute(order_of_groups_in_multiple_suites, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_tests_in_multiple_suites_with_skip(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{cases,TestDir1,groups_12_SUITE,[testcase_1a]}, + {cases,TestDir1,groups_11_SUITE,[testcase_1]}, + {cases,TestDir1,groups_12_SUITE,[testcase_1b]}, + {cases,TestDir1,groups_11_SUITE,[testcase_2]}, + {skip_cases,TestDir1,groups_12_SUITE,[testcase_1b],"Skip it!"}], + + setup_and_execute(order_of_tests_in_multiple_suites_with_skip, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +all_plus_one_tc_no_merge_tests(Config) when is_list(Config) -> + + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{merge_tests,false}, + {suites,TestDir1,groups_12_SUITE}, + {cases,TestDir1,groups_12_SUITE,[testcase_1a]}], + + setup_and_execute(all_plus_one_tc_no_merge_tests, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +all_plus_one_tc(Config) when is_list(Config) -> + + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{suites,TestDir1,groups_12_SUITE}, + {cases,TestDir1,groups_12_SUITE,[testcase_1a]}], + + setup_and_execute(all_plus_one_tc, + TestSpec, Config). %%%----------------------------------------------------------------- %%% HELP FUNCTIONS @@ -503,7 +662,7 @@ events_to_check(Test, N) -> test_events(Test) ++ events_to_check(Test, N-1). -test_events(order_of_tests_in_multiple_dirs) -> +test_events(order_of_tests_in_multiple_dirs_no_merge_tests) -> [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1a}}, {ct_test_support_eh,tc_done, @@ -517,7 +676,7 @@ test_events(order_of_tests_in_multiple_dirs) -> {failed,{error,{test_case_failed,no_group_data}}}}}, {ct_test_support_eh,stop_logging,[]} ]; -test_events(order_of_tests_in_multiple_suites) -> +test_events(order_of_tests_in_multiple_suites_no_merge_tests) -> [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1a}}, {ct_test_support_eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, @@ -527,7 +686,7 @@ test_events(order_of_tests_in_multiple_suites) -> {ct_test_support_eh,tc_done,{groups_12_SUITE,testcase_1b,'_'}}, {ct_test_support_eh,stop_logging,[]} ]; -test_events(order_of_suites_in_multiple_dirs) -> +test_events(order_of_suites_in_multiple_dirs_no_merge_tests) -> [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, {ct_test_support_eh,tc_start,{groups_12_SUITE,init_per_suite}}, {ct_test_support_eh,tc_done,{groups_12_SUITE,init_per_suite,'_'}}, @@ -542,7 +701,7 @@ test_events(order_of_suites_in_multiple_dirs) -> {ct_test_support_eh,tc_start,{groups_11_SUITE,end_per_suite}}, {ct_test_support_eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, {ct_test_support_eh,stop_logging,[]}]; -test_events(order_of_groups_in_multiple_dirs) -> +test_events(order_of_groups_in_multiple_dirs_no_merge_tests) -> [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, {ct_test_support_eh,tc_start, @@ -561,7 +720,7 @@ test_events(order_of_groups_in_multiple_dirs) -> {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, {ct_test_support_eh,stop_logging,[]}]; -test_events(order_of_groups_in_multiple_suites) -> +test_events(order_of_groups_in_multiple_suites_no_merge_tests) -> [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, {ct_test_support_eh,tc_start, @@ -580,6 +739,131 @@ test_events(order_of_groups_in_multiple_suites) -> {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, {ct_test_support_eh,stop_logging,[]}]; +test_events(order_of_tests_in_multiple_suites_with_skip_no_merge_tests) -> + [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, + {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {ct_test_support_eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, + {ct_test_support_eh,tc_start,{groups_11_SUITE,testcase_1}}, + {ct_test_support_eh,tc_done,{groups_11_SUITE,testcase_1,ok}}, + {ct_test_support_eh,tc_user_skip,{groups_12_SUITE,testcase_1b,'_'}}, + {ct_test_support_eh,tc_start,{groups_11_SUITE,testcase_2}}, + {ct_test_support_eh,tc_done,{groups_11_SUITE,testcase_2,ok}}, + {ct_test_support_eh,stop_logging,[]} + ]; + +test_events(order_of_tests_in_multiple_dirs) -> + [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, + {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {ct_test_support_eh,tc_done, + {groups_12_SUITE,testcase_1a, + {failed,{error,{test_case_failed,no_group_data}}}}}, + {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1b}}, + {ct_test_support_eh,tc_done, + {groups_12_SUITE,testcase_1b, + {failed,{error,{test_case_failed,no_group_data}}}}}, + {ct_test_support_eh,tc_start,{groups_22_SUITE,testcase_1}}, + {ct_test_support_eh,tc_done,{groups_22_SUITE,testcase_1,ok}}, + {ct_test_support_eh,stop_logging,[]} + ]; +test_events(order_of_tests_in_multiple_suites) -> + [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, + {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {ct_test_support_eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, + + {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1b}}, + {ct_test_support_eh,tc_done,{groups_12_SUITE,testcase_1b,'_'}}, + + {ct_test_support_eh,tc_start,{groups_11_SUITE,testcase_1}}, + {ct_test_support_eh,tc_done,{groups_11_SUITE,testcase_1,ok}}, + {ct_test_support_eh,stop_logging,[]} + ]; +test_events(order_of_suites_in_multiple_dirs) -> + [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, + {ct_test_support_eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {ct_test_support_eh,tc_done,{groups_12_SUITE,init_per_suite,'_'}}, + {ct_test_support_eh,tc_start,{groups_12_SUITE,end_per_suite}}, + {ct_test_support_eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + + {ct_test_support_eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {ct_test_support_eh,tc_done,{groups_11_SUITE,init_per_suite,'_'}}, + {ct_test_support_eh,tc_start,{groups_11_SUITE,end_per_suite}}, + {ct_test_support_eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + + {ct_test_support_eh,tc_start,{groups_22_SUITE,init_per_suite}}, + {ct_test_support_eh,tc_done,{groups_22_SUITE,init_per_suite,'_'}}, + {ct_test_support_eh,tc_start,{groups_22_SUITE,end_per_suite}}, + {ct_test_support_eh,tc_done,{groups_22_SUITE,end_per_suite,'_'}}, + {ct_test_support_eh,stop_logging,[]}]; +test_events(order_of_groups_in_multiple_dirs) -> + [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, + + {ct_test_support_eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, + {ct_test_support_eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {ct_test_support_eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}}, + {ct_test_support_eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, + + {ct_test_support_eh,tc_start, + {groups_22_SUITE,{init_per_group,test_group_1a,'_'}}}, + {ct_test_support_eh,tc_done, + {groups_22_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {ct_test_support_eh,stop_logging,[]}]; +test_events(order_of_groups_in_multiple_suites) -> + [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, + + {ct_test_support_eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, + {ct_test_support_eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {ct_test_support_eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}}, + {ct_test_support_eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, + + {ct_test_support_eh,tc_start, + {groups_11_SUITE,{init_per_group,test_group_1a,'_'}}}, + {ct_test_support_eh,tc_done, + {groups_11_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {ct_test_support_eh,stop_logging,[]}]; + +test_events(order_of_tests_in_multiple_suites_with_skip) -> + [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, + {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {ct_test_support_eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, + {ct_test_support_eh,tc_user_skip,{groups_12_SUITE,testcase_1b,'_'}}, + {ct_test_support_eh,tc_start,{groups_11_SUITE,testcase_1}}, + {ct_test_support_eh,tc_done,{groups_11_SUITE,testcase_1,ok}}, + {ct_test_support_eh,tc_start,{groups_11_SUITE,testcase_2}}, + {ct_test_support_eh,tc_done,{groups_11_SUITE,testcase_2,ok}}, + {ct_test_support_eh,stop_logging,[]} + ]; + +test_events(all_plus_one_tc_no_merge_tests) -> + + [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, + {ct_test_support_eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {ct_test_support_eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {ct_test_support_eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {ct_test_support_eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {ct_test_support_eh,stop_logging,[]} + ]; + +test_events(all_plus_one_tc) -> + + [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, + {ct_test_support_eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {ct_test_support_eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{ct_test_support_eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {ct_test_support_eh,stop_logging,[]}} + ]; + test_events(_) -> [ ]. -- cgit v1.2.3 From d66d7ad951d89e5af7f10c171dd1c7a848990273 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Thu, 10 Feb 2011 17:03:08 +0100 Subject: Add test event info for the old tests --- lib/common_test/test/ct_testspec_1_SUITE.erl | 870 +++++++++++++++++++++------ 1 file changed, 690 insertions(+), 180 deletions(-) diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl index 7b09310e2e..88273edc22 100644 --- a/lib/common_test/test/ct_testspec_1_SUITE.erl +++ b/lib/common_test/test/ct_testspec_1_SUITE.erl @@ -61,30 +61,30 @@ all(doc) -> all(suite) -> [ - %% all_suites, skip_all_suites, - %% suite, skip_suite, - %% all_testcases, skip_all_testcases, - %% testcase, skip_testcase, - %% all_groups, skip_all_groups, - %% group, skip_group, - %% group_all_testcases, skip_group_all_testcases, - %% group_testcase, skip_group_testcase, - %% topgroup, - %% subgroup, skip_subgroup, - %% subgroup_all_testcases, skip_subgroup_all_testcases, - %% subgroup_testcase, skip_subgroup_testcase, - %% sub_skipped_by_top, - %% testcase_in_multiple_groups, - %% order_of_tests_in_multiple_dirs_no_merge_tests, - %% order_of_tests_in_multiple_suites_no_merge_tests, - %% order_of_suites_in_multiple_dirs_no_merge_tests, - %% order_of_groups_in_multiple_dirs_no_merge_tests, - %% order_of_groups_in_multiple_suites_no_merge_tests, - %% order_of_tests_in_multiple_dirs, - %% order_of_tests_in_multiple_suites, - %% order_of_suites_in_multiple_dirs, - %% order_of_groups_in_multiple_dirs, - %% order_of_groups_in_multiple_suites, + all_suites, skip_all_suites, + suite, skip_suite, + all_testcases, skip_all_testcases, + testcase, skip_testcase, + all_groups, skip_all_groups, + group, skip_group, + group_all_testcases, skip_group_all_testcases, + group_testcase, skip_group_testcase, + topgroup, + subgroup, skip_subgroup, + subgroup_all_testcases, skip_subgroup_all_testcases, + subgroup_testcase, skip_subgroup_testcase, + sub_skipped_by_top, + testcase_in_multiple_groups, + order_of_tests_in_multiple_dirs_no_merge_tests, + order_of_tests_in_multiple_suites_no_merge_tests, + order_of_suites_in_multiple_dirs_no_merge_tests, + order_of_groups_in_multiple_dirs_no_merge_tests, + order_of_groups_in_multiple_suites_no_merge_tests, + order_of_tests_in_multiple_dirs, + order_of_tests_in_multiple_suites, + order_of_suites_in_multiple_dirs, + order_of_groups_in_multiple_dirs, + order_of_groups_in_multiple_suites, order_of_tests_in_multiple_suites_with_skip_no_merge_tests, order_of_tests_in_multiple_suites_with_skip, all_plus_one_tc_no_merge_tests, @@ -661,207 +661,717 @@ events_to_check(_, 0) -> events_to_check(Test, N) -> test_events(Test) ++ events_to_check(Test, N-1). +test_events(all_suites) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{simple_1_SUITE,init_per_suite}}, + {?eh,tc_done,{simple_1_SUITE,end_per_suite,'_'}}, + {?eh,tc_start,{simple_2_SUITE,init_per_suite}}, + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,tc_done,{simple_2_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_all_suites) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_user_skip,{simple_1_SUITE,all,"SKIPPED!"}}, + {?eh,tc_user_skip,{simple_2_SUITE,all,"SKIPPED!"}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(suite) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{simple_1_SUITE,init_per_suite}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_done,{simple_1_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_suite) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_user_skip,{simple_1_SUITE,all,"SKIPPED!"}}, + {?eh,tc_done,{simple_2_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(all_testcases) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{simple_1_SUITE,init_per_suite}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_done,{simple_1_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_all_testcases) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_user_skip,{simple_1_SUITE,all,"SKIPPED!"}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(testcase) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{simple_1_SUITE,init_per_suite}}, + {?eh,test_stats,{1,0,{0,0}}}, + {negative,{?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_done,{simple_1_SUITE,end_per_suite,'_'}}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_testcase) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{simple_1_SUITE,init_per_suite}}, + {?eh,tc_user_skip,{simple_1_SUITE,tc1,"SKIPPED!"}}, + {?eh,tc_start,{simple_1_SUITE,tc2}}, + {?eh,tc_start,{simple_1_SUITE,end_per_suite}}, + + {?eh,tc_start,{simple_2_SUITE,init_per_suite}}, + {?eh,tc_user_skip,{simple_2_SUITE,tc2,"SKIPPED!"}}, + {?eh,tc_start,{simple_2_SUITE,tc1}}, + {?eh,test_stats,{2,0,{2,0}}}, + {?eh,tc_start,{simple_2_SUITE,end_per_suite}}, + + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(all_groups) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,test_stats,{12,0,{0,0}}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_all_groups) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1a},"SKIPPED!"}}, + {?eh,test_stats,{0,0,{1,0}}}, + {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}}, + {?eh,test_stats,{0,0,{2,0}}}, + {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_2},"SKIPPED!"}}, + {?eh,test_stats,{0,0,{3,0}}}, + {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_4},"SKIPPED!"}}, + {?eh,test_stats,{0,0,{4,0}}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(group) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1a}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1b}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_group) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1a}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1b}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}}, + + {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}}, + {?eh,test_stats,{2,0,{1,0}}}, + {negative,{?eh,tc_user_skip,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(group_all_testcases) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1a}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1b}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_group_all_testcases) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1a},"SKIPPED!"}}, + {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}}, + {?eh,test_stats,{0,0,{2,0}}}, + {?eh,tc_start,{groups_11_SUITE,end_per_suite}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(group_testcase) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1a}}, + {?eh,test_stats,{1,0,{0,0}}}, + {negative,{?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}}}, + + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_group_testcase) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1a}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}}, + {?eh,test_stats,{1,0,{1,0}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}}, + + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1b,[]}}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1b}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}}, + {?eh,test_stats,{2,0,{2,0}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1b,[]},'_'}}, + + {negative,{?eh,tc_user_skip,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(topgroup) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]},ok}}, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}} + ], + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[]}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[]}}} + ], + {?eh,test_stats,{6,0,{0,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]},ok}}]}, + + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}}, + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}], + {shuffle, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_8, + [{shuffle,'_'},sequence]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_8, + [{shuffle,'_'},sequence]},ok}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_8, + [shuffle,sequence]}}}, + {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_8, + [shuffle,sequence]},ok}} + ]}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}} + ]}, + {?eh,test_stats,{12,0,{0,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}], + + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(subgroup) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]},ok}}, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}} + ], + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[]}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[]}}} + ], + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]},ok}}]}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_subgroup) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}}, + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}], + {?eh,tc_user_skip, + {groups_12_SUITE,{group,test_group_8},"SKIPPED!"}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}} + ]}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}], + + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(subgroup_all_testcases) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}}, + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}], + {shuffle, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_8, + [{shuffle,'_'},sequence]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_8, + [{shuffle,'_'},sequence]},ok}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_8, + [shuffle,sequence]}}}, + {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_8, + [shuffle,sequence]},ok}} + ]}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}} + ]}, + {?eh,test_stats,{6,0,{0,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}], + + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]},ok}}, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}} + ], + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[]}}}, + {?eh,test_stats,{10,0,{0,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[]}}} + ], + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]},ok}}]}, + + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_subgroup_all_testcases) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_4,[]},ok}}, + {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_5},"SKIPPED!"}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}, + {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_4,[]},ok}} + ], + + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(subgroup_testcase) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}}, + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}], + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}} + ]}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}], + + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]},ok}}, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}} + ], + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[]}}}, + {?eh,test_stats,{3,0,{0,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[]}}} + ], + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]},ok}}]}, + + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_subgroup_testcase) -> + [ + + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}}, + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}}, + {?eh,tc_user_skip, {groups_12_SUITE,testcase_7a,"SKIPPED!"}}, + {?eh,test_stats,{1,0,{1,0}}}, + {?eh,tc_user_skip, {groups_12_SUITE,testcase_7b,"SKIPPED!"}}, + {?eh,test_stats,{1,0,{2,0}}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}], + {shuffle, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_8, + [{shuffle,'_'},sequence]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_8, + [{shuffle,'_'},sequence]},ok}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_8, + [shuffle,sequence]}}}, + {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_8, + [shuffle,sequence]},ok}} + ]}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}} + ]}, + {?eh,test_stats,{4,0,{2,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}], + + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + + ]; + +test_events(sub_skipped_by_top) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_4},"SKIPPED!"}}, + + {negative, + {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_4},"SKIPPED!"}}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}}, + + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(testcase_in_multiple_groups) -> + []; test_events(order_of_tests_in_multiple_dirs_no_merge_tests) -> - [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, - {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1a}}, - {ct_test_support_eh,tc_done, - {groups_12_SUITE,testcase_1a, - {failed,{error,{test_case_failed,no_group_data}}}}}, - {ct_test_support_eh,tc_start,{groups_22_SUITE,testcase_1}}, - {ct_test_support_eh,tc_done,{groups_22_SUITE,testcase_1,ok}}, - {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1b}}, - {ct_test_support_eh,tc_done, - {groups_12_SUITE,testcase_1b, - {failed,{error,{test_case_failed,no_group_data}}}}}, - {ct_test_support_eh,stop_logging,[]} + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {?eh,tc_done, {groups_12_SUITE,testcase_1a, + {failed,{error,{test_case_failed,no_group_data}}}}}, + {?eh,tc_start,{groups_22_SUITE,testcase_1}}, + {?eh,tc_done,{groups_22_SUITE,testcase_1,ok}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1b}}, + {?eh,tc_done, {groups_12_SUITE,testcase_1b, + {failed,{error,{test_case_failed,no_group_data}}}}}, + {?eh,stop_logging,[]} ]; test_events(order_of_tests_in_multiple_suites_no_merge_tests) -> - [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, - {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1a}}, - {ct_test_support_eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, - {ct_test_support_eh,tc_start,{groups_11_SUITE,testcase_1}}, - {ct_test_support_eh,tc_done,{groups_11_SUITE,testcase_1,ok}}, - {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1b}}, - {ct_test_support_eh,tc_done,{groups_12_SUITE,testcase_1b,'_'}}, - {ct_test_support_eh,stop_logging,[]} + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1}}, + {?eh,tc_done,{groups_11_SUITE,testcase_1,ok}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1b}}, + {?eh,tc_done,{groups_12_SUITE,testcase_1b,'_'}}, + {?eh,stop_logging,[]} ]; test_events(order_of_suites_in_multiple_dirs_no_merge_tests) -> - [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, - {ct_test_support_eh,tc_start,{groups_12_SUITE,init_per_suite}}, - {ct_test_support_eh,tc_done,{groups_12_SUITE,init_per_suite,'_'}}, - {ct_test_support_eh,tc_start,{groups_12_SUITE,end_per_suite}}, - {ct_test_support_eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, - {ct_test_support_eh,tc_start,{groups_22_SUITE,init_per_suite}}, - {ct_test_support_eh,tc_done,{groups_22_SUITE,init_per_suite,'_'}}, - {ct_test_support_eh,tc_start,{groups_22_SUITE,end_per_suite}}, - {ct_test_support_eh,tc_done,{groups_22_SUITE,end_per_suite,'_'}}, - {ct_test_support_eh,tc_start,{groups_11_SUITE,init_per_suite}}, - {ct_test_support_eh,tc_done,{groups_11_SUITE,init_per_suite,'_'}}, - {ct_test_support_eh,tc_start,{groups_11_SUITE,end_per_suite}}, - {ct_test_support_eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, - {ct_test_support_eh,stop_logging,[]}]; + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,init_per_suite,'_'}}, + {?eh,tc_start,{groups_12_SUITE,end_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {?eh,tc_start,{groups_22_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_22_SUITE,init_per_suite,'_'}}, + {?eh,tc_start,{groups_22_SUITE,end_per_suite}}, + {?eh,tc_done,{groups_22_SUITE,end_per_suite,'_'}}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_11_SUITE,init_per_suite,'_'}}, + {?eh,tc_start,{groups_11_SUITE,end_per_suite}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {?eh,stop_logging,[]}]; test_events(order_of_groups_in_multiple_dirs_no_merge_tests) -> - [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, + [{?eh,start_logging,{'DEF','RUNDIR'}}, - {ct_test_support_eh,tc_start, - {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, - {ct_test_support_eh,tc_done, - {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, - - {ct_test_support_eh,tc_start, - {groups_22_SUITE,{init_per_group,test_group_1a,'_'}}}, - {ct_test_support_eh,tc_done, - {groups_22_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {?eh,tc_start, {groups_22_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_22_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, - {ct_test_support_eh,tc_start, - {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}}, - {ct_test_support_eh,tc_done, - {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, - {ct_test_support_eh,stop_logging,[]}]; + {?eh,stop_logging,[]}]; test_events(order_of_groups_in_multiple_suites_no_merge_tests) -> - [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, + [{?eh,start_logging,{'DEF','RUNDIR'}}, - {ct_test_support_eh,tc_start, - {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, - {ct_test_support_eh,tc_done, - {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, - - {ct_test_support_eh,tc_start, - {groups_11_SUITE,{init_per_group,test_group_1a,'_'}}}, - {ct_test_support_eh,tc_done, - {groups_11_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {?eh,tc_start, {groups_11_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_11_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, - {ct_test_support_eh,tc_start, - {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}}, - {ct_test_support_eh,tc_done, - {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, - {ct_test_support_eh,stop_logging,[]}]; + {?eh,stop_logging,[]}]; test_events(order_of_tests_in_multiple_suites_with_skip_no_merge_tests) -> - [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, - {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1a}}, - {ct_test_support_eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, - {ct_test_support_eh,tc_start,{groups_11_SUITE,testcase_1}}, - {ct_test_support_eh,tc_done,{groups_11_SUITE,testcase_1,ok}}, - {ct_test_support_eh,tc_user_skip,{groups_12_SUITE,testcase_1b,'_'}}, - {ct_test_support_eh,tc_start,{groups_11_SUITE,testcase_2}}, - {ct_test_support_eh,tc_done,{groups_11_SUITE,testcase_2,ok}}, - {ct_test_support_eh,stop_logging,[]} + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1}}, + {?eh,tc_done,{groups_11_SUITE,testcase_1,ok}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_1b,'_'}}, + {?eh,tc_start,{groups_11_SUITE,testcase_2}}, + {?eh,tc_done,{groups_11_SUITE,testcase_2,ok}}, + {?eh,stop_logging,[]} ]; test_events(order_of_tests_in_multiple_dirs) -> - [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, - {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1a}}, - {ct_test_support_eh,tc_done, + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {?eh,tc_done, {groups_12_SUITE,testcase_1a, {failed,{error,{test_case_failed,no_group_data}}}}}, - {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1b}}, - {ct_test_support_eh,tc_done, + {?eh,tc_start,{groups_12_SUITE,testcase_1b}}, + {?eh,tc_done, {groups_12_SUITE,testcase_1b, {failed,{error,{test_case_failed,no_group_data}}}}}, - {ct_test_support_eh,tc_start,{groups_22_SUITE,testcase_1}}, - {ct_test_support_eh,tc_done,{groups_22_SUITE,testcase_1,ok}}, - {ct_test_support_eh,stop_logging,[]} + {?eh,tc_start,{groups_22_SUITE,testcase_1}}, + {?eh,tc_done,{groups_22_SUITE,testcase_1,ok}}, + {?eh,stop_logging,[]} ]; test_events(order_of_tests_in_multiple_suites) -> - [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, - {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1a}}, - {ct_test_support_eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, - {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1b}}, - {ct_test_support_eh,tc_done,{groups_12_SUITE,testcase_1b,'_'}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1b}}, + {?eh,tc_done,{groups_12_SUITE,testcase_1b,'_'}}, - {ct_test_support_eh,tc_start,{groups_11_SUITE,testcase_1}}, - {ct_test_support_eh,tc_done,{groups_11_SUITE,testcase_1,ok}}, - {ct_test_support_eh,stop_logging,[]} + {?eh,tc_start,{groups_11_SUITE,testcase_1}}, + {?eh,tc_done,{groups_11_SUITE,testcase_1,ok}}, + {?eh,stop_logging,[]} ]; test_events(order_of_suites_in_multiple_dirs) -> - [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, - {ct_test_support_eh,tc_start,{groups_12_SUITE,init_per_suite}}, - {ct_test_support_eh,tc_done,{groups_12_SUITE,init_per_suite,'_'}}, - {ct_test_support_eh,tc_start,{groups_12_SUITE,end_per_suite}}, - {ct_test_support_eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, - - {ct_test_support_eh,tc_start,{groups_11_SUITE,init_per_suite}}, - {ct_test_support_eh,tc_done,{groups_11_SUITE,init_per_suite,'_'}}, - {ct_test_support_eh,tc_start,{groups_11_SUITE,end_per_suite}}, - {ct_test_support_eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, - - {ct_test_support_eh,tc_start,{groups_22_SUITE,init_per_suite}}, - {ct_test_support_eh,tc_done,{groups_22_SUITE,init_per_suite,'_'}}, - {ct_test_support_eh,tc_start,{groups_22_SUITE,end_per_suite}}, - {ct_test_support_eh,tc_done,{groups_22_SUITE,end_per_suite,'_'}}, - {ct_test_support_eh,stop_logging,[]}]; + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,init_per_suite,'_'}}, + {?eh,tc_start,{groups_12_SUITE,end_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_11_SUITE,init_per_suite,'_'}}, + {?eh,tc_start,{groups_11_SUITE,end_per_suite}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + + {?eh,tc_start,{groups_22_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_22_SUITE,init_per_suite,'_'}}, + {?eh,tc_start,{groups_22_SUITE,end_per_suite}}, + {?eh,tc_done,{groups_22_SUITE,end_per_suite,'_'}}, + {?eh,stop_logging,[]}]; test_events(order_of_groups_in_multiple_dirs) -> - [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, + [{?eh,start_logging,{'DEF','RUNDIR'}}, - {ct_test_support_eh,tc_start, - {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, - {ct_test_support_eh,tc_done, - {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, - - {ct_test_support_eh,tc_start, - {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}}, - {ct_test_support_eh,tc_done, - {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, - - {ct_test_support_eh,tc_start, - {groups_22_SUITE,{init_per_group,test_group_1a,'_'}}}, - {ct_test_support_eh,tc_done, - {groups_22_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, - - {ct_test_support_eh,stop_logging,[]}]; + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, + + {?eh,tc_start, {groups_22_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_22_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {?eh,stop_logging,[]}]; test_events(order_of_groups_in_multiple_suites) -> - [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, + [{?eh,start_logging,{'DEF','RUNDIR'}}, - {ct_test_support_eh,tc_start, - {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, - {ct_test_support_eh,tc_done, - {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, - {ct_test_support_eh,tc_start, - {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}}, - {ct_test_support_eh,tc_done, - {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, - {ct_test_support_eh,tc_start, - {groups_11_SUITE,{init_per_group,test_group_1a,'_'}}}, - {ct_test_support_eh,tc_done, - {groups_11_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + {?eh,tc_start, {groups_11_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_11_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, - {ct_test_support_eh,stop_logging,[]}]; + {?eh,stop_logging,[]}]; test_events(order_of_tests_in_multiple_suites_with_skip) -> - [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, - {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1a}}, - {ct_test_support_eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, - {ct_test_support_eh,tc_user_skip,{groups_12_SUITE,testcase_1b,'_'}}, - {ct_test_support_eh,tc_start,{groups_11_SUITE,testcase_1}}, - {ct_test_support_eh,tc_done,{groups_11_SUITE,testcase_1,ok}}, - {ct_test_support_eh,tc_start,{groups_11_SUITE,testcase_2}}, - {ct_test_support_eh,tc_done,{groups_11_SUITE,testcase_2,ok}}, - {ct_test_support_eh,stop_logging,[]} + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_1b,'_'}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1}}, + {?eh,tc_done,{groups_11_SUITE,testcase_1,ok}}, + {?eh,tc_start,{groups_11_SUITE,testcase_2}}, + {?eh,tc_done,{groups_11_SUITE,testcase_2,ok}}, + {?eh,stop_logging,[]} ]; test_events(all_plus_one_tc_no_merge_tests) -> - [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, - {ct_test_support_eh,tc_start,{groups_12_SUITE,init_per_suite}}, - {ct_test_support_eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, - {ct_test_support_eh,tc_start,{groups_12_SUITE,init_per_suite}}, - {ct_test_support_eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, - {ct_test_support_eh,stop_logging,[]} + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {?eh,stop_logging,[]} ]; test_events(all_plus_one_tc) -> - [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, - {ct_test_support_eh,tc_start,{groups_12_SUITE,init_per_suite}}, - {ct_test_support_eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, - {negative,{ct_test_support_eh,tc_start,{groups_12_SUITE,init_per_suite}}, - {ct_test_support_eh,stop_logging,[]}} + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {?eh,stop_logging,[]}} ]; test_events(_) -> -- cgit v1.2.3