diff options
Diffstat (limited to 'lib/common_test')
20 files changed, 1391 insertions, 395 deletions
diff --git a/lib/common_test/doc/src/ct_junit_report.xml b/lib/common_test/doc/src/ct_junit_report.xml deleted file mode 100644 index 49a40cc1de..0000000000 --- a/lib/common_test/doc/src/ct_junit_report.xml +++ /dev/null @@ -1,109 +0,0 @@ -<?xml version="1.0" encoding="latin1" ?> -<!DOCTYPE erlref SYSTEM "erlref.dtd"> -<erlref> -<header> -<title>ct_junit_report</title> -<prepared></prepared> -<responsible></responsible> -<docno>1</docno> -<approved></approved> -<checked></checked> -<date></date> -<rev>A</rev> -<file>ct_junit_report.xml</file></header> -<module>ct_junit_report</module> -<modulesummary>Common Test Framework functions handling test specifications.</modulesummary> -<description> -<p>Common Test Framework functions handling test specifications.</p> - - <p>This module creates a junit report of the test run if plugged in - as a suite_callback.</p></description> -<funcs> -<func> -<name>init(Opts) -> term() -</name> -<fsummary> </fsummary> - -<desc><marker id="init-1"/> - </desc></func> -<func> -<name>post_end_group(Group, Config, State) -> term() -</name> -<fsummary> </fsummary> - -<desc><marker id="post_end_group-3"/> - </desc></func> -<func> -<name>post_end_suite(Suite, Config, State) -> term() -</name> -<fsummary> </fsummary> - -<desc><marker id="post_end_suite-3"/> - </desc></func> -<func> -<name>post_end_tc(TC, Config, State) -> term() -</name> -<fsummary> </fsummary> - -<desc><marker id="post_end_tc-3"/> - </desc></func> -<func> -<name>post_init_group(Group, Config, State) -> term() -</name> -<fsummary> </fsummary> - -<desc><marker id="post_init_group-3"/> - </desc></func> -<func> -<name>post_init_suite(Suite, Config, State) -> term() -</name> -<fsummary> </fsummary> - -<desc><marker id="post_init_suite-3"/> - </desc></func> -<func> -<name>pre_end_group(Group, Config, State) -> term() -</name> -<fsummary> </fsummary> - -<desc><marker id="pre_end_group-3"/> - </desc></func> -<func> -<name>pre_end_suite(Suite, Config, State) -> term() -</name> -<fsummary> </fsummary> - -<desc><marker id="pre_end_suite-3"/> - </desc></func> -<func> -<name>pre_init_group(Group, Config, State) -> term() -</name> -<fsummary> </fsummary> - -<desc><marker id="pre_init_group-3"/> - </desc></func> -<func> -<name>pre_init_suite(Suite, Config, State) -> term() -</name> -<fsummary> </fsummary> - -<desc><marker id="pre_init_suite-3"/> - </desc></func> -<func> -<name>pre_init_tc(TC, Config, State) -> term() -</name> -<fsummary> </fsummary> - -<desc><marker id="pre_init_tc-3"/> - </desc></func> -<func> -<name>terminate(Config, State) -> term() -</name> -<fsummary> </fsummary> - -<desc><marker id="terminate-2"/> - </desc></func></funcs> - -<authors> -<aname> </aname> -<email> </email></authors></erlref>
\ No newline at end of file diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index 7b485bdc35..f052adf25e 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -368,15 +368,17 @@ either one or more suites, one or more test case groups, or one or more test cases in a group or suite.</p> <p>An arbitrary number of test terms may be declared in sequence. - Common Test will compile the terms into one or more tests to be - performed in one resulting test run. Note that a term that + Common Test will by default compile the terms into one or more tests + to be performed in one resulting test run. Note that a term that specifies a set of test cases will "swallow" one that only specifies a subset of these cases. E.g. the result of merging one term that specifies that all cases in suite S should be executed, with another term specifying only test case X and Y in S, is a test of all cases in S. However, if a term specifying test case X and Y in S is merged with a term specifying case Z - in S, the result is a test of X, Y and Z in S.</p> + in S, the result is a test of X, Y and Z in S. To disable this + behaviour, it is possible in test specification to set the + <c>merge_tests</c> term to <c>false</c>.</p> <p>A test term can also specify one or more test suites, groups, or test cases to be skipped. Skipped suites, groups and cases are not executed and show up in the HTML test log files as @@ -435,6 +437,8 @@ {userconfig, NodeRefs, {CallbackModule, ConfigStrings}}. {alias, DirAlias, Dir}. + + {merge_tests, Bool}. {logdir, LogDir}. {logdir, NodeRefs, LogDir}. diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 04829004f4..b61eda7152 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -743,12 +743,12 @@ get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) -> %% init/end functions for top groups will be executed case catch proplists:get_value(name, element(2, hd(ConfTests))) of Name -> % top group - ConfTests; + delete_subs(ConfTests, ConfTests); _ -> [] end; false -> - ConfTests + delete_subs(ConfTests, ConfTests) end end; _ -> @@ -765,9 +765,25 @@ get_suite(Mod, Name) -> find_groups(Mod, Name, TCs, GroupDefs) -> Found = find(Mod, Name, TCs, GroupDefs, [], GroupDefs, false), Trimmed = trim(Found), - delete_subs(Trimmed, Trimmed). - -find(Mod, all, _TCs, [{Name,Props,Tests} | Gs], Known, Defs, _) -> + %% I cannot find a reason to why this function is called, + %% It deletes any group which is referenced in any other + %% group. i.e. + %% groups() -> + %% [{test, [], [testcase1]}, + %% {testcases, [], [{group, test}]}]. + %% Would be changed to + %% groups() -> + %% [{testcases, [], [testcase1]}]. + %% instead of what I believe is correct: + %% groups() -> + %% [{test, [], [testcase1]}, + %% {testcases, [], [testcase1]}]. + %% Have to double check with peppe + delete_subs(Trimmed, Trimmed), + Trimmed. + +find(Mod, all, _TCs, [{Name,Props,Tests} | Gs], Known, Defs, _) + when is_atom(Name), is_list(Props), is_list(Tests) -> cyclic_test(Mod, Name, Known), [make_conf(Mod, Name, Props, find(Mod, all, all, Tests, [Name | Known], Defs, true)) | @@ -789,8 +805,8 @@ find(Mod, Name, TCs, [{Name,Props,Tests} | _Gs], Known, Defs, false) find(Mod, Name, TCs, [{Name1,Props,Tests} | Gs], Known, Defs, false) when is_atom(Name1), is_list(Props), is_list(Tests) -> cyclic_test(Mod, Name1, Known), - [make_conf(Mod, Name1, Props, - find(Mod, Name, TCs, Tests, [Name1 | Known], Defs, false)) | + [make_conf(Mod,Name1,Props, + find(Mod, Name, TCs, Tests, [Name1 | Known], Defs, false)) | find(Mod, Name, TCs, Gs, [], Defs, false)]; find(Mod, Name, _TCs, [{Name,_Props,_Tests} | _Gs], _Known, _Defs, true) @@ -806,17 +822,31 @@ find(Mod, Name, all, [{Name1,Props,Tests} | Gs], Known, Defs, true) find(Mod, Name, all, Tests, [Name1 | Known], Defs, true)) | find(Mod, Name, all, Gs, [], Defs, true)]; -find(Mod, Name, TCs, [{group,Name1} | Gs], Known, Defs, Found) when is_atom(Name1) -> +find(Mod, Name, TCs, [{group,Name1} | Gs], Known, Defs, Found) + when is_atom(Name1) -> find(Mod, Name, TCs, [expand(Mod, Name1, Defs) | Gs], Known, Defs, Found); +%% Undocumented remote group feature, use with caution +find(Mod, Name, TCs, [{group, ExtMod, ExtGrp} | Gs], Known, Defs, true) + when is_atom(ExtMod), is_atom(ExtGrp) -> + ExternalDefs = ExtMod:groups(), + ExternalTCs = find(ExtMod, ExtGrp, TCs, [{group, ExtGrp}], + [], ExternalDefs, false), + ExternalTCs ++ find(Mod, Name, TCs, Gs, Known, Defs, true); + find(Mod, Name, TCs, [{Name1,Tests} | Gs], Known, Defs, Found) when is_atom(Name1), is_list(Tests) -> find(Mod, Name, TCs, [{Name1,[],Tests} | Gs], Known, Defs, Found); -find(Mod, Name, TCs, [TC | Gs], Known, Defs, false) when is_atom(TC) -> +find(Mod, Name, TCs, [_TC | Gs], Known, Defs, false) -> find(Mod, Name, TCs, Gs, Known, Defs, false); find(Mod, Name, TCs, [TC | Gs], Known, Defs, true) when is_atom(TC) -> + [{Mod, TC} | find(Mod, Name, TCs, Gs, Known, Defs, true)]; + +find(Mod, Name, TCs, [{ExternalTC, Case} = TC | Gs], Known, Defs, true) + when is_atom(ExternalTC), + is_atom(Case) -> [TC | find(Mod, Name, TCs, Gs, Known, Defs, true)]; find(Mod, _Name, _TCs, [BadTerm | _Gs], Known, _Defs, _Found) -> @@ -836,7 +866,7 @@ find(_Mod, _Name, _TCs, [], _Known, _Defs, false) -> find(_Mod, _Name, _TCs, [], _Known, _Defs, _Found) -> []. -delete_subs([Conf | Confs], All) -> +delete_subs([{conf, _,_,_,_} = Conf | Confs], All) -> All1 = delete_conf(Conf, All), case is_sub(Conf, All1) of true -> @@ -844,7 +874,8 @@ delete_subs([Conf | Confs], All) -> false -> delete_subs(Confs, All) end; - +delete_subs([_Else | Confs], All) -> + delete_subs(Confs, All); delete_subs([], All) -> All. @@ -936,7 +967,9 @@ make_all_conf(Mod) -> [] -> {error,{invalid_group_spec,Mod}}; ConfTests -> - [{conf,Props,Init,all,End} || {conf,Props,Init,_,End} <- ConfTests] + [{conf,Props,Init,all,End} || + {conf,Props,Init,_,End} + <- delete_subs(ConfTests, ConfTests)] end end. @@ -982,31 +1015,11 @@ get_all(Mod, ConfTests) -> [{?MODULE,error_in_suite,[[{error,What}]]}]; SeqsAndTCs -> %% expand group references in all() using ConfTests - Expand = - fun({group,Name}) -> - FindConf = - fun({conf,Props,_,_,_}) -> - case proplists:get_value(name, Props) of - Name -> true; - _ -> false - end - end, - case lists:filter(FindConf, ConfTests) of - [ConfTest|_] -> - ConfTest; - [] -> - E = "Invalid reference to group "++ - atom_to_list(Name)++" in "++ - atom_to_list(Mod)++":all/0", - throw({error,list_to_atom(E)}) - end; - (SeqOrTC) -> SeqOrTC - end, - case catch lists:map(Expand, SeqsAndTCs) of + case catch expand_groups(SeqsAndTCs, ConfTests, Mod) of {error,_} = Error -> [{?MODULE,error_in_suite,[[Error]]}]; Tests -> - Tests + delete_subs(Tests, Tests) end end; Skip = {skip,_Reason} -> @@ -1017,6 +1030,30 @@ get_all(Mod, ConfTests) -> [{?MODULE,error_in_suite,[[{error,Reason}]]}] end. +expand_groups([H | T], ConfTests, Mod) -> + [expand_groups(H, ConfTests, Mod) | expand_groups(T, ConfTests, Mod)]; +expand_groups([], _ConfTests, _Mod) -> + []; +expand_groups({group,Name}, ConfTests, Mod) -> + FindConf = + fun({conf,Props,_,_,_}) -> + case proplists:get_value(name, Props) of + Name -> true; + _ -> false + end + end, + case lists:filter(FindConf, ConfTests) of + [ConfTest|_] -> + expand_groups(ConfTest, ConfTests, Mod); + [] -> + E = "Invalid reference to group "++ + atom_to_list(Name)++" in "++ + atom_to_list(Mod)++":all/0", + throw({error,list_to_atom(E)}) + end; +expand_groups(SeqOrTC, _ConfTests, _Mod) -> + SeqOrTC. + %%!============================================================ %%! The support for sequences by means of using sequences/0 diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index 2b6abefb72..2c7cd3577c 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -68,7 +68,8 @@ prepare_tests(TestSpec) when is_record(TestSpec,testspec) -> %% Create initial list of {Node,{Run,Skip}} tuples NodeList = lists:map(fun(N) -> {N,{[],[]}} end, list_nodes(TestSpec)), %% Get all Run tests sorted per node basis. - NodeList1 = run_per_node(Run,NodeList), + NodeList1 = run_per_node(Run,NodeList, + TestSpec#testspec.merge_tests), %% Get all Skip entries sorted per node basis. NodeList2 = skip_per_node(Skip,NodeList1), %% Change representation. @@ -89,11 +90,17 @@ prepare_tests(TestSpec) when is_record(TestSpec,testspec) -> %% run_per_node/2 takes the Run list as input and returns a list %% of {Node,RunPerNode,[]} tuples where the tests have been sorted %% on a per node basis. -run_per_node([{{Node,Dir},Test}|Ts],Result) -> +run_per_node([{{Node,Dir},Test}|Ts],Result, MergeTests) -> {value,{Node,{Run,Skip}}} = lists:keysearch(Node,1,Result), - Run1 = merge_tests(Dir,Test,Run), - run_per_node(Ts,insert_in_order({Node,{Run1,Skip}},Result)); -run_per_node([],Result) -> + Run1 = case MergeTests of + false -> + append({Dir, Test}, Run); + true -> + merge_tests(Dir,Test,Run) + end, + run_per_node(Ts,insert_in_order({Node,{Run1,Skip}},Result), + MergeTests); +run_per_node([],Result,_) -> Result. merge_tests(Dir,Test={all,_},TestDirs) -> @@ -281,6 +288,8 @@ collect_tests(Terms,TestSpec,Relaxed) -> {Terms2, TestSpec3} = filter_init_terms(Terms, [], TestSpec2), add_tests(Terms2,TestSpec3). +get_global([{merge_tests, Bool} | Ts], Spec) -> + get_global(Ts,Spec#testspec{ merge_tests = Bool }); get_global([{alias,Ref,Dir}|Ts],Spec=#testspec{alias=Refs}) -> get_global(Ts,Spec#testspec{alias=[{Ref,get_absdir(Dir,Spec)}|Refs]}); get_global([{node,Ref,Node}|Ts],Spec=#testspec{nodes=Refs}) -> @@ -668,7 +677,7 @@ add_tests([{suites,Node,Dir,Ss}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = insert_suites(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Ss,Tests), + Ss,Tests, Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); %% --- groups --- @@ -694,13 +703,15 @@ add_tests([{groups,Node,Dir,Suite,Gs}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = insert_groups(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Suite,Gs,all,Tests), + Suite,Gs,all,Tests, + Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); add_tests([{groups,Node,Dir,Suite,Gs,{cases,TCs}}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = insert_groups(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Suite,Gs,TCs,Tests), + Suite,Gs,TCs,Tests, + Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); %% --- cases --- @@ -715,7 +726,7 @@ add_tests([{cases,Node,Dir,Suite,Cs}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = insert_cases(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Suite,Cs,Tests), + Suite,Cs,Tests, Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); %% --- skip_suites --- @@ -730,7 +741,8 @@ add_tests([{skip_suites,Node,Dir,Ss,Cmt}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = skip_suites(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Ss,Cmt,Tests), + Ss,Cmt,Tests, + Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); %% --- skip_groups --- @@ -752,13 +764,15 @@ add_tests([{skip_groups,Node,Dir,Suite,Gs,Cmt}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = skip_groups(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Suite,Gs,all,Cmt,Tests), + Suite,Gs,all,Cmt,Tests, + Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); add_tests([{skip_groups,Node,Dir,Suite,Gs,{cases,TCs},Cmt}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = skip_groups(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Suite,Gs,TCs,Cmt,Tests), + Suite,Gs,TCs,Cmt,Tests, + Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); %% --- skip_cases --- @@ -773,7 +787,7 @@ add_tests([{skip_cases,Node,Dir,Suite,Cs,Cmt}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = skip_cases(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Suite,Cs,Cmt,Tests), + Suite,Cs,Cmt,Tests,Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); %% --- handled/errors --- @@ -783,6 +797,9 @@ add_tests([{alias,_,_}|Ts],Spec) -> % handled add_tests([{node,_,_}|Ts],Spec) -> % handled add_tests(Ts,Spec); +add_tests([{merge_tests, _} | Ts], Spec) -> % handled + add_tests(Ts,Spec); + %% check if it's a CT term that has bad format or if the user seems to %% have added something of his/her own, which we'll let pass if relaxed %% mode is enabled. @@ -835,17 +852,22 @@ separate([],_,_,_) -> %% {Suite2,[{GrOrCase21,{skip,Cmt}},GrOrCase22,...]},...]} %% GrOrCase = {GroupName,[Case1,Case2,...]} | Case -insert_suites(Node,Dir,[S|Ss],Tests) -> - Tests1 = insert_cases(Node,Dir,S,all,Tests), - insert_suites(Node,Dir,Ss,Tests1); -insert_suites(_Node,_Dir,[],Tests) -> +insert_suites(Node,Dir,[S|Ss],Tests, MergeTests) -> + Tests1 = insert_cases(Node,Dir,S,all,Tests,MergeTests), + insert_suites(Node,Dir,Ss,Tests1,MergeTests); +insert_suites(_Node,_Dir,[],Tests,_MergeTests) -> Tests; -insert_suites(Node,Dir,S,Tests) -> - insert_suites(Node,Dir,[S],Tests). +insert_suites(Node,Dir,S,Tests,MergeTests) -> + insert_suites(Node,Dir,[S],Tests,MergeTests). -insert_groups(Node,Dir,Suite,Group,Cases,Tests) when is_atom(Group) -> - insert_groups(Node,Dir,Suite,[Group],Cases,Tests); -insert_groups(Node,Dir,Suite,Groups,Cases,Tests) when +insert_groups(Node,Dir,Suite,Group,Cases,Tests,MergeTests) + when is_atom(Group) -> + insert_groups(Node,Dir,Suite,[Group],Cases,Tests,MergeTests); +insert_groups(Node,Dir,Suite,Groups,Cases,Tests,false) when + ((Cases == all) or is_list(Cases)) and is_list(Groups) -> + Groups1 = [{Gr,Cases} || Gr <- Groups], + append({{Node,Dir},[{Suite,Groups1}]},Tests); +insert_groups(Node,Dir,Suite,Groups,Cases,Tests,true) when ((Cases == all) or is_list(Cases)) and is_list(Groups) -> case lists:keysearch({Node,Dir},1,Tests) of {value,{{Node,Dir},[{all,_}]}} -> @@ -859,9 +881,10 @@ insert_groups(Node,Dir,Suite,Groups,Cases,Tests) when Groups1 = [{Gr,Cases} || Gr <- Groups], insert_in_order({{Node,Dir},[{Suite,Groups1}]},Tests) end; -insert_groups(Node,Dir,Suite,Groups,Case,Tests) when is_atom(Case) -> +insert_groups(Node,Dir,Suite,Groups,Case,Tests, MergeTests) + when is_atom(Case) -> Cases = if Case == all -> all; true -> [Case] end, - insert_groups(Node,Dir,Suite,Groups,Cases,Tests). + insert_groups(Node,Dir,Suite,Groups,Cases,Tests, MergeTests). insert_groups1(_Suite,_Groups,all) -> all; @@ -891,7 +914,9 @@ insert_groups2([Group={GrName,Cases}|Groups],GrAndCases) -> insert_groups2([],GrAndCases) -> GrAndCases. -insert_cases(Node,Dir,Suite,Cases,Tests) when is_list(Cases) -> +insert_cases(Node,Dir,Suite,Cases,Tests,false) when is_list(Cases) -> + append({{Node,Dir},[{Suite,Cases}]},Tests); +insert_cases(Node,Dir,Suite,Cases,Tests,true) when is_list(Cases) -> case lists:keysearch({Node,Dir},1,Tests) of {value,{{Node,Dir},[{all,_}]}} -> Tests; @@ -901,8 +926,8 @@ insert_cases(Node,Dir,Suite,Cases,Tests) when is_list(Cases) -> false -> insert_in_order({{Node,Dir},[{Suite,Cases}]},Tests) end; -insert_cases(Node,Dir,Suite,Case,Tests) when is_atom(Case) -> - insert_cases(Node,Dir,Suite,[Case],Tests). +insert_cases(Node,Dir,Suite,Case,Tests,MergeTests) when is_atom(Case) -> + insert_cases(Node,Dir,Suite,[Case],Tests,MergeTests). insert_cases1(_Suite,_Cases,all) -> all; @@ -917,22 +942,28 @@ insert_cases1(Suite,Cases,Suites0) -> insert_in_order({Suite,Cases},Suites0) end. -skip_suites(Node,Dir,[S|Ss],Cmt,Tests) -> - Tests1 = skip_cases(Node,Dir,S,all,Cmt,Tests), - skip_suites(Node,Dir,Ss,Cmt,Tests1); -skip_suites(_Node,_Dir,[],_Cmt,Tests) -> +skip_suites(Node,Dir,[S|Ss],Cmt,Tests,MergeTests) -> + Tests1 = skip_cases(Node,Dir,S,all,Cmt,Tests,MergeTests), + skip_suites(Node,Dir,Ss,Cmt,Tests1,MergeTests); +skip_suites(_Node,_Dir,[],_Cmt,Tests,_MergeTests) -> Tests; -skip_suites(Node,Dir,S,Cmt,Tests) -> - skip_suites(Node,Dir,[S],Cmt,Tests). - -skip_groups(Node,Dir,Suite,Group,all,Cmt,Tests) when is_atom(Group) -> - skip_groups(Node,Dir,Suite,[Group],all,Cmt,Tests); -skip_groups(Node,Dir,Suite,Group,Cases,Cmt,Tests) when is_atom(Group) -> - skip_groups(Node,Dir,Suite,[Group],Cases,Cmt,Tests); -skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests) when is_atom(Case), - Case =/= all -> - skip_groups(Node,Dir,Suite,Groups,[Case],Cmt,Tests); -skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests) when +skip_suites(Node,Dir,S,Cmt,Tests,MergeTests) -> + skip_suites(Node,Dir,[S],Cmt,Tests,MergeTests). + +skip_groups(Node,Dir,Suite,Group,all,Cmt,Tests,MergeTests) + when is_atom(Group) -> + skip_groups(Node,Dir,Suite,[Group],all,Cmt,Tests,MergeTests); +skip_groups(Node,Dir,Suite,Group,Cases,Cmt,Tests,MergeTests) + when is_atom(Group) -> + skip_groups(Node,Dir,Suite,[Group],Cases,Cmt,Tests,MergeTests); +skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests,MergeTests) + when is_atom(Case),Case =/= all -> + skip_groups(Node,Dir,Suite,Groups,[Case],Cmt,Tests,MergeTests); +skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests,false) when + ((Cases == all) or is_list(Cases)) and is_list(Groups) -> + Suites1 = skip_groups1(Suite,[{Gr,Cases} || Gr <- Groups],Cmt,[]), + append({{Node,Dir},Suites1},Tests); +skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests,true) when ((Cases == all) or is_list(Cases)) and is_list(Groups) -> Suites = case lists:keysearch({Node,Dir},1,Tests) of @@ -943,9 +974,10 @@ skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests) when end, Suites1 = skip_groups1(Suite,[{Gr,Cases} || Gr <- Groups],Cmt,Suites), insert_in_order({{Node,Dir},Suites1},Tests); -skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests) when is_atom(Case) -> +skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests,MergeTests) + when is_atom(Case) -> Cases = if Case == all -> all; true -> [Case] end, - skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests). + skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests,MergeTests). skip_groups1(Suite,Groups,Cmt,Suites0) -> SkipGroups = lists:map(fun(Group) -> @@ -959,7 +991,10 @@ skip_groups1(Suite,Groups,Cmt,Suites0) -> insert_in_order({Suite,SkipGroups},Suites0) end. -skip_cases(Node,Dir,Suite,Cases,Cmt,Tests) when is_list(Cases) -> +skip_cases(Node,Dir,Suite,Cases,Cmt,Tests,false) when is_list(Cases) -> + Suites1 = skip_cases1(Suite,Cases,Cmt,[]), + append({{Node,Dir},Suites1},Tests); +skip_cases(Node,Dir,Suite,Cases,Cmt,Tests,true) when is_list(Cases) -> Suites = case lists:keysearch({Node,Dir},1,Tests) of {value,{{Node,Dir},Suites0}} -> @@ -969,8 +1004,8 @@ skip_cases(Node,Dir,Suite,Cases,Cmt,Tests) when is_list(Cases) -> end, Suites1 = skip_cases1(Suite,Cases,Cmt,Suites), insert_in_order({{Node,Dir},Suites1},Tests); -skip_cases(Node,Dir,Suite,Case,Cmt,Tests) when is_atom(Case) -> - skip_cases(Node,Dir,Suite,[Case],Cmt,Tests). +skip_cases(Node,Dir,Suite,Case,Cmt,Tests,MergeTests) when is_atom(Case) -> + skip_cases(Node,Dir,Suite,[Case],Cmt,Tests,MergeTests). skip_cases1(Suite,Cases,Cmt,Suites0) -> SkipCases = lists:map(fun(C) -> @@ -984,6 +1019,9 @@ skip_cases1(Suite,Cases,Cmt,Suites0) -> insert_in_order({Suite,SkipCases},Suites0) end. +append(Elem, List) -> + List ++ [Elem]. + insert_in_order([E|Es],List) -> List1 = insert_elem(E,List,[]), insert_in_order(Es,List1); @@ -1056,6 +1094,7 @@ valid_terms() -> {userconfig,2}, {userconfig,3}, {alias,3}, + {merge_tests,1}, {logdir,2}, {logdir,3}, {label,2}, diff --git a/lib/common_test/src/ct_util.erl b/lib/common_test/src/ct_util.erl index 45146de57c..0476e1599f 100644 --- a/lib/common_test/src/ct_util.erl +++ b/lib/common_test/src/ct_util.erl @@ -288,6 +288,9 @@ loop(Mode,TestData,StartDir) -> TestData1 = lists:keydelete(Key,1,TestData), return(From,ok), loop(Mode,[New|TestData1],StartDir); + {{get_testdata, all}, From} -> + return(From, TestData), + loop(From, TestData, StartDir); {{get_testdata,Key},From} -> case lists:keysearch(Key,1,TestData) of {value,{Key,Val}} -> diff --git a/lib/common_test/src/ct_util.hrl b/lib/common_test/src/ct_util.hrl index 4ed29bdcd1..f0255c533c 100644 --- a/lib/common_test/src/ct_util.hrl +++ b/lib/common_test/src/ct_util.hrl @@ -41,7 +41,8 @@ multiply_timetraps=[], scale_timetraps=[], alias=[], - tests=[]}). + tests=[], + merge_tests = true }). -record(cover, {app=none, level=details, diff --git a/lib/common_test/test/common_test.spec b/lib/common_test/test/common_test.spec index 7619a75b31..8755b08117 100644 --- a/lib/common_test/test/common_test.spec +++ b/lib/common_test/test/common_test.spec @@ -1 +1 @@ -{topcase, {dir, "../common_test"}}. +{suites,"../common_test_test",all}.
\ No newline at end of file diff --git a/lib/common_test/test/ct_config_SUITE.erl b/lib/common_test/test/ct_config_SUITE.erl index fc15abc5bc..2ddc7d6422 100644 --- a/lib/common_test/test/ct_config_SUITE.erl +++ b/lib/common_test/test/ct_config_SUITE.erl @@ -29,7 +29,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct_event.hrl"). -define(eh, ct_test_support_eh). @@ -64,19 +64,22 @@ end_per_testcase(install_config = TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all(doc) -> - [""]; +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [require, install_config, userconfig_static, + userconfig_dynamic, testspec_legacy, testspec_static, + testspec_dynamic]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> - [ - require, - install_config, - userconfig_static, - userconfig_dynamic, - testspec_legacy, - testspec_static, - testspec_dynamic - ]. %%-------------------------------------------------------------------- %% TEST CASES @@ -199,74 +202,74 @@ events_to_check(Test, N) -> expected_events(Test) ++ events_to_check(Test, N-1). expected_events(config_static_SUITE)-> -[ - {?eh,start_logging,{'DEF','RUNDIR'}}, - {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, - {?eh,start_info,{1,1,8}}, - {?eh,tc_start,{config_static_SUITE,init_per_suite}}, - {?eh,tc_done,{config_static_SUITE,init_per_suite,ok}}, - {?eh,tc_start,{config_static_SUITE,test_get_config_simple}}, - {?eh,tc_done,{config_static_SUITE,test_get_config_simple,ok}}, - {?eh,test_stats,{1,0,{0,0}}}, - {?eh,tc_start,{config_static_SUITE,test_get_config_nested}}, - {?eh,tc_done,{config_static_SUITE,test_get_config_nested,ok}}, - {?eh,test_stats,{2,0,{0,0}}}, - {?eh,tc_start,{config_static_SUITE,test_default_suitewide}}, - {?eh,tc_done,{config_static_SUITE,test_default_suitewide,ok}}, - {?eh,test_stats,{3,0,{0,0}}}, - {?eh,tc_start,{config_static_SUITE,test_config_name_already_in_use1}}, - {?eh,tc_done, - {config_static_SUITE,test_config_name_already_in_use1,{skipped,{config_name_already_in_use,[x1]}}}}, - {?eh,test_stats,{3,0,{1,0}}}, - {?eh,tc_start,{config_static_SUITE,test_default_tclocal}}, - {?eh,tc_done,{config_static_SUITE,test_default_tclocal,ok}}, - {?eh,test_stats,{4,0,{1,0}}}, - {?eh,tc_start,{config_static_SUITE,test_config_name_already_in_use2}}, - {?eh,tc_done, - {config_static_SUITE,test_config_name_already_in_use2, - {skipped,{config_name_already_in_use,[x1,alias]}}}}, - {?eh,test_stats,{4,0,{2,0}}}, - {?eh,tc_start,{config_static_SUITE,test_alias_tclocal}}, - {?eh,tc_done,{config_static_SUITE,test_alias_tclocal,ok}}, - {?eh,test_stats,{5,0,{2,0}}}, - {?eh,tc_start,{config_static_SUITE,test_get_config_undefined}}, - {?eh,tc_done,{config_static_SUITE,test_get_config_undefined,ok}}, - {?eh,test_stats,{6,0,{2,0}}}, - {?eh,tc_start,{config_static_SUITE,end_per_suite}}, - {?eh,tc_done,{config_static_SUITE,end_per_suite,ok}}, - {?eh,test_done,{'DEF','STOP_TIME'}}, - {?eh,stop_logging,[]} -]; + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,8}}, + {?eh,tc_start,{config_static_SUITE,init_per_suite}}, + {?eh,tc_done,{config_static_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{config_static_SUITE,test_get_config_simple}}, + {?eh,tc_done,{config_static_SUITE,test_get_config_simple,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{config_static_SUITE,test_get_config_nested}}, + {?eh,tc_done,{config_static_SUITE,test_get_config_nested,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_start,{config_static_SUITE,test_default_suitewide}}, + {?eh,tc_done,{config_static_SUITE,test_default_suitewide,ok}}, + {?eh,test_stats,{3,0,{0,0}}}, + {?eh,tc_start,{config_static_SUITE,test_config_name_already_in_use1}}, + {?eh,tc_done, + {config_static_SUITE,test_config_name_already_in_use1,{skipped,{config_name_already_in_use,[x1]}}}}, + {?eh,test_stats,{3,0,{1,0}}}, + {?eh,tc_start,{config_static_SUITE,test_default_tclocal}}, + {?eh,tc_done,{config_static_SUITE,test_default_tclocal,ok}}, + {?eh,test_stats,{4,0,{1,0}}}, + {?eh,tc_start,{config_static_SUITE,test_config_name_already_in_use2}}, + {?eh,tc_done, + {config_static_SUITE,test_config_name_already_in_use2, + {skipped,{config_name_already_in_use,[x1,alias]}}}}, + {?eh,test_stats,{4,0,{2,0}}}, + {?eh,tc_start,{config_static_SUITE,test_alias_tclocal}}, + {?eh,tc_done,{config_static_SUITE,test_alias_tclocal,ok}}, + {?eh,test_stats,{5,0,{2,0}}}, + {?eh,tc_start,{config_static_SUITE,test_get_config_undefined}}, + {?eh,tc_done,{config_static_SUITE,test_get_config_undefined,ok}}, + {?eh,test_stats,{6,0,{2,0}}}, + {?eh,tc_start,{config_static_SUITE,end_per_suite}}, + {?eh,tc_done,{config_static_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; expected_events(config_dynamic_SUITE)-> -[ - {?eh,start_logging,{'DEF','RUNDIR'}}, - {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, - {?eh,start_info,{1,1,5}}, - {?eh,tc_start,{config_dynamic_SUITE,init_per_suite}}, - {?eh,tc_done,{config_dynamic_SUITE,init_per_suite,ok}}, - {?eh,tc_start,{config_dynamic_SUITE,test_get_known_variable}}, - {?eh,tc_done, - {config_dynamic_SUITE,test_get_known_variable,ok}}, - {?eh,test_stats,{1,0,{0,0}}}, - {?eh,tc_start,{config_dynamic_SUITE,test_localtime_update}}, - {?eh,tc_done,{config_dynamic_SUITE,test_localtime_update,ok}}, - {?eh,test_stats,{2,0,{0,0}}}, - {?eh,tc_start,{config_dynamic_SUITE,test_server_pid}}, - {?eh,tc_done,{config_dynamic_SUITE,test_server_pid,ok}}, - {?eh,test_stats,{3,0,{0,0}}}, - {?eh,tc_start, - {config_dynamic_SUITE,test_disappearable_variable}}, - {?eh,tc_done, - {config_dynamic_SUITE,test_disappearable_variable,ok}}, - {?eh,test_stats,{4,0,{0,0}}}, - {?eh,tc_start, - {config_dynamic_SUITE,test_disappearable_variable_alias}}, - {?eh,tc_done, - {config_dynamic_SUITE,test_disappearable_variable_alias,ok}}, - {?eh,test_stats,{5,0,{0,0}}}, - {?eh,tc_start,{config_dynamic_SUITE,end_per_suite}}, - {?eh,tc_done,{config_dynamic_SUITE,end_per_suite,ok}}, - {?eh,test_done,{'DEF','STOP_TIME'}}, - {?eh,stop_logging,[]} -]. + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,5}}, + {?eh,tc_start,{config_dynamic_SUITE,init_per_suite}}, + {?eh,tc_done,{config_dynamic_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{config_dynamic_SUITE,test_get_known_variable}}, + {?eh,tc_done, + {config_dynamic_SUITE,test_get_known_variable,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{config_dynamic_SUITE,test_localtime_update}}, + {?eh,tc_done,{config_dynamic_SUITE,test_localtime_update,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_start,{config_dynamic_SUITE,test_server_pid}}, + {?eh,tc_done,{config_dynamic_SUITE,test_server_pid,ok}}, + {?eh,test_stats,{3,0,{0,0}}}, + {?eh,tc_start, + {config_dynamic_SUITE,test_disappearable_variable}}, + {?eh,tc_done, + {config_dynamic_SUITE,test_disappearable_variable,ok}}, + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,tc_start, + {config_dynamic_SUITE,test_disappearable_variable_alias}}, + {?eh,tc_done, + {config_dynamic_SUITE,test_disappearable_variable_alias,ok}}, + {?eh,test_stats,{5,0,{0,0}}}, + {?eh,tc_start,{config_dynamic_SUITE,end_per_suite}}, + {?eh,tc_done,{config_dynamic_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]. diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index 2fa031b884..17faf5ee3f 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -29,7 +29,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct_event.hrl"). -define(eh, ct_test_support_eh). @@ -56,19 +56,22 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all(doc) -> - [""]; +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [cfg_error, lib_error, no_compile, timetrap_end_conf, + timetrap_normal, timetrap_extended]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + -all(suite) -> - [ - cfg_error, - lib_error, - no_compile, - timetrap_end_conf, - timetrap_normal, - timetrap_extended - ]. - %%-------------------------------------------------------------------- %% TEST CASES @@ -122,7 +125,7 @@ lib_error(Config) when is_list(Config) -> TestEvents = events_to_check(lib_error), ok = ct_test_support:verify_events(TestEvents, Events, Config). - + %%%----------------------------------------------------------------- %%% @@ -140,7 +143,7 @@ no_compile(Config) when is_list(Config) -> TestEvents = events_to_check(no_compile), ok = ct_test_support:verify_events(TestEvents, Events, Config). - + %%%----------------------------------------------------------------- %%% timetrap_end_conf(Config) when is_list(Config) -> @@ -214,8 +217,8 @@ setup(Test, Config) -> reformat(Events, EH) -> ct_test_support:reformat(Events, EH). -%reformat(Events, _EH) -> -% Events. + %reformat(Events, _EH) -> + % Events. %%%----------------------------------------------------------------- %%% TEST EVENTS @@ -248,7 +251,7 @@ test_events(cfg_error) -> {?eh,test_stats,{0,0,{0,2}}}, {?eh,tc_auto_skip, {cfg_error_1_SUITE,end_per_suite,{failed,{cfg_error_1_SUITE,init_per_suite, - {'EXIT',init_per_suite_fails}}}}}, + {'EXIT',init_per_suite_fails}}}}}, {?eh,tc_start,{cfg_error_2_SUITE,init_per_suite}}, {?eh,tc_done, @@ -378,12 +381,12 @@ test_events(cfg_error) -> {?eh,tc_auto_skip, {cfg_error_8_SUITE,tc1, {failed,{cfg_error_8_SUITE,init_per_group, - {'EXIT',{init_per_group_fails,g1}}}}}}, + {'EXIT',{init_per_group_fails,g1}}}}}}, {?eh,test_stats,{4,0,{0,11}}}, {?eh,tc_auto_skip, {cfg_error_8_SUITE,end_per_group, {failed,{cfg_error_8_SUITE,init_per_group, - {'EXIT',{init_per_group_fails,g1}}}}}}], + {'EXIT',{init_per_group_fails,g1}}}}}}], [{?eh,tc_start,{cfg_error_8_SUITE,{init_per_group,g2,[]}}}, {?eh,tc_done,{cfg_error_8_SUITE, @@ -396,7 +399,7 @@ test_events(cfg_error) -> {?eh,tc_auto_skip,{cfg_error_8_SUITE,end_per_group, {failed,{cfg_error_8_SUITE,init_per_group, {timetrap_timeout,2000}}}}}], - + [{?eh,tc_start,{cfg_error_8_SUITE,{init_per_group,g3,[]}}}, {?eh,tc_done, {cfg_error_8_SUITE,{init_per_group,g3,[]}, @@ -436,7 +439,7 @@ test_events(cfg_error) -> {?eh,test_stats,{5,0,{0,13}}}, {?eh,tc_start,{cfg_error_8_SUITE,{end_per_group,g4,[]}}}, {?eh,tc_done,{cfg_error_8_SUITE,{end_per_group,g4,[]},ok}}], - + [{?eh,tc_start,{cfg_error_8_SUITE,{init_per_group,g5,[]}}}, {?eh,tc_done,{cfg_error_8_SUITE,{init_per_group,g5,[]},ok}}, {?eh,tc_start,{cfg_error_8_SUITE,tc1}}, diff --git a/lib/common_test/test/ct_event_handler_SUITE.erl b/lib/common_test/test/ct_event_handler_SUITE.erl index 00a4c4ded3..b27770881d 100644 --- a/lib/common_test/test/ct_event_handler_SUITE.erl +++ b/lib/common_test/test/ct_event_handler_SUITE.erl @@ -28,7 +28,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). %-include_lib("common_test/include/ct_event.hrl"). @@ -56,12 +56,21 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all(doc) -> - []; +suite() -> [{ct_hooks,[ts_install_cth]}]. -all(suite) -> +all() -> [start_stop, results]. +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + %%-------------------------------------------------------------------- %% TEST CASES diff --git a/lib/common_test/test/ct_groups_test_1_SUITE.erl b/lib/common_test/test/ct_groups_test_1_SUITE.erl index 64d61fc104..3712bc0e33 100644 --- a/lib/common_test/test/ct_groups_test_1_SUITE.erl +++ b/lib/common_test/test/ct_groups_test_1_SUITE.erl @@ -29,7 +29,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct_event.hrl"). -define(eh, ct_test_support_eh). @@ -56,12 +56,21 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all(doc) -> - ["Run smoke tests of Common Test."]; +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [groups_suite_1, groups_suite_2, groups_suites_1, + groups_dir_1, groups_dirs_1]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> - [groups_suite_1, groups_suite_2, - groups_suites_1, groups_dir_1, groups_dirs_1]. %%-------------------------------------------------------------------- %% TEST CASES diff --git a/lib/common_test/test/ct_groups_test_2_SUITE.erl b/lib/common_test/test/ct_groups_test_2_SUITE.erl index c4371501b3..32e8d0c6d7 100644 --- a/lib/common_test/test/ct_groups_test_2_SUITE.erl +++ b/lib/common_test/test/ct_groups_test_2_SUITE.erl @@ -29,7 +29,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct_event.hrl"). -define(eh, ct_test_support_eh). @@ -56,12 +56,21 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all(doc) -> - ["Run smoke tests of Common Test."]; +suite() -> [{ct_hooks,[ts_install_cth]}]. -all(suite) -> +all() -> [missing_conf, repeat_1]. +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + %%-------------------------------------------------------------------- %% TEST CASES %%-------------------------------------------------------------------- diff --git a/lib/common_test/test/ct_master_SUITE.erl b/lib/common_test/test/ct_master_SUITE.erl index 5ac2866227..e208397296 100644 --- a/lib/common_test/test/ct_master_SUITE.erl +++ b/lib/common_test/test/ct_master_SUITE.erl @@ -28,7 +28,7 @@ -module(ct_master_SUITE). -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct_event.hrl"). -define(eh, ct_test_support_eh). @@ -81,15 +81,19 @@ end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all() -> - all(suite). -all(doc) -> - [""]; +suite() -> [{ct_hooks,[ts_install_cth]}]. -all(suite) -> - [ - ct_master_test - ]. +all() -> + [ct_master_test]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. %%-------------------------------------------------------------------- %% TEST CASES diff --git a/lib/common_test/test/ct_misc_1_SUITE.erl b/lib/common_test/test/ct_misc_1_SUITE.erl index 8c8b2d0d41..f5904ca180 100644 --- a/lib/common_test/test/ct_misc_1_SUITE.erl +++ b/lib/common_test/test/ct_misc_1_SUITE.erl @@ -29,7 +29,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("test_server/include/test_server_line.hrl"). -include_lib("common_test/include/ct_event.hrl"). @@ -57,13 +57,23 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all(doc) -> - [""]; +suite() -> [{ct_hooks,[ts_install_cth]}]. -all(suite) -> - [ - beam_me_up, parse_table - ]. +all() -> + [beam_me_up, {group,parse_table}]. + +groups() -> + [{parse_table,[parallel], + [parse_table_empty, parse_table_single, + parse_table_multiline_row, + parse_table_one_column_multiline, + parse_table_one_column_simple]}]. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. %%-------------------------------------------------------------------- %% TEST CASES @@ -106,13 +116,6 @@ beam_me_up(Config) when is_list(Config) -> TestEvents = events_to_check(beam_me_up, 1), ok = ct_test_support:verify_events(TestEvents, Events, Config). - -parse_table(suite) -> - [parse_table_empty, parse_table_single, - parse_table_multiline_row, - parse_table_one_column_multiline, - parse_table_one_column_simple]. - parse_table_empty(Config) when is_list(Config) -> String = ["+----+-------+---------+---------+----------+------+--------+", diff --git a/lib/common_test/test/ct_repeat_1_SUITE.erl b/lib/common_test/test/ct_repeat_1_SUITE.erl index 1b4cafc9d3..40ef3e42fb 100644 --- a/lib/common_test/test/ct_repeat_1_SUITE.erl +++ b/lib/common_test/test/ct_repeat_1_SUITE.erl @@ -29,7 +29,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct_event.hrl"). -define(eh, ct_test_support_eh). @@ -56,24 +56,26 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all(doc) -> - []; +suite() -> [{ct_hooks,[ts_install_cth]}]. -all(suite) -> - [repeat_cs, - repeat_cs_and_grs, - repeat_seq, - repeat_cs_until_any_ok, - repeat_gr_until_any_ok, - repeat_cs_until_any_fail, - repeat_gr_until_any_fail, - repeat_cs_until_all_ok, - repeat_gr_until_all_ok, - repeat_cs_until_all_fail, - repeat_gr_until_all_fail, +all() -> + [repeat_cs, repeat_cs_and_grs, repeat_seq, + repeat_cs_until_any_ok, repeat_gr_until_any_ok, + repeat_cs_until_any_fail, repeat_gr_until_any_fail, + repeat_cs_until_all_ok, repeat_gr_until_all_ok, + repeat_cs_until_all_fail, repeat_gr_until_all_fail, repeat_seq_until_any_fail, - repeat_shuffled_seq_until_any_fail - ]. + repeat_shuffled_seq_until_any_fail]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + %%-------------------------------------------------------------------- %% TEST CASES diff --git a/lib/common_test/test/ct_sequence_1_SUITE.erl b/lib/common_test/test/ct_sequence_1_SUITE.erl index 0cf40f106a..0876a6f8b8 100644 --- a/lib/common_test/test/ct_sequence_1_SUITE.erl +++ b/lib/common_test/test/ct_sequence_1_SUITE.erl @@ -29,7 +29,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct_event.hrl"). -define(eh, ct_test_support_eh). @@ -56,13 +56,24 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all(suite) -> - [subgroup_return_fail, - subgroup_init_fail, +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [subgroup_return_fail, subgroup_init_fail, subgroup_after_failed_case, case_after_subgroup_return_fail, case_after_subgroup_fail_init]. +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + %%-------------------------------------------------------------------- %% TEST CASES %%-------------------------------------------------------------------- diff --git a/lib/common_test/test/ct_skip_SUITE.erl b/lib/common_test/test/ct_skip_SUITE.erl index 2e02061dec..2b64062e4d 100644 --- a/lib/common_test/test/ct_skip_SUITE.erl +++ b/lib/common_test/test/ct_skip_SUITE.erl @@ -29,7 +29,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct_event.hrl"). -define(eh, ct_test_support_eh). @@ -56,14 +56,20 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all(doc) -> - [""]; +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [auto_skip, user_skip]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> - [ - auto_skip, - user_skip - ]. %%-------------------------------------------------------------------- diff --git a/lib/common_test/test/ct_smoke_test_SUITE.erl b/lib/common_test/test/ct_smoke_test_SUITE.erl index 05a2c20695..096171f951 100644 --- a/lib/common_test/test/ct_smoke_test_SUITE.erl +++ b/lib/common_test/test/ct_smoke_test_SUITE.erl @@ -29,7 +29,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct_event.hrl"). -define(eh, ct_test_support_eh). @@ -112,14 +112,22 @@ end_per_testcase(TestCase, Config) -> %% Description: Returns a description of the test suite (doc) and a %% list of all test cases in the suite (suite). %%-------------------------------------------------------------------- -all(doc) -> - ["Run smoke tests of Common Test."]; +suite() -> [{ct_hooks,[ts_install_cth]}]. -all(suite) -> - [dir1, dir2, dir1_2, - suite11, suite21, suite11_21, +all() -> + [dir1, dir2, dir1_2, suite11, suite21, suite11_21, tc111, tc211, tc111_112]. +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + %%-------------------------------------------------------------------- %% TEST CASES diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE.erl index eb85409073..44c30d7a38 100644 --- a/lib/common_test/test/ct_test_server_if_1_SUITE.erl +++ b/lib/common_test/test/ct_test_server_if_1_SUITE.erl @@ -29,7 +29,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct_event.hrl"). -define(eh, ct_test_support_eh). @@ -56,13 +56,20 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all(doc) -> - [""]; +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [ts_if_1]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> - [ - ts_if_1 - ]. %%-------------------------------------------------------------------- diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl index dc399bfb4c..e080e074bf 100644 --- a/lib/common_test/test/ct_testspec_1_SUITE.erl +++ b/lib/common_test/test/ct_testspec_1_SUITE.erl @@ -29,7 +29,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct_event.hrl"). -define(eh, ct_test_support_eh). @@ -56,24 +56,41 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all(doc) -> - ["Run smoke tests of Common Test."]; - -all(suite) -> - [all_suites, skip_all_suites, - suite, skip_suite, - all_testcases, skip_all_testcases, - testcase, skip_testcase, - all_groups, skip_all_groups, - group, skip_group, - group_all_testcases, skip_group_all_testcases, - group_testcase, skip_group_testcase, - topgroup, - subgroup, skip_subgroup, +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [all_suites, skip_all_suites, suite, skip_suite, + all_testcases, skip_all_testcases, testcase, + skip_testcase, all_groups, skip_all_groups, group, + skip_group, group_all_testcases, + skip_group_all_testcases, group_testcase, + skip_group_testcase, topgroup, subgroup, skip_subgroup, subgroup_all_testcases, skip_subgroup_all_testcases, subgroup_testcase, skip_subgroup_testcase, - sub_skipped_by_top, - testcase_in_multiple_groups]. + sub_skipped_by_top, testcase_in_multiple_groups, + order_of_tests_in_multiple_dirs_no_merge_tests, + order_of_tests_in_multiple_suites_no_merge_tests, + order_of_suites_in_multiple_dirs_no_merge_tests, + order_of_groups_in_multiple_dirs_no_merge_tests, + order_of_groups_in_multiple_suites_no_merge_tests, + order_of_tests_in_multiple_dirs, + order_of_tests_in_multiple_suites, + order_of_suites_in_multiple_dirs, + order_of_groups_in_multiple_dirs, + order_of_groups_in_multiple_suites, + order_of_tests_in_multiple_suites_with_skip_no_merge_tests, + order_of_tests_in_multiple_suites_with_skip, + all_plus_one_tc_no_merge_tests, + all_plus_one_tc]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. %%-------------------------------------------------------------------- %% TEST CASES @@ -366,6 +383,223 @@ testcase_in_multiple_groups(Config) when is_list(Config) -> setup_and_execute(testcase_in_multiple_groups, TestSpec, Config). %%%----------------------------------------------------------------- +%%% + +order_of_tests_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestDir2 = filename:join(DataDir, "groups_2"), + TestSpec = [{merge_tests, false}, + {cases,TestDir1,groups_12_SUITE,[testcase_1a]}, + {cases,TestDir2,groups_22_SUITE,[testcase_1]}, + {cases,TestDir1,groups_12_SUITE,[testcase_1b]}], + + setup_and_execute(order_of_tests_in_multiple_dirs_no_merge_tests, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_tests_in_multiple_suites_no_merge_tests(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{merge_tests, false}, + {cases,TestDir1,groups_12_SUITE,[testcase_1a]}, + {cases,TestDir1,groups_11_SUITE,[testcase_1]}, + {cases,TestDir1,groups_12_SUITE,[testcase_1b]}], + + setup_and_execute(order_of_tests_in_multiple_suites_no_merge_tests, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_suites_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestDir2 = filename:join(DataDir, "groups_2"), + TestSpec = [{merge_tests, false}, + {suites,TestDir1,groups_12_SUITE}, + {suites,TestDir2,groups_22_SUITE}, + {suites,TestDir1,groups_11_SUITE}], + + setup_and_execute(order_of_suites_in_multiple_dirs_no_merge_tests, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_groups_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestDir2 = filename:join(DataDir, "groups_2"), + TestSpec = [{merge_tests, false}, + {groups,TestDir1,groups_12_SUITE,test_group_1a}, + {groups,TestDir2,groups_22_SUITE,test_group_1a}, + {groups,TestDir1,groups_12_SUITE,test_group_1b}], + + setup_and_execute(order_of_groups_in_multiple_dirs_no_merge_tests, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_groups_in_multiple_suites_no_merge_tests(Config) + when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{merge_tests, false}, + {groups,TestDir1,groups_12_SUITE,test_group_1a}, + {groups,TestDir1,groups_11_SUITE,test_group_1a}, + {groups,TestDir1,groups_12_SUITE,test_group_1b}], + + setup_and_execute(order_of_groups_in_multiple_suites_no_merge_tests, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_tests_in_multiple_suites_with_skip_no_merge_tests(Config) + when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{merge_tests, false}, + {cases,TestDir1,groups_12_SUITE,[testcase_1a]}, + {cases,TestDir1,groups_11_SUITE,[testcase_1]}, + {cases,TestDir1,groups_12_SUITE,[testcase_1b]}, + {cases,TestDir1,groups_11_SUITE,[testcase_2]}, + {skip_cases,TestDir1,groups_12_SUITE,[testcase_1b],"Skip it"}], + + setup_and_execute( + order_of_tests_in_multiple_suites_with_skip_no_merge_tests, + TestSpec, Config). + + +%%%----------------------------------------------------------------- +%%% + +order_of_tests_in_multiple_dirs(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestDir2 = filename:join(DataDir, "groups_2"), + TestSpec = [{cases,TestDir1,groups_12_SUITE,[testcase_1a]}, + {cases,TestDir2,groups_22_SUITE,[testcase_1]}, + {cases,TestDir1,groups_12_SUITE,[testcase_1b]}], + + setup_and_execute(order_of_tests_in_multiple_dirs, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_tests_in_multiple_suites(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{cases,TestDir1,groups_12_SUITE,[testcase_1a]}, + {cases,TestDir1,groups_11_SUITE,[testcase_1]}, + {cases,TestDir1,groups_12_SUITE,[testcase_1b]}], + + setup_and_execute(order_of_tests_in_multiple_suites, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_suites_in_multiple_dirs(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestDir2 = filename:join(DataDir, "groups_2"), + TestSpec = [{suites,TestDir1,groups_12_SUITE}, + {suites,TestDir2,groups_22_SUITE}, + {suites,TestDir1,groups_11_SUITE}], + + setup_and_execute(order_of_suites_in_multiple_dirs, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_groups_in_multiple_dirs(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestDir2 = filename:join(DataDir, "groups_2"), + TestSpec = [{groups,TestDir1,groups_12_SUITE,test_group_1a}, + {groups,TestDir2,groups_22_SUITE,test_group_1a}, + {groups,TestDir1,groups_12_SUITE,test_group_1b}], + + setup_and_execute(order_of_groups_in_multiple_dirs, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_groups_in_multiple_suites(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir1,groups_12_SUITE,test_group_1a}, + {groups,TestDir1,groups_11_SUITE,test_group_1a}, + {groups,TestDir1,groups_12_SUITE,test_group_1b}], + + setup_and_execute(order_of_groups_in_multiple_suites, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_tests_in_multiple_suites_with_skip(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{cases,TestDir1,groups_12_SUITE,[testcase_1a]}, + {cases,TestDir1,groups_11_SUITE,[testcase_1]}, + {cases,TestDir1,groups_12_SUITE,[testcase_1b]}, + {cases,TestDir1,groups_11_SUITE,[testcase_2]}, + {skip_cases,TestDir1,groups_12_SUITE,[testcase_1b],"Skip it!"}], + + setup_and_execute(order_of_tests_in_multiple_suites_with_skip, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +all_plus_one_tc_no_merge_tests(Config) when is_list(Config) -> + + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{merge_tests,false}, + {suites,TestDir1,groups_12_SUITE}, + {cases,TestDir1,groups_12_SUITE,[testcase_1a]}], + + setup_and_execute(all_plus_one_tc_no_merge_tests, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +all_plus_one_tc(Config) when is_list(Config) -> + + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{suites,TestDir1,groups_12_SUITE}, + {cases,TestDir1,groups_12_SUITE,[testcase_1a]}], + + setup_and_execute(all_plus_one_tc, + TestSpec, Config). + +%%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- @@ -428,6 +662,719 @@ events_to_check(_, 0) -> events_to_check(Test, N) -> test_events(Test) ++ events_to_check(Test, N-1). +test_events(all_suites) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{simple_1_SUITE,init_per_suite}}, + {?eh,tc_done,{simple_1_SUITE,end_per_suite,'_'}}, + {?eh,tc_start,{simple_2_SUITE,init_per_suite}}, + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,tc_done,{simple_2_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_all_suites) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_user_skip,{simple_1_SUITE,all,"SKIPPED!"}}, + {?eh,tc_user_skip,{simple_2_SUITE,all,"SKIPPED!"}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(suite) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{simple_1_SUITE,init_per_suite}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_done,{simple_1_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_suite) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_user_skip,{simple_1_SUITE,all,"SKIPPED!"}}, + {?eh,tc_done,{simple_2_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(all_testcases) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{simple_1_SUITE,init_per_suite}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_done,{simple_1_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_all_testcases) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_user_skip,{simple_1_SUITE,all,"SKIPPED!"}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(testcase) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{simple_1_SUITE,init_per_suite}}, + {?eh,test_stats,{1,0,{0,0}}}, + {negative,{?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_done,{simple_1_SUITE,end_per_suite,'_'}}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_testcase) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{simple_1_SUITE,init_per_suite}}, + {?eh,tc_user_skip,{simple_1_SUITE,tc1,"SKIPPED!"}}, + {?eh,tc_start,{simple_1_SUITE,tc2}}, + {?eh,tc_start,{simple_1_SUITE,end_per_suite}}, + + {?eh,tc_start,{simple_2_SUITE,init_per_suite}}, + {?eh,tc_user_skip,{simple_2_SUITE,tc2,"SKIPPED!"}}, + {?eh,tc_start,{simple_2_SUITE,tc1}}, + {?eh,test_stats,{2,0,{2,0}}}, + {?eh,tc_start,{simple_2_SUITE,end_per_suite}}, + + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(all_groups) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,test_stats,{12,0,{0,0}}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_all_groups) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1a},"SKIPPED!"}}, + {?eh,test_stats,{0,0,{1,0}}}, + {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}}, + {?eh,test_stats,{0,0,{2,0}}}, + {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_2},"SKIPPED!"}}, + {?eh,test_stats,{0,0,{3,0}}}, + {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_4},"SKIPPED!"}}, + {?eh,test_stats,{0,0,{4,0}}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(group) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1a}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1b}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_group) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1a}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1b}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}}, + + {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}}, + {?eh,test_stats,{2,0,{1,0}}}, + {negative,{?eh,tc_user_skip,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(group_all_testcases) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1a}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1b}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_group_all_testcases) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1a},"SKIPPED!"}}, + {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}}, + {?eh,test_stats,{0,0,{2,0}}}, + {?eh,tc_start,{groups_11_SUITE,end_per_suite}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(group_testcase) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1a}}, + {?eh,test_stats,{1,0,{0,0}}}, + {negative,{?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}}}, + + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_group_testcase) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1a}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}}, + {?eh,test_stats,{1,0,{1,0}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}}, + + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1b,[]}}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1b}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}}, + {?eh,test_stats,{2,0,{2,0}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1b,[]},'_'}}, + + {negative,{?eh,tc_user_skip,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(topgroup) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]},ok}}, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}} + ], + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[]}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[]}}} + ], + {?eh,test_stats,{6,0,{0,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]},ok}}]}, + + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}}, + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}], + {shuffle, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_8, + [{shuffle,'_'},sequence]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_8, + [{shuffle,'_'},sequence]},ok}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_8, + [shuffle,sequence]}}}, + {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_8, + [shuffle,sequence]},ok}} + ]}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}} + ]}, + {?eh,test_stats,{12,0,{0,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}], + + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(subgroup) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]},ok}}, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}} + ], + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[]}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[]}}} + ], + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]},ok}}]}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_subgroup) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}}, + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}], + {?eh,tc_user_skip, + {groups_12_SUITE,{group,test_group_8},"SKIPPED!"}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}} + ]}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}], + + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(subgroup_all_testcases) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}}, + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}], + {shuffle, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_8, + [{shuffle,'_'},sequence]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_8, + [{shuffle,'_'},sequence]},ok}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_8, + [shuffle,sequence]}}}, + {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_8, + [shuffle,sequence]},ok}} + ]}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}} + ]}, + {?eh,test_stats,{6,0,{0,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}], + + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]},ok}}, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}} + ], + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[]}}}, + {?eh,test_stats,{10,0,{0,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[]}}} + ], + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]},ok}}]}, + + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_subgroup_all_testcases) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_4,[]},ok}}, + {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_5},"SKIPPED!"}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}, + {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_4,[]},ok}} + ], + + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(subgroup_testcase) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}}, + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}], + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}} + ]}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}], + + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]},ok}}, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}} + ], + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[]}}}, + {?eh,test_stats,{3,0,{0,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[]}}} + ], + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]},ok}}]}, + + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_subgroup_testcase) -> + [ + + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}}, + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}}, + {?eh,tc_user_skip, {groups_12_SUITE,testcase_7a,"SKIPPED!"}}, + {?eh,test_stats,{1,0,{1,0}}}, + {?eh,tc_user_skip, {groups_12_SUITE,testcase_7b,"SKIPPED!"}}, + {?eh,test_stats,{1,0,{2,0}}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}], + {shuffle, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_8, + [{shuffle,'_'},sequence]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_8, + [{shuffle,'_'},sequence]},ok}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_8, + [shuffle,sequence]}}}, + {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_8, + [shuffle,sequence]},ok}} + ]}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}} + ]}, + {?eh,test_stats,{4,0,{2,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}], + + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + + ]; + +test_events(sub_skipped_by_top) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_4},"SKIPPED!"}}, + + {negative, + {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_4},"SKIPPED!"}}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}}, + + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(testcase_in_multiple_groups) -> + []; + +test_events(order_of_tests_in_multiple_dirs_no_merge_tests) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {?eh,tc_done, {groups_12_SUITE,testcase_1a, + {failed,{error,{test_case_failed,no_group_data}}}}}, + {?eh,tc_start,{groups_22_SUITE,testcase_1}}, + {?eh,tc_done,{groups_22_SUITE,testcase_1,ok}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1b}}, + {?eh,tc_done, {groups_12_SUITE,testcase_1b, + {failed,{error,{test_case_failed,no_group_data}}}}}, + {?eh,stop_logging,[]} + ]; +test_events(order_of_tests_in_multiple_suites_no_merge_tests) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1}}, + {?eh,tc_done,{groups_11_SUITE,testcase_1,ok}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1b}}, + {?eh,tc_done,{groups_12_SUITE,testcase_1b,'_'}}, + {?eh,stop_logging,[]} + ]; +test_events(order_of_suites_in_multiple_dirs_no_merge_tests) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,init_per_suite,'_'}}, + {?eh,tc_start,{groups_12_SUITE,end_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {?eh,tc_start,{groups_22_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_22_SUITE,init_per_suite,'_'}}, + {?eh,tc_start,{groups_22_SUITE,end_per_suite}}, + {?eh,tc_done,{groups_22_SUITE,end_per_suite,'_'}}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_11_SUITE,init_per_suite,'_'}}, + {?eh,tc_start,{groups_11_SUITE,end_per_suite}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {?eh,stop_logging,[]}]; +test_events(order_of_groups_in_multiple_dirs_no_merge_tests) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {?eh,tc_start, {groups_22_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_22_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, + + {?eh,stop_logging,[]}]; +test_events(order_of_groups_in_multiple_suites_no_merge_tests) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {?eh,tc_start, {groups_11_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_11_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, + + {?eh,stop_logging,[]}]; +test_events(order_of_tests_in_multiple_suites_with_skip_no_merge_tests) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1}}, + {?eh,tc_done,{groups_11_SUITE,testcase_1,ok}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_1b,'_'}}, + {?eh,tc_start,{groups_11_SUITE,testcase_2}}, + {?eh,tc_done,{groups_11_SUITE,testcase_2,ok}}, + {?eh,stop_logging,[]} + ]; + +test_events(order_of_tests_in_multiple_dirs) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {?eh,tc_done, + {groups_12_SUITE,testcase_1a, + {failed,{error,{test_case_failed,no_group_data}}}}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1b}}, + {?eh,tc_done, + {groups_12_SUITE,testcase_1b, + {failed,{error,{test_case_failed,no_group_data}}}}}, + {?eh,tc_start,{groups_22_SUITE,testcase_1}}, + {?eh,tc_done,{groups_22_SUITE,testcase_1,ok}}, + {?eh,stop_logging,[]} + ]; +test_events(order_of_tests_in_multiple_suites) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, + + {?eh,tc_start,{groups_12_SUITE,testcase_1b}}, + {?eh,tc_done,{groups_12_SUITE,testcase_1b,'_'}}, + + {?eh,tc_start,{groups_11_SUITE,testcase_1}}, + {?eh,tc_done,{groups_11_SUITE,testcase_1,ok}}, + {?eh,stop_logging,[]} + ]; +test_events(order_of_suites_in_multiple_dirs) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,init_per_suite,'_'}}, + {?eh,tc_start,{groups_12_SUITE,end_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_11_SUITE,init_per_suite,'_'}}, + {?eh,tc_start,{groups_11_SUITE,end_per_suite}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + + {?eh,tc_start,{groups_22_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_22_SUITE,init_per_suite,'_'}}, + {?eh,tc_start,{groups_22_SUITE,end_per_suite}}, + {?eh,tc_done,{groups_22_SUITE,end_per_suite,'_'}}, + {?eh,stop_logging,[]}]; +test_events(order_of_groups_in_multiple_dirs) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, + + {?eh,tc_start, {groups_22_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_22_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {?eh,stop_logging,[]}]; +test_events(order_of_groups_in_multiple_suites) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, + + {?eh,tc_start, {groups_11_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_11_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {?eh,stop_logging,[]}]; + +test_events(order_of_tests_in_multiple_suites_with_skip) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_1b,'_'}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1}}, + {?eh,tc_done,{groups_11_SUITE,testcase_1,ok}}, + {?eh,tc_start,{groups_11_SUITE,testcase_2}}, + {?eh,tc_done,{groups_11_SUITE,testcase_2,ok}}, + {?eh,stop_logging,[]} + ]; + +test_events(all_plus_one_tc_no_merge_tests) -> + + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {?eh,stop_logging,[]} + ]; + +test_events(all_plus_one_tc) -> + + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {?eh,stop_logging,[]}} + ]; + test_events(_) -> [ ]. |