From 093890dc793e85a09b40f1eca878f410c73cf625 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Thu, 14 Nov 2013 17:39:23 +0100 Subject: Modify the auto_skip report for group config funcs to include group name Also correct failing test cases and find and fix remaining bugs. --- lib/common_test/doc/src/ct_hooks.xml | 5 +- lib/common_test/src/ct_framework.erl | 72 ++++--- lib/common_test/src/cth_surefire.erl | 3 + lib/common_test/test/ct_config_info_SUITE.erl | 4 +- lib/common_test/test/ct_error_SUITE.erl | 20 +- lib/common_test/test/ct_group_info_SUITE.erl | 121 ++++++----- lib/common_test/test/ct_repeat_1_SUITE.erl | 26 +-- lib/common_test/test/ct_repeat_testrun_SUITE.erl | 23 +-- lib/common_test/test/ct_sequence_1_SUITE.erl | 4 +- lib/common_test/test/ct_skip_SUITE.erl | 43 ++-- lib/common_test/test/ct_surefire_SUITE.erl | 2 +- lib/common_test/test/ct_test_server_if_1_SUITE.erl | 74 +++---- lib/common_test/test/ct_testspec_1_SUITE.erl | 230 +++++++++++++++------ 13 files changed, 386 insertions(+), 241 deletions(-) (limited to 'lib/common_test') diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml index b3e713c77f..859ff9df14 100644 --- a/lib/common_test/doc/src/ct_hooks.xml +++ b/lib/common_test/doc/src/ct_hooks.xml @@ -485,8 +485,9 @@ NewCTHState Called after the CTH scope ends - TestcaseName = end_per_suite | init_per_group | - end_per_group | atom() + TestcaseName = end_per_suite | {init_per_group,GroupName} | + {end_per_group,GroupName} | atom() + GroupName = atom() Reason = {tc_auto_skip | tc_user_skip, term()} CTHState = NewCTHState = term() diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 2de75f0845..e81b69a1b5 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -69,7 +69,7 @@ init_tc(Mod,Func,Config) -> andalso Func=/=end_per_group andalso ct_util:get_testdata(skip_rest) of true -> - {skip,"Repeated test stopped by force_stop option"}; + {auto_skip,"Repeated test stopped by force_stop option"}; _ -> case ct_util:get_testdata(curr_tc) of {Suite,{suite0_failed,{require,Reason}}} -> @@ -159,25 +159,27 @@ init_tc1(Mod,Suite,Func,[Config0]) when is_list(Config0) -> true -> ct_config:delete_default_config(testcase) end, + Initialize = fun() -> + ct_logs:init_tc(false), + ct_event:notify(#event{name=tc_start, + node=node(), + data={Mod,FuncSpec}}) + end, case add_defaults(Mod,Func,AllGroups) of Error = {suite0_failed,_} -> - ct_logs:init_tc(false), - ct_event:notify(#event{name=tc_start, - node=node(), - data={Mod,FuncSpec}}), + Initialize(), ct_util:set_testdata({curr_tc,{Suite,Error}}), {error,Error}; Error = {group0_failed,_} -> + Initialize(), {auto_skip,Error}; Error = {testcase0_failed,_} -> + Initialize(), {auto_skip,Error}; {SuiteInfo,MergeResult} -> case MergeResult of {error,Reason} -> - ct_logs:init_tc(false), - ct_event:notify(#event{name=tc_start, - node=node(), - data={Mod,FuncSpec}}), + Initialize(), {fail,Reason}; _ -> init_tc2(Mod,Suite,Func,SuiteInfo,MergeResult,Config) @@ -1318,36 +1320,48 @@ report(What,Data) -> add_to_stats(SkipOrFail) end; tc_user_skip -> - %% test case specified as skipped in testspec, or init - %% config func for suite/group has returned {skip,Reason} - %% Data = {Suite,Case,Comment} + %% test case or config function specified as skipped in testspec, + %% or init config func for suite/group has returned {skip,Reason} + %% Data = {Suite,Case,Comment} | + %% {Suite,{GroupConfigFunc,GroupName},Comment} + {Func,Data1} = case Data of + {Suite,{ConfigFunc,undefined},Cmt} -> + {ConfigFunc,{Suite,ConfigFunc,Cmt}}; + {_,{ConfigFunc,_},_} -> {ConfigFunc,Data}; + {_,Case,_} -> {Case,Data} + end, + ct_event:sync_notify(#event{name=tc_user_skip, node=node(), - data=Data}), - case Data of - {_,Func,_} when Func /= init_per_suite, - Func /= init_per_group, - Func /= end_per_suite, - Func /= end_per_group -> - ct_hooks:on_tc_skip(What, Data), + data=Data1}), + ct_hooks:on_tc_skip(What, Data1), + + if Func /= init_per_suite, Func /= init_per_group, + Func /= end_per_suite, Func /= end_per_group -> add_to_stats(user_skipped); - _ -> + true -> ok end; tc_auto_skip -> - %% test case skipped because of error in init_per_suite - %% Data = {Suite,Case,Comment} - - {_Suite,Case,_Result} = Data, - + %% test case skipped because of error in config function, or + %% config function skipped because of error in info function + %% Data = {Suite,Case,Comment} | + %% {Suite,{GroupConfigFunc,GroupName},Comment} + {Func,Data1} = case Data of + {Suite,{ConfigFunc,undefined},Cmt} -> + {ConfigFunc,{Suite,ConfigFunc,Cmt}}; + {_,{ConfigFunc,_},_} -> {ConfigFunc,Data}; + {_,Case,_} -> {Case,Data} + end, %% this test case does not have a log, so printouts %% from event handlers should end up in the main log ct_event:sync_notify(#event{name=tc_auto_skip, node=node(), - data=Data}), - ct_hooks:on_tc_skip(What, Data), - if Case /= end_per_suite, - Case /= end_per_group -> + data=Data1}), + ct_hooks:on_tc_skip(What, Data1), + + if Func /= end_per_suite, + Func /= end_per_group -> add_to_stats(auto_skipped); true -> ok diff --git a/lib/common_test/src/cth_surefire.erl b/lib/common_test/src/cth_surefire.erl index bbbca3828e..7ed2018bdf 100644 --- a/lib/common_test/src/cth_surefire.erl +++ b/lib/common_test/src/cth_surefire.erl @@ -138,6 +138,9 @@ on_tc_fail(_TC, Res, State) -> {fail,lists:flatten(io_lib:format("~p",[Res]))} }, State#state{ test_cases = [NewTC | tl(TCs)]}. +on_tc_skip({ConfigFunc,_GrName},{Type,_Reason} = Res, State0) + when Type == tc_auto_skip; Type == tc_user_skip -> + on_tc_skip(ConfigFunc, Res, State0); on_tc_skip(Tc,{Type,_Reason} = Res, State0) when Type == tc_auto_skip -> TcStr = atom_to_list(Tc), State = diff --git a/lib/common_test/test/ct_config_info_SUITE.erl b/lib/common_test/test/ct_config_info_SUITE.erl index d6a49cb3bc..8f2f0eb75f 100644 --- a/lib/common_test/test/ct_config_info_SUITE.erl +++ b/lib/common_test/test/ct_config_info_SUITE.erl @@ -127,7 +127,7 @@ test_events(config_info) -> {failed,{timetrap_timeout,350}}}}, {?eh,tc_auto_skip,{config_info_1_SUITE,t11, {failed,{config_info_1_SUITE,init_per_group,{timetrap_timeout,350}}}}}, - {?eh,tc_auto_skip,{config_info_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{config_info_1_SUITE,{end_per_group,g1}, {failed,{config_info_1_SUITE,init_per_group, {timetrap_timeout,350}}}}}], @@ -145,7 +145,7 @@ test_events(config_info) -> {?eh,tc_auto_skip,{config_info_1_SUITE,t41, {failed,{config_info_1_SUITE,init_per_group, {timetrap_timeout,400}}}}}, - {?eh,tc_auto_skip,{config_info_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{config_info_1_SUITE,{end_per_group,g4}, {failed,{config_info_1_SUITE,init_per_group, {timetrap_timeout,400}}}}}], {?eh,tc_start,{config_info_1_SUITE,t31}}, diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index 775a8ffdf7..28f0494d20 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -465,7 +465,7 @@ test_events(cfg_error) -> {'EXIT',{init_per_group_fails,g1}}}}}}, {?eh,test_stats,{4,0,{0,11}}}, {?eh,tc_auto_skip, - {cfg_error_8_SUITE,end_per_group, + {cfg_error_8_SUITE,{end_per_group,g1}, {failed,{cfg_error_8_SUITE,init_per_group, {'EXIT',{init_per_group_fails,g1}}}}}}], @@ -476,7 +476,7 @@ test_events(cfg_error) -> {failed,{cfg_error_8_SUITE,init_per_group, {timetrap_timeout,2000}}}}}, {?eh,test_stats,{4,0,{0,12}}}, - {?eh,tc_auto_skip,{cfg_error_8_SUITE,end_per_group, + {?eh,tc_auto_skip,{cfg_error_8_SUITE,{end_per_group,g2}, {failed,{cfg_error_8_SUITE,init_per_group, {timetrap_timeout,2000}}}}}], @@ -490,7 +490,7 @@ test_events(cfg_error) -> {'EXIT',{{badmatch,42},'_'}}}}}}, {?eh,test_stats,{4,0,{0,13}}}, {?eh,tc_auto_skip, - {cfg_error_8_SUITE,end_per_group, + {cfg_error_8_SUITE,{end_per_group,g3}, {failed,{cfg_error_8_SUITE,init_per_group, {'EXIT',{{badmatch,42},'_'}}}}}}], @@ -516,7 +516,7 @@ test_events(cfg_error) -> {'EXIT',{sub_group_failed,g6}}}}}}, {?eh,test_stats,{6,0,{0,14}}}, {?eh,tc_auto_skip, - {cfg_error_8_SUITE,end_per_group, + {cfg_error_8_SUITE,{end_per_group,g6}, {failed,{cfg_error_8_SUITE,init_per_group, {'EXIT',{sub_group_failed,g6}}}}}}], {?eh,tc_start,{cfg_error_8_SUITE,tc3}}, @@ -1102,7 +1102,7 @@ test_events(timetrap_fun_group) -> {failed,{timetrap_8_SUITE,init_per_group, {user_timetrap_error,{kaboom,'_'}}}}}}, {?eh,test_stats,{0,11,{0,2}}}, - {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,g4}, {failed,{timetrap_8_SUITE,init_per_group, {user_timetrap_error,{kaboom,'_'}}}}}}], @@ -1117,7 +1117,7 @@ test_events(timetrap_fun_group) -> {failed,{timetrap_8_SUITE,init_per_group, {user_timetrap_error,{kaboom,'_'}}}}}}, {?eh,test_stats,{0,11,{0,4}}}, - {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,g5}, {failed,{timetrap_8_SUITE,init_per_group, {user_timetrap_error,{kaboom,'_'}}}}}}], @@ -1132,7 +1132,7 @@ test_events(timetrap_fun_group) -> {failed,{timetrap_8_SUITE,init_per_group, {timetrap_timeout,'_'}}}}}, {?eh,test_stats,{0,11,{0,6}}}, - {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,g6}, {failed,{timetrap_8_SUITE,init_per_group, {timetrap_timeout,'_'}}}}}], @@ -1285,7 +1285,7 @@ test_events(timetrap_fun_group) -> {failed,{timetrap_8_SUITE,init_per_group, {user_timetrap_error,{kaboom,'_'}}}}}}, {?eh,test_stats,{4,26,{0,8}}}, - {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,pg4}, {failed,{timetrap_8_SUITE,init_per_group, {user_timetrap_error,{kaboom,'_'}}}}}}]}, @@ -1301,7 +1301,7 @@ test_events(timetrap_fun_group) -> {failed,{timetrap_8_SUITE,init_per_group, {user_timetrap_error,{kaboom,'_'}}}}}}, {?eh,test_stats,{4,26,{0,10}}}, - {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,pg5}, {failed,{timetrap_8_SUITE,init_per_group, {user_timetrap_error,{kaboom,'_'}}}}}}]}, @@ -1317,7 +1317,7 @@ test_events(timetrap_fun_group) -> {failed,{timetrap_8_SUITE,init_per_group, {timetrap_timeout,'_'}}}}}, {?eh,test_stats,{4,26,{0,12}}}, - {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,pg6}, {failed,{timetrap_8_SUITE,init_per_group, {timetrap_timeout,'_'}}}}}]}, diff --git a/lib/common_test/test/ct_group_info_SUITE.erl b/lib/common_test/test/ct_group_info_SUITE.erl index c715142ff5..e7bc5baaa1 100644 --- a/lib/common_test/test/ct_group_info_SUITE.erl +++ b/lib/common_test/test/ct_group_info_SUITE.erl @@ -269,13 +269,19 @@ test_events(timetrap_all) -> {?eh,tc_done,{group_timetrap_1_SUITE,{end_per_group,g10,[]},ok}}], [{?eh,tc_start,{group_timetrap_1_SUITE,{init_per_group,g11,[]}}}, - {?eh,tc_done,{group_timetrap_1_SUITE,{init_per_group,g11,[]},ok}}, - {?eh,tc_done,{group_timetrap_1_SUITE,t111,{failed,{timetrap_timeout,1000}}}}, - {?eh,test_stats,{0,14,{0,0}}}, - {?eh,tc_start,{group_timetrap_1_SUITE,{end_per_group,g11,[]}}}, - {?eh,tc_done,{group_timetrap_1_SUITE,{end_per_group,g11,[]},ok}}], - + {?eh,tc_done,{group_timetrap_1_SUITE, + {init_per_group,g11,[]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,tc_auto_skip, + {group_timetrap_1_SUITE,t111,{group0_failed,bad_return_value}}}, + {?eh,test_stats,{0,13,{0,1}}}, + {?eh,tc_auto_skip,{group_timetrap_1_SUITE, + {end_per_group,g11}, + {group0_failed,bad_return_value}}}], + + {?eh,tc_start,{group_timetrap_1_SUITE,end_per_suite}}, {?eh,tc_done,{group_timetrap_1_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} ]; @@ -422,13 +428,15 @@ test_events(timetrap_all_no_ips) -> {?eh,tc_done,{group_timetrap_2_SUITE,{end_per_group,g10,[]},ok}}], [{?eh,tc_start,{group_timetrap_2_SUITE,{init_per_group,g11,[]}}}, - {?eh,tc_done,{group_timetrap_2_SUITE,{init_per_group,g11,[]},ok}}, - {?eh,tc_done,{group_timetrap_2_SUITE,t111,{failed,{timetrap_timeout,1000}}}}, - {?eh,test_stats,{0,14,{0,0}}}, - {?eh,tc_start,{group_timetrap_2_SUITE,{end_per_group,g11,[]}}}, - {?eh,tc_done,{group_timetrap_2_SUITE,{end_per_group,g11,[]},ok}}], - - {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,tc_done,{group_timetrap_2_SUITE, + {init_per_group,g11,[]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,tc_auto_skip,{group_timetrap_2_SUITE,t111, + {group0_failed,bad_return_value}}}, + {?eh,test_stats,{0,13,{0,1}}}, + {?eh,tc_auto_skip,{group_timetrap_2_SUITE, + {end_per_group,g11}, + {group0_failed,bad_return_value}}}], {?eh,stop_logging,[]} ]; @@ -501,11 +509,13 @@ test_events(timetrap_all_no_ipg) -> {?eh,tc_done,{ct_framework,{end_per_group,g10,[{suite,group_timetrap_3_SUITE}]},ok}}], [{?eh,tc_start,{ct_framework,{init_per_group,g11,[{suite,group_timetrap_3_SUITE}]}}}, - {?eh,tc_done,{ct_framework,{init_per_group,g11,[{suite,group_timetrap_3_SUITE}]},ok}}, - {?eh,tc_done,{group_timetrap_3_SUITE,t111,{failed,{timetrap_timeout,1000}}}}, - {?eh,test_stats,{0,14,{0,0}}}, - {?eh,tc_start,{ct_framework,{end_per_group,g11,[{suite,group_timetrap_3_SUITE}]}}}, - {?eh,tc_done,{ct_framework,{end_per_group,g11,[{suite,group_timetrap_3_SUITE}]},ok}}], + {?eh,tc_done,{ct_framework, + {init_per_group,g11,[{suite,group_timetrap_3_SUITE}]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,tc_auto_skip,{group_timetrap_3_SUITE,t111,{group0_failed,bad_return_value}}}, + {?eh,test_stats,{0,13,{0,1}}}, + {?eh,tc_auto_skip,{ct_framework,{end_per_group,g11}, + {group0_failed,bad_return_value}}}], {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} @@ -545,7 +555,7 @@ test_events(require) -> {require_failed, {name_in_use,common2_alias,common2}}}}, {?eh,test_stats,{4,0,{0,1}}}, - {?eh,tc_auto_skip,{group_require_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{group_require_1_SUITE,{end_per_group,g4}, {require_failed,{name_in_use,common2_alias,common2}}}}], [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g5,[]}}}, @@ -573,7 +583,7 @@ test_events(require) -> {?eh,tc_auto_skip,{group_require_1_SUITE,t81, {require_failed,{not_available,non_existing}}}}, {?eh,test_stats,{8,0,{0,2}}}, - {?eh,tc_auto_skip,{group_require_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{group_require_1_SUITE,{end_per_group,g8}, {require_failed,{not_available,non_existing}}}}], [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g9,[]}}}, {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g9,[]},ok}}, @@ -591,12 +601,16 @@ test_events(require) -> {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g10,[]},ok}}], [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g11,[]}}}, - {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g11,[]},ok}}, - {?eh,tc_done,{group_require_1_SUITE,t111,ok}}, - {?eh,test_stats,{10,0,{0,3}}}, - {?eh,tc_start,{group_require_1_SUITE,{end_per_group,g11,[]}}}, - {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g11,[]},ok}}], - + {?eh,tc_done,{group_require_1_SUITE, + {init_per_group,g11,[]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,tc_auto_skip,{group_require_1_SUITE,t111, + {group0_failed,bad_return_value}}}, + {?eh,test_stats,{9,0,{0,4}}}, + {?eh,tc_auto_skip,{group_require_1_SUITE, + {end_per_group,g11}, + {group0_failed,bad_return_value}}}], + {?eh,tc_done,{group_require_1_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} @@ -635,7 +649,7 @@ test_events(require_default) -> {?eh,tc_auto_skip,{group_require_1_SUITE,t41, {require_failed,{not_available,common3}}}}, {?eh,test_stats,{4,0,{0,1}}}, - {?eh,tc_auto_skip,{group_require_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{group_require_1_SUITE,{end_per_group,g4}, {require_failed,{not_available,common3}}}}], [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g5,[]}}}, @@ -663,7 +677,7 @@ test_events(require_default) -> {?eh,tc_auto_skip,{group_require_1_SUITE,t81, {require_failed,{not_available,non_existing}}}}, {?eh,test_stats,{8,0,{0,2}}}, - {?eh,tc_auto_skip,{group_require_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{group_require_1_SUITE,{end_per_group,g8}, {require_failed,{not_available,non_existing}}}}], [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g9,[]}}}, @@ -682,11 +696,15 @@ test_events(require_default) -> {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g10,[]},ok}}], [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g11,[]}}}, - {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g11,[]},ok}}, - {?eh,tc_done,{group_require_1_SUITE,t111,ok}}, - {?eh,test_stats,{10,0,{0,3}}}, - {?eh,tc_start,{group_require_1_SUITE,{end_per_group,g11,[]}}}, - {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g11,[]},ok}}], + {?eh,tc_done,{group_require_1_SUITE, + {init_per_group,g11,[]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,tc_auto_skip,{group_require_1_SUITE,t111, + {group0_failed,bad_return_value}}}, + {?eh,test_stats,{9,0,{0,4}}}, + {?eh,tc_auto_skip,{group_require_1_SUITE, + {end_per_group,g11}, + {group0_failed,bad_return_value}}}], {?eh,tc_done,{group_require_1_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, @@ -725,7 +743,7 @@ test_events(require_no_ips) -> {?eh,tc_auto_skip,{group_require_2_SUITE,t41, {require_failed,{name_in_use,common2_alias,common2}}}}, {?eh,test_stats,{4,0,{0,1}}}, - {?eh,tc_auto_skip,{group_require_2_SUITE,end_per_group, + {?eh,tc_auto_skip,{group_require_2_SUITE,{end_per_group,g4}, {require_failed,{name_in_use,common2_alias,common2}}}}], [{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g5,[]}}}, @@ -753,7 +771,7 @@ test_events(require_no_ips) -> {?eh,tc_auto_skip,{group_require_2_SUITE,t81, {require_failed,{not_available,non_existing}}}}, {?eh,test_stats,{8,0,{0,2}}}, - {?eh,tc_auto_skip,{group_require_2_SUITE,end_per_group, + {?eh,tc_auto_skip,{group_require_2_SUITE,{end_per_group,g8}, {require_failed,{not_available,non_existing}}}}], [{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g9,[]}}}, {?eh,tc_done,{group_require_2_SUITE,{init_per_group,g9,[]},ok}}, @@ -769,13 +787,17 @@ test_events(require_no_ips) -> {?eh,tc_done,{group_require_2_SUITE,t101,ok}}, {?eh,tc_start,{group_require_2_SUITE,{end_per_group,g10,[]}}}, {?eh,tc_done,{group_require_2_SUITE,{end_per_group,g10,[]},ok}}], - + [{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g11,[]}}}, - {?eh,tc_done,{group_require_2_SUITE,{init_per_group,g11,[]},ok}}, - {?eh,tc_done,{group_require_2_SUITE,t111,ok}}, - {?eh,test_stats,{10,0,{0,3}}}, - {?eh,tc_start,{group_require_2_SUITE,{end_per_group,g11,[]}}}, - {?eh,tc_done,{group_require_2_SUITE,{end_per_group,g11,[]},ok}}], + {?eh,tc_done,{group_require_2_SUITE, + {init_per_group,g11,[]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,tc_auto_skip,{group_require_2_SUITE,t111, + {group0_failed,bad_return_value}}}, + {?eh,test_stats,{9,0,{0,4}}}, + {?eh,tc_auto_skip,{group_require_2_SUITE, + {end_per_group,g11}, + {group0_failed,bad_return_value}}}], {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} @@ -812,7 +834,7 @@ test_events(require_no_ipg) -> {?eh,tc_auto_skip,{group_require_3_SUITE,t41, {require_failed,{name_in_use,common2_alias,common2}}}}, {?eh,test_stats,{4,0,{0,1}}}, - {?eh,tc_auto_skip,{ct_framework,end_per_group, + {?eh,tc_auto_skip,{ct_framework,{end_per_group,g4}, {require_failed,{name_in_use,common2_alias,common2}}}}], [{?eh,tc_start,{ct_framework,{init_per_group,g5,[{suite,group_require_3_SUITE}]}}}, @@ -838,7 +860,7 @@ test_events(require_no_ipg) -> {?eh,tc_auto_skip,{group_require_3_SUITE,t81, {require_failed,{not_available,non_existing}}}}, {?eh,test_stats,{8,0,{0,2}}}, - {?eh,tc_auto_skip,{ct_framework,end_per_group, + {?eh,tc_auto_skip,{ct_framework,{end_per_group,g8}, {require_failed,{not_available,non_existing}}}}], [{?eh,tc_start,{ct_framework,{init_per_group,g9,[{suite,group_require_3_SUITE}]}}}, {?eh,tc_done,{ct_framework,{init_per_group,g9,[{suite,group_require_3_SUITE}]},ok}}, @@ -853,13 +875,14 @@ test_events(require_no_ipg) -> {?eh,tc_done,{group_require_3_SUITE,t101,ok}}, {?eh,tc_start,{ct_framework,{end_per_group,g10,[{suite,group_require_3_SUITE}]}}}, {?eh,tc_done,{ct_framework,{end_per_group,g10,[{suite,group_require_3_SUITE}]},ok}}], - + [{?eh,tc_start,{ct_framework,{init_per_group,g11,[{suite,group_require_3_SUITE}]}}}, - {?eh,tc_done,{ct_framework,{init_per_group,g11,[{suite,group_require_3_SUITE}]},ok}}, - {?eh,tc_done,{group_require_3_SUITE,t111,ok}}, - {?eh,test_stats,{10,0,{0,3}}}, - {?eh,tc_start,{ct_framework,{end_per_group,g11,[{suite,group_require_3_SUITE}]}}}, - {?eh,tc_done,{ct_framework,{end_per_group,g11,[{suite,group_require_3_SUITE}]},ok}}], + {?eh,tc_done,{ct_framework,{init_per_group,g11,[{suite,group_require_3_SUITE}]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,tc_auto_skip,{group_require_3_SUITE,t111,{group0_failed,bad_return_value}}}, + {?eh,test_stats,{9,0,{0,4}}}, + {?eh,tc_auto_skip,{ct_framework,{end_per_group,g11}, + {group0_failed,bad_return_value}}}], {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} diff --git a/lib/common_test/test/ct_repeat_1_SUITE.erl b/lib/common_test/test/ct_repeat_1_SUITE.erl index 090002d0c2..98eaa28763 100644 --- a/lib/common_test/test/ct_repeat_1_SUITE.erl +++ b/lib/common_test/test/ct_repeat_1_SUITE.erl @@ -229,7 +229,7 @@ test_events(repeat_cs_and_grs) -> {failed,{repeat_1_SUITE,init_per_group, {'EXIT',fails_on_purpose}}}}}, {?eh,test_stats,{3,1,{0,1}}}, - {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init}, {failed,{repeat_1_SUITE,init_per_group, {'EXIT',fails_on_purpose}}}}}], {?eh,test_stats,{4,1,{0,1}}}, @@ -251,7 +251,7 @@ test_events(repeat_cs_and_grs) -> {failed,{repeat_1_SUITE,init_per_group, {'EXIT',fails_on_purpose}}}}}, {?eh,test_stats,{7,2,{0,2}}}, - {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init}, {failed,{repeat_1_SUITE,init_per_group, {'EXIT',fails_on_purpose}}}}}], {?eh,test_stats,{8,2,{0,2}}}, @@ -311,7 +311,7 @@ test_events(repeat_seq) -> {init_per_group,gr_fail_init,[]}, {failed,{error,fails_on_purpose}}}}, {?eh,test_stats,{7,2,{0,5}}}, - {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init}, {failed, {repeat_1_SUITE,init_per_group, {'EXIT',fails_on_purpose}}}}}], @@ -410,7 +410,7 @@ test_events(repeat_gr_until_any_ok) -> {init_per_group,gr_fail_init,[]}, {failed,{error,fails_on_purpose}}}}, {?eh,test_stats,{1,1,{0,1}}}, - {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init}, {failed,{repeat_1_SUITE,init_per_group, {'EXIT',fails_on_purpose}}}}}], {?eh,test_stats,{1,2,{0,1}}}, @@ -452,7 +452,7 @@ test_events(repeat_gr_until_any_ok) -> {init_per_group,gr_fail_init,[]}, {failed,{error,fails_on_purpose}}}}, {?eh,test_stats,{5,6,{0,3}}}, - {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init}, {failed,{repeat_1_SUITE,init_per_group, {'EXIT',fails_on_purpose}}}}}], {?eh,tc_done,{repeat_1_SUITE, @@ -469,7 +469,7 @@ test_events(repeat_gr_until_any_ok) -> {init_per_group,gr_fail_init,[]}, {failed,{error,fails_on_purpose}}}}, {?eh,test_stats,{7,7,{0,4}}}, - {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init}, {failed,{repeat_1_SUITE,init_per_group, {'EXIT',fails_on_purpose}}}}}], {?eh,tc_done,{repeat_1_SUITE, @@ -770,7 +770,7 @@ test_events(repeat_gr_until_any_fail) -> {'EXIT',failing_this_time}}}}}, {?eh,test_stats,{14,0,{0,1}}}, {?eh,tc_auto_skip, - {repeat_1_SUITE,end_per_group, + {repeat_1_SUITE,{end_per_group,gr_ok_then_fail_init}, {failed, {repeat_1_SUITE,init_per_group, {'EXIT',failing_this_time}}}}}], @@ -967,7 +967,7 @@ test_events(repeat_gr_until_all_ok) -> {failed,{repeat_1_SUITE,init_per_group, {'EXIT',failing_this_time}}}}}, {?eh,test_stats,{7,1,{0,1}}}, - {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init_then_ok}, {failed,{repeat_1_SUITE,init_per_group, {'EXIT',failing_this_time}}}}}], {?eh,tc_done,{repeat_1_SUITE,tc_ok_1,ok}}, @@ -1083,7 +1083,7 @@ test_events(repeat_gr_until_all_fail) -> {init_per_group,gr_fail_init,[]}, {failed,{error,fails_on_purpose}}}}, {?eh,test_stats,{0,1,{0,1}}}, - {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init}, {failed,{repeat_1_SUITE,init_per_group, {'EXIT',fails_on_purpose}}}}}], {?eh,tc_done,{repeat_1_SUITE,tc_ok_then_fail_1,ok}}, @@ -1102,7 +1102,7 @@ test_events(repeat_gr_until_all_fail) -> {init_per_group,gr_fail_init,[]}, {failed,{error,fails_on_purpose}}}}, {?eh,test_stats,{2,2,{0,2}}}, - {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init}, {failed,{repeat_1_SUITE,init_per_group, {'EXIT',fails_on_purpose}}}}}], {?eh,tc_done,{repeat_1_SUITE,tc_ok_then_fail_1, @@ -1134,7 +1134,7 @@ test_events(repeat_gr_until_all_fail) -> {init_per_group,gr_ok_then_fail_init,[]}, {failed,{error,failing_this_time}}}}, {?eh,test_stats,{4,4,{0,3}}}, - {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_ok_then_fail_init}, {failed,{repeat_1_SUITE,init_per_group, {'EXIT',failing_this_time}}}}}], {?eh,tc_start,{repeat_1_SUITE,tc_fail_1}}, @@ -1293,7 +1293,7 @@ test_events(repeat_seq_until_any_fail) -> {init_per_group,gr_ok_then_fail_init,[]}, {failed,{error,failing_this_time}}}}, {?eh,test_stats,{24,1,{0,5}}}, - {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_ok_then_fail_init}, {failed,{repeat_1_SUITE,init_per_group, {'EXIT',failing_this_time}}}}}], {?eh,tc_auto_skip,{repeat_1_SUITE,tc_ok_1, @@ -1516,7 +1516,7 @@ test_events(repeat_shuffled_seq_until_any_fail) -> {init_per_group,repeat_shuffled_seq_until_any_fail_5, [{shuffle,'_'},{repeat_until_any_fail,2}, sequence]},ok}}, - [{?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + [{?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_ok_then_fail_init}, {failed,{repeat_1_SUITE,init_per_group, {'EXIT',failing_this_time}}}}}], {?eh,tc_start,{repeat_1_SUITE, diff --git a/lib/common_test/test/ct_repeat_testrun_SUITE.erl b/lib/common_test/test/ct_repeat_testrun_SUITE.erl index 9d5216449b..bb2aba2c5a 100644 --- a/lib/common_test/test/ct_repeat_testrun_SUITE.erl +++ b/lib/common_test/test/ct_repeat_testrun_SUITE.erl @@ -36,7 +36,7 @@ -define(eh, ct_test_support_eh). -define(skip_reason, "Repeated test stopped by force_stop option"). --define(skipped, {skipped, ?skip_reason}). +-define(skipped, {auto_skipped, ?skip_reason}). %% Timers used in this test. @@ -222,7 +222,7 @@ until_force_stop_skip_rest_group(Config) when is_list(Config) -> fun() -> [_] = ct_test_support:run_ct_run_test( Opts++[{until,until_str(?t3,1,Config)}],Config), - 0 = ct_test_support:run_ct_script_start( + 1 = ct_test_support:run_ct_script_start( Opts++[{until,until_str(?t3,1,Config)}],Config) end, ok = execute(ExecuteFun, @@ -341,19 +341,18 @@ skip_first_tc1(Suite) -> {?eh,tc_done,{Suite,tc1,ok}}, {?eh,test_stats,{'_',0,{0,0}}}, {?eh,tc_done,{Suite,tc2,?skipped}}, - {?eh,test_stats,{'_',0,{1,0}}}, + {?eh,test_stats,{'_',0,{0,1}}}, {?eh,tc_done,{Suite,{init_per_group,g,[]},?skipped}}, - {?eh,tc_user_skip,{Suite,tc1,?skip_reason}}, - {?eh,test_stats,{'_',0,{2,0}}}, - {?eh,tc_user_skip,{Suite,tc2,?skip_reason}}, - {?eh,test_stats,{'_',0,{3,0}}}, - {?eh,tc_user_skip,{Suite,end_per_group,?skip_reason}}, + {?eh,tc_auto_skip,{Suite,tc1,?skip_reason}}, + {?eh,test_stats,{'_',0,{0,2}}}, + {?eh,tc_auto_skip,{Suite,tc2,?skip_reason}}, + {?eh,test_stats,{'_',0,{0,3}}}, + {?eh,tc_auto_skip,{Suite,{end_per_group,g},?skip_reason}}, {?eh,tc_done,{Suite,tc2,?skipped}}, - {?eh,test_stats,{'_',0,{4,0}}}, + {?eh,test_stats,{'_',0,{0,4}}}, {?eh,tc_start,{Suite,end_per_suite}}, {?eh,tc_done,{Suite,end_per_suite,ok}}]. - skip_tc1_in_group(Suite) -> [{?eh,tc_start,{Suite,init_per_suite}}, {?eh,tc_done,{Suite,init_per_suite,ok}}, @@ -369,10 +368,10 @@ skip_tc1_in_group(Suite) -> {?eh,tc_done,{Suite,tc1,ok}}, {?eh,test_stats,{'_',0,{0,0}}}, {?eh,tc_done,{Suite,tc2,?skipped}}, - {?eh,test_stats,{'_',0,{1,0}}}, + {?eh,test_stats,{'_',0,{0,1}}}, {?eh,tc_start,{Suite,{end_per_group,g,[]}}}, {?eh,tc_done,{Suite,{end_per_group,g,[]},ok}}], {?eh,tc_done,{Suite,tc2,?skipped}}, - {?eh,test_stats,{'_',0,{2,0}}}, + {?eh,test_stats,{'_',0,{0,2}}}, {?eh,tc_start,{Suite,end_per_suite}}, {?eh,tc_done,{Suite,end_per_suite,ok}}]. diff --git a/lib/common_test/test/ct_sequence_1_SUITE.erl b/lib/common_test/test/ct_sequence_1_SUITE.erl index 5facf90656..8c87236838 100644 --- a/lib/common_test/test/ct_sequence_1_SUITE.erl +++ b/lib/common_test/test/ct_sequence_1_SUITE.erl @@ -212,7 +212,7 @@ test_events(subgroup_init_fail) -> {failed,{subgroups_1_SUITE,init_per_group, {'EXIT',init_per_group_fails_on_purpose}}}}}, {?eh,test_stats,{0,0,{0,1}}}, - {?eh,tc_auto_skip,{subgroups_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{subgroups_1_SUITE,{end_per_group,fail_init}, {failed,{subgroups_1_SUITE,init_per_group, {'EXIT',init_per_group_fails_on_purpose}}}}}], {?eh,tc_auto_skip,{subgroups_1_SUITE,ok_tc,{group_result,fail_init,failed}}}, @@ -294,7 +294,7 @@ test_events(case_after_subgroup_fail_init) -> {subgroups_1_SUITE,init_per_group, {'EXIT',init_per_group_fails_on_purpose}}}}}, {?eh,test_stats,{0,0,{0,1}}}, - {?eh,tc_auto_skip,{subgroups_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{subgroups_1_SUITE,{end_per_group,fail_init}, {failed, {subgroups_1_SUITE,init_per_group, {'EXIT',init_per_group_fails_on_purpose}}}}}], diff --git a/lib/common_test/test/ct_skip_SUITE.erl b/lib/common_test/test/ct_skip_SUITE.erl index 5f88ac4825..b0a6c839a2 100644 --- a/lib/common_test/test/ct_skip_SUITE.erl +++ b/lib/common_test/test/ct_skip_SUITE.erl @@ -282,8 +282,9 @@ test_events(auto_skip) -> {'EXIT',{group,g1,failed}}}}}}, {?eh,test_stats,{2,0,{0,7}}}, {?eh,tc_auto_skip, - {auto_skip_5_SUITE,end_per_group,{failed,{auto_skip_5_SUITE,init_per_group, - {'EXIT',{group,g1,failed}}}}}}], + {auto_skip_5_SUITE,{end_per_group,g1}, + {failed,{auto_skip_5_SUITE,init_per_group, + {'EXIT',{group,g1,failed}}}}}}], {?eh,tc_start,{auto_skip_5_SUITE,end_per_suite}}, {?eh,tc_done,{auto_skip_5_SUITE,end_per_suite,ok}}, @@ -310,8 +311,9 @@ test_events(auto_skip) -> {'EXIT',{group,g1,failed}}}}}}, {?eh,test_stats,{2,0,{0,11}}}, {?eh,tc_auto_skip, - {auto_skip_6_SUITE,end_per_group,{failed,{auto_skip_6_SUITE,init_per_group, - {'EXIT',{group,g1,failed}}}}}}], + {auto_skip_6_SUITE,{end_per_group,g1}, + {failed,{auto_skip_6_SUITE,init_per_group, + {'EXIT',{group,g1,failed}}}}}}], [{?eh,tc_start,{auto_skip_6_SUITE,{init_per_group,g3,[]}}}, {?eh,tc_done,{auto_skip_6_SUITE,{init_per_group,g3,[]},ok}}, @@ -330,8 +332,9 @@ test_events(auto_skip) -> {'EXIT',{group,g4,failed}}}}}}, {?eh,test_stats,{3,0,{0,13}}}, {?eh,tc_auto_skip, - {auto_skip_6_SUITE,end_per_group,{failed,{auto_skip_6_SUITE,init_per_group, - {'EXIT',{group,g4,failed}}}}}}], + {auto_skip_6_SUITE,{end_per_group,g4}, + {failed,{auto_skip_6_SUITE,init_per_group, + {'EXIT',{group,g4,failed}}}}}}], {?eh,tc_start,{auto_skip_6_SUITE,tc2}}, {?eh,tc_done,{auto_skip_6_SUITE,tc2,ok}}, {?eh,test_stats,{4,0,{0,13}}}, @@ -504,7 +507,7 @@ test_events(auto_skip) -> {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3, {require_failed,{not_available,unknown_variable_g1}}}}, {?eh,test_stats,{10,0,{0,27}}}, - {?eh,tc_auto_skip,{ct_framework,end_per_group, + {?eh,tc_auto_skip,{ct_framework,{end_per_group,g1}, {require_failed,{not_available,unknown_variable_g1}}}}], [{?eh,tc_start,{ct_framework,{init_per_group,g1, @@ -522,7 +525,7 @@ test_events(auto_skip) -> {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3, {require_failed,{not_available,unknown_variable_g1}}}}, {?eh,test_stats,{10,0,{0,30}}}, - {?eh,tc_auto_skip,{ct_framework,end_per_group, + {?eh,tc_auto_skip,{ct_framework,{end_per_group,g1}, {require_failed,{not_available,unknown_variable_g1}}}}], [{?eh,tc_start,{ct_framework,{init_per_group,g3, @@ -544,7 +547,7 @@ test_events(auto_skip) -> {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3, {require_failed,{not_available,unknown_variable_g4}}}}, {?eh,test_stats,{12,0,{0,31}}}, - {?eh,tc_auto_skip,{ct_framework,end_per_group, + {?eh,tc_auto_skip,{ct_framework,{end_per_group,g4}, {require_failed,{not_available,unknown_variable_g4}}}}], {?eh,tc_start,{ct_framework,{end_per_group,g3, @@ -639,7 +642,7 @@ test_events(user_skip) -> {?eh,test_stats,{3,0,{10,0}}}, {?eh,tc_user_skip,{user_skip_4_SUITE,tc2,"Group skipped"}}, {?eh,test_stats,{3,0,{11,0}}}, - {?eh,tc_user_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}], + {?eh,tc_user_skip,{user_skip_4_SUITE,{end_per_group,g1},"Group skipped"}}], [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g2,[]}}}, {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g2,[]},ok}}, @@ -659,7 +662,7 @@ test_events(user_skip) -> {?eh,tc_user_skip,{user_skip_4_SUITE,tc7,"Group skipped"}}, {?eh,tc_user_skip,{user_skip_4_SUITE,tc8,"Group skipped"}}, {?eh,test_stats,{5,0,{15,0}}}, - {?eh,tc_user_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}], + {?eh,tc_user_skip,{user_skip_4_SUITE,{end_per_group,g3},"Group skipped"}}], [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g5,[]}}}, {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g5,[]},ok}}, @@ -672,7 +675,7 @@ test_events(user_skip) -> {?eh,test_stats,{6,0,{16,0}}}, {?eh,tc_user_skip,{user_skip_4_SUITE,tc11,"Group skipped"}}, {?eh,test_stats,{6,0,{17,0}}}, - {?eh,tc_user_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}], + {?eh,tc_user_skip,{user_skip_4_SUITE,{end_per_group,g6},"Group skipped"}}], {?eh,tc_start,{user_skip_4_SUITE,{end_per_group,g5,[]}}}, {?eh,tc_done,{user_skip_4_SUITE,{end_per_group,g5,[]},ok}}], @@ -701,7 +704,8 @@ test_events(user_skip) -> {?eh,tc_user_skip,{user_skip_6_SUITE,tc3,"Top group skipped"}}, {?eh,tc_user_skip,{user_skip_6_SUITE,tc4,"Top group skipped"}}, {?eh,tc_user_skip,{user_skip_6_SUITE,tc2,"Top group skipped"}}, - {?eh,tc_user_skip,{user_skip_6_SUITE,end_per_group,"Top group skipped"}}]}, + {?eh,tc_user_skip,{user_skip_6_SUITE,{end_per_group,ptop1}, + "Top group skipped"}}]}, {parallel, [{?eh,tc_start,{user_skip_6_SUITE,{init_per_group,ptop2,[parallel]}}}, @@ -716,7 +720,8 @@ test_events(user_skip) -> {skipped,"Sub group skipped"}}}, {?eh,tc_user_skip,{user_skip_6_SUITE,tc3,"Sub group skipped"}}, {?eh,tc_user_skip,{user_skip_6_SUITE,tc4,"Sub group skipped"}}, - {?eh,tc_user_skip,{user_skip_6_SUITE,end_per_group,"Sub group skipped"}}]}, + {?eh,tc_user_skip,{user_skip_6_SUITE,{end_per_group,psub2}, + "Sub group skipped"}}]}, {?eh,tc_start,{user_skip_6_SUITE,tc2}}, {?eh,tc_done,{user_skip_6_SUITE,tc2,ok}}, @@ -772,7 +777,7 @@ test_events(testspec_skip) -> {?eh,start_info,{1,1,4}}, {?eh,tc_start,{ct_framework,init_per_suite}}, {?eh,tc_done,{ct_framework,init_per_suite,ok}}, - {?eh,tc_user_skip,{user_skip_7_SUITE,init_per_group,"SKIPPED"}}, + {?eh,tc_user_skip,{user_skip_7_SUITE,{init_per_group,ptop1},"SKIPPED"}}, {?eh,tc_user_skip,{user_skip_7_SUITE,tc1,"SKIPPED"}}, {?eh,test_stats,{0,0,{1,0}}}, {?eh,tc_user_skip,{user_skip_7_SUITE,tc3,"SKIPPED"}}, @@ -781,7 +786,7 @@ test_events(testspec_skip) -> {?eh,test_stats,{0,0,{3,0}}}, {?eh,tc_user_skip,{user_skip_7_SUITE,tc2,"SKIPPED"}}, {?eh,test_stats,{0,0,{4,0}}}, - {?eh,tc_user_skip,{user_skip_7_SUITE,end_per_group,"SKIPPED"}}, + {?eh,tc_user_skip,{user_skip_7_SUITE,{end_per_group,ptop1},"SKIPPED"}}, {?eh,tc_start,{ct_framework,end_per_suite}}, {?eh,tc_done,{ct_framework,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, @@ -798,11 +803,11 @@ test_events(testspec_skip) -> {?eh,tc_done, {user_skip_7_SUITE,{init_per_group,ptop1,[parallel]},ok}}, {?eh,tc_user_skip, - {user_skip_7_SUITE,init_per_group,"SKIPPED"}}, + {user_skip_7_SUITE,{init_per_group,psub1},"SKIPPED"}}, {?eh,tc_user_skip,{user_skip_7_SUITE,tc3,"SKIPPED"}}, {?eh,tc_user_skip,{user_skip_7_SUITE,tc4,"SKIPPED"}}, {?eh,test_stats,{0,0,{2,0}}}, - {?eh,tc_user_skip,{user_skip_7_SUITE,end_per_group,"SKIPPED"}}, + {?eh,tc_user_skip,{user_skip_7_SUITE,{end_per_group,psub1},"SKIPPED"}}, {?eh,tc_start,{user_skip_7_SUITE,tc1}}, {?eh,tc_done,{user_skip_7_SUITE,tc1,ok}}, {?eh,tc_start,{user_skip_7_SUITE,tc2}}, @@ -841,7 +846,7 @@ test_events(testspec_skip) -> {?eh,tc_user_skip,{user_skip_6_SUITE,tc2,"Top group skipped"}}, {?eh,test_stats,{0,0,{4,0}}}, {?eh,tc_user_skip, - {user_skip_6_SUITE,end_per_group,"Top group skipped"}}]}, + {user_skip_6_SUITE,{end_per_group,ptop1},"Top group skipped"}}]}, {?eh,tc_start,{ct_framework,end_per_suite}}, {?eh,tc_done,{ct_framework,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, diff --git a/lib/common_test/test/ct_surefire_SUITE.erl b/lib/common_test/test/ct_surefire_SUITE.erl index d9f46af3ae..c5e44682b0 100644 --- a/lib/common_test/test/ct_surefire_SUITE.erl +++ b/lib/common_test/test/ct_surefire_SUITE.erl @@ -210,7 +210,7 @@ test_events(_) -> {surefire_SUITE,init_per_group, {'EXIT',all_cases_should_be_skipped}}}}}, {?eh,test_stats,{2,2,{2,3}}}, - {?eh,tc_auto_skip,{surefire_SUITE,end_per_group, + {?eh,tc_auto_skip,{surefire_SUITE,{end_per_group,g_fail}, {failed, {surefire_SUITE,init_per_group, {'EXIT',all_cases_should_be_skipped}}}}}], diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE.erl index c4b3191801..9882fa980c 100644 --- a/lib/common_test/test/ct_test_server_if_1_SUITE.erl +++ b/lib/common_test/test/ct_test_server_if_1_SUITE.erl @@ -201,7 +201,7 @@ test_events(ts_if_1) -> {skipped,g1_got_skipped}}}, {?eh,tc_user_skip,{ts_if_1_SUITE,gtc1,g1_got_skipped}}, {?eh,test_stats,{1,4,{3,6}}}, - {?eh,tc_user_skip,{ts_if_1_SUITE,end_per_group,g1_got_skipped}}], + {?eh,tc_user_skip,{ts_if_1_SUITE,{end_per_group,g1},g1_got_skipped}}], {parallel, [{?eh,tc_start,{ts_if_1_SUITE,{init_per_group,g2,[parallel]}}}, @@ -210,7 +210,7 @@ test_events(ts_if_1) -> {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g3,[]},{skipped,g3_got_skipped}}}, {?eh,tc_user_skip,{ts_if_1_SUITE,gtc2,g3_got_skipped}}, {?eh,test_stats,{1,4,{4,6}}}, - {?eh,tc_user_skip,{ts_if_1_SUITE,end_per_group,g3_got_skipped}}], + {?eh,tc_user_skip,{ts_if_1_SUITE,{end_per_group,g3},g3_got_skipped}}], {?eh,tc_start,{ts_if_1_SUITE,{end_per_group,g2,[parallel]}}}, {?eh,tc_done,{ts_if_1_SUITE,{end_per_group,g2,[parallel]},ok}}]}, @@ -282,40 +282,40 @@ test_events(ts_if_1) -> {ts_if_7_SUITE,tc2,{group0_failed,bad_return_value}}}, {?eh,test_stats,{2,7,{4,11}}}, {?eh,tc_auto_skip, - {ts_if_7_SUITE,end_per_group,{group0_failed,bad_return_value}}}], - - [{?eh,tc_start,{ts_if_7_SUITE,{init_per_group,g2,[]}}}, - {?eh,tc_done,{ts_if_7_SUITE,{init_per_group,g2,[]},ok}}, - {?eh,tc_done,{ts_if_7_SUITE,tc2, - {auto_skipped,{testcase0_failed,bad_return_value}}}}, - {?eh,test_stats,{2,7,{4,12}}}, - {?eh,tc_start,{ts_if_7_SUITE,{end_per_group,g2,[]}}}, - {?eh,tc_done,{ts_if_7_SUITE,{end_per_group,g2,[]},ok}}], - - {?eh,tc_start,{ct_framework,end_per_suite}}, - {?eh,tc_done,{ct_framework,end_per_suite,ok}}, - - - {?eh,tc_start,{ct_framework,init_per_suite}}, - {?eh,tc_done,{ct_framework,init_per_suite,ok}}, - {?eh,tc_start,{ts_if_8_SUITE,tc1}}, - {?eh,tc_done,{ts_if_8_SUITE,tc1,{failed,{error,failed_on_purpose}}}}, - {?eh,test_stats,{2,8,{4,12}}}, - {?eh,tc_start,{ct_framework,end_per_suite}}, - {?eh,tc_done,{ct_framework,end_per_suite,ok}}, - - - {?eh,tc_user_skip,{skipped_by_spec_1_SUITE,all,"should be skipped"}}, - {?eh,test_stats,{2,8,{5,12}}}, - {?eh,tc_start,{skipped_by_spec_2_SUITE,init_per_suite}}, - {?eh,tc_done,{skipped_by_spec_2_SUITE,init_per_suite,ok}}, - {?eh,tc_user_skip,{skipped_by_spec_2_SUITE,tc1,"should be skipped"}}, - {?eh,test_stats,{2,8,{6,12}}}, - {?eh,tc_start,{skipped_by_spec_2_SUITE,end_per_suite}}, - {?eh,tc_done,{skipped_by_spec_2_SUITE,end_per_suite,ok}}, - - {?eh,test_done,{'DEF','STOP_TIME'}}, - {?eh,stop_logging,[]} - ]. + {ts_if_7_SUITE,{end_per_group,g1},{group0_failed,bad_return_value}}}, + + [{?eh,tc_start,{ts_if_7_SUITE,{init_per_group,g2,[]}}}, + {?eh,tc_done,{ts_if_7_SUITE,{init_per_group,g2,[]},ok}}, + {?eh,tc_done,{ts_if_7_SUITE,tc2, + {auto_skipped,{testcase0_failed,bad_return_value}}}}, + {?eh,test_stats,{2,7,{4,12}}}, + {?eh,tc_start,{ts_if_7_SUITE,{end_per_group,g2,[]}}}, + {?eh,tc_done,{ts_if_7_SUITE,{end_per_group,g2,[]},ok}}], + + {?eh,tc_start,{ct_framework,end_per_suite}}, + {?eh,tc_done,{ct_framework,end_per_suite,ok}}, + + + {?eh,tc_start,{ct_framework,init_per_suite}}, + {?eh,tc_done,{ct_framework,init_per_suite,ok}}, + {?eh,tc_start,{ts_if_8_SUITE,tc1}}, + {?eh,tc_done,{ts_if_8_SUITE,tc1,{failed,{error,failed_on_purpose}}}}, + {?eh,test_stats,{2,8,{4,12}}}, + {?eh,tc_start,{ct_framework,end_per_suite}}, + {?eh,tc_done,{ct_framework,end_per_suite,ok}}, + + + {?eh,tc_user_skip,{skipped_by_spec_1_SUITE,all,"should be skipped"}}, + {?eh,test_stats,{2,8,{5,12}}}, + {?eh,tc_start,{skipped_by_spec_2_SUITE,init_per_suite}}, + {?eh,tc_done,{skipped_by_spec_2_SUITE,init_per_suite,ok}}, + {?eh,tc_user_skip,{skipped_by_spec_2_SUITE,tc1,"should be skipped"}}, + {?eh,test_stats,{2,8,{6,12}}}, + {?eh,tc_start,{skipped_by_spec_2_SUITE,end_per_suite}}, + {?eh,tc_done,{skipped_by_spec_2_SUITE,end_per_suite,ok}}, + + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]. diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl index 6a4a4acd80..187b5e6d3a 100644 --- a/lib/common_test/test/ct_testspec_1_SUITE.erl +++ b/lib/common_test/test/ct_testspec_1_SUITE.erl @@ -760,16 +760,42 @@ test_events(all_groups) -> test_events(skip_all_groups) -> [ {?eh,start_logging,'_'}, + {?eh,start_info,{1,1,12}}, {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, - {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1a},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1a},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}}, {?eh,test_stats,{0,0,{1,0}}}, - {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}}, {?eh,test_stats,{0,0,{2,0}}}, - {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_2},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1a},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1b},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}}, {?eh,test_stats,{0,0,{3,0}}}, - {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_4},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}}, {?eh,test_stats,{0,0,{4,0}}}, - {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1b},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_2},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_2a,"SKIPPED!"}}, + {?eh,test_stats,{0,0,{5,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_3a,"SKIPPED!"}}, + {?eh,test_stats,{0,0,{6,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_3b,"SKIPPED!"}}, + {?eh,test_stats,{0,0,{7,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_2b,"SKIPPED!"}}, + {?eh,test_stats,{0,0,{8,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_2},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_4},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_5a,"SKIPPED!"}}, + {?eh,test_stats,{0,0,{9,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_7a,"SKIPPED!"}}, + {?eh,test_stats,{0,0,{10,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_7b,"SKIPPED!"}}, + {?eh,test_stats,{0,0,{11,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_5b,"SKIPPED!"}}, + {?eh,test_stats,{0,0,{12,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_4},"SKIPPED!"}}, + {?eh,tc_start,{groups_11_SUITE,end_per_suite}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,init}}, {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} ]; @@ -788,21 +814,35 @@ test_events(group) -> test_events(skip_group) -> [ - {?eh,start_logging,'_'}, - {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, - - {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}}, - {?eh,tc_start,{groups_11_SUITE,testcase_1a}}, - {?eh,tc_start,{groups_11_SUITE,testcase_1b}}, - {?eh,test_stats,{2,0,{0,0}}}, - {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}}, - - {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}}, - {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_2},"SKIPPED!"}}, - %%! But not test_group_7 since it's a sub-group! - {?eh,test_stats,{2,0,{2,0}}}, - {negative,{?eh,tc_user_skip,'_'},{?eh,stop_logging,'_'}} - ]; + {?eh,start_logging,'_'}, + {?eh,start_info,{1,1,8}}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + [{?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}}, + {?eh,tc_done,{groups_11_SUITE,{init_per_group,test_group_1a,[]},ok}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1a}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1b}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_start,{groups_11_SUITE,{end_per_group,test_group_1a,[]}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},ok}}], + {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1b}, + "SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}}, + {?eh,test_stats,{2,0,{2,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1b}, + "SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_2}, + "SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_2a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_3a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_3b,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_2b,"SKIPPED!"}}, + {?eh,test_stats,{2,0,{6,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_2}, + "SKIPPED!"}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,init}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; test_events(group_all_testcases) -> [ @@ -820,11 +860,23 @@ test_events(group_all_testcases) -> test_events(skip_group_all_testcases) -> [ {?eh,start_logging,'_'}, + {?eh,start_info,{1,1,4}}, {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, - {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1a},"SKIPPED!"}}, - {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1a}, + "SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}}, {?eh,test_stats,{0,0,{2,0}}}, - {?eh,tc_start,{groups_11_SUITE,end_per_suite}}, + {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1a}, + "SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1b}, + "SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}}, + {?eh,test_stats,{0,0,{4,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1b}, + "SKIPPED!"}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,init}}, {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} ]; @@ -934,7 +986,7 @@ test_events(subgroup) -> [ {?eh,start_logging,'_'}, {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, - + {parallel, [{?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_2,[parallel]}}}, @@ -962,36 +1014,58 @@ test_events(subgroup) -> test_events(skip_subgroup) -> [ {?eh,start_logging,'_'}, + {?eh,start_info,{1,1,6}}, {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, - [{?eh,tc_start, - {groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_4,[]},ok}}, + {parallel, - [{?eh,tc_start, - {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}}, - {?eh,tc_done, - {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}}, - {parallel, - [{?eh,tc_start, - {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}}, - {?eh,tc_done, - {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}}, - [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}}, - {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}], - {?eh,tc_user_skip, - {groups_12_SUITE,{group,test_group_8},"SKIPPED!"}}, - {?eh,tc_start, - {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}}, - {?eh,tc_done, - {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}} - ]}, - {?eh,tc_start, - {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}}, - {?eh,tc_done, - {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]}, - {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}], + [{?eh,tc_start,{groups_12_SUITE, + {init_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done,{groups_12_SUITE, + {init_per_group,test_group_5,[parallel]},ok}}, + + {parallel, + [{?eh,tc_start,{groups_12_SUITE, + {init_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done,{groups_12_SUITE, + {init_per_group,test_group_6,[parallel]},ok}}, + + [{?eh,tc_start,{groups_12_SUITE, + {init_per_group,test_group_7,[sequence]}}}, + {?eh,tc_done,{groups_12_SUITE, + {init_per_group,test_group_7,[sequence]},ok}}, + {?eh,tc_done,{groups_12_SUITE,testcase_7a,ok}}, + {?eh,tc_done,{groups_12_SUITE,testcase_7b,ok}}, + {?eh,tc_start,{groups_12_SUITE, + {end_per_group,test_group_7,[sequence]}}}, + {?eh,tc_done,{groups_12_SUITE, + {end_per_group,test_group_7,[sequence]},ok}}], + + {?eh,tc_user_skip,{groups_12_SUITE, + {init_per_group,test_group_8},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_8a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_8b,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE, + {end_per_group,test_group_8},"SKIPPED!"}}, + + {?eh,tc_start,{groups_12_SUITE, + {end_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done,{groups_12_SUITE, + {end_per_group,test_group_6,[parallel]},ok}}]}, + + {?eh,test_stats,{4,0,{2,0}}}, + {?eh,tc_start,{groups_12_SUITE, + {end_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done,{groups_12_SUITE, + {end_per_group,test_group_5,[parallel]},ok}}]}, - {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}, + {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_4,[]},ok}}], + + {?eh,tc_start,{groups_12_SUITE,end_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,init}}, {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} ]; @@ -1066,17 +1140,26 @@ test_events(subgroup_all_testcases) -> test_events(skip_subgroup_all_testcases) -> [ {?eh,start_logging,'_'}, + {?eh,start_info,{1,1,6}}, {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, - - [{?eh,tc_start, - {groups_12_SUITE,{init_per_group,test_group_4,[]}}}, - {?eh,tc_done, - {groups_12_SUITE,{init_per_group,test_group_4,[]},ok}}, - {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_5},"SKIPPED!"}}, - {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}, - {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_4,[]},ok}} - ], - + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_4,[]},ok}}, + {?eh,tc_user_skip,{groups_12_SUITE, + {init_per_group,test_group_5},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_5a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_7a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_7b,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_8a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_8b,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_5b,"SKIPPED!"}}, + {?eh,test_stats,{0,0,{6,0}}}, + {?eh,tc_user_skip,{groups_12_SUITE, + {end_per_group,test_group_5},"SKIPPED!"}}, + {?eh,tc_start,{groups_12_SUITE, + {end_per_group,test_group_4,[]}}}, + {?eh,tc_done,{groups_12_SUITE, + {end_per_group,test_group_4,[]},ok}}], + {?eh,tc_start,{groups_12_SUITE,end_per_suite}}, {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} ]; @@ -1194,13 +1277,30 @@ test_events(skip_subgroup_testcase) -> test_events(sub_skipped_by_top) -> [ {?eh,start_logging,'_'}, + {?eh,start_info,{1,1,12}}, {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, - - {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_4},"SKIPPED!"}}, - {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_4},"SKIPPED!"}}, - + {?eh,tc_user_skip,{groups_12_SUITE,{init_per_group,test_group_4}, + "SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_5a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_7a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_7b,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_8a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_8b,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_5b,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE, + {end_per_group,test_group_4},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE, + {init_per_group,test_group_4},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_5a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_7a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_7b,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_8a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_8b,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_5b,"SKIPPED!"}}, + {?eh,test_stats,{0,0,{12,0}}}, + {?eh,tc_user_skip,{groups_12_SUITE, + {end_per_group,test_group_4},"SKIPPED!"}}, {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, - {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} ]; -- cgit v1.2.3