diff options
author | Peter Andersson <[email protected]> | 2013-11-07 00:50:37 +0100 |
---|---|---|
committer | Peter Andersson <[email protected]> | 2013-11-18 18:05:50 +0100 |
commit | f34f567125c86a9f12bff473e7a7e2fd4b9a0b3f (patch) | |
tree | aff89e8c5edb28d5e0453d7d1ba64dc22a16af04 | |
parent | 02a0023b265bba5518b60b524d64fa8056fee911 (diff) | |
download | otp-f34f567125c86a9f12bff473e7a7e2fd4b9a0b3f.tar.gz otp-f34f567125c86a9f12bff473e7a7e2fd4b9a0b3f.tar.bz2 otp-f34f567125c86a9f12bff473e7a7e2fd4b9a0b3f.zip |
Correct tests cases that fail because of modified events
-rw-r--r-- | lib/common_test/src/ct_framework.erl | 30 | ||||
-rw-r--r-- | lib/common_test/test/ct_config_SUITE.erl | 34 | ||||
-rw-r--r-- | lib/common_test/test/ct_error_SUITE.erl | 18 | ||||
-rw-r--r-- | lib/common_test/test/ct_group_info_SUITE.erl | 35 | ||||
-rw-r--r-- | lib/common_test/test/ct_groups_test_2_SUITE.erl | 12 | ||||
-rw-r--r-- | lib/common_test/test/ct_hooks_SUITE.erl | 26 | ||||
-rw-r--r-- | lib/common_test/test/ct_repeat_testrun_SUITE.erl | 14 | ||||
-rw-r--r-- | lib/common_test/test/ct_skip_SUITE.erl | 102 | ||||
-rw-r--r-- | lib/common_test/test/ct_surefire_SUITE.erl | 4 | ||||
-rw-r--r-- | lib/common_test/test/ct_test_server_if_1_SUITE.erl | 70 | ||||
-rw-r--r-- | lib/common_test/test/ct_test_support.erl | 30 | ||||
-rw-r--r-- | lib/test_server/src/test_server.erl | 2 | ||||
-rw-r--r-- | lib/test_server/src/test_server_ctrl.erl | 55 |
13 files changed, 271 insertions, 161 deletions
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index b72ecba809..15809637fd 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -103,7 +103,7 @@ init_tc(Mod,Func,Config) -> end, init_tc1(Mod,Suite,Func,Config); {failed,Seq,BadFunc} -> - {fail,{sequence_failed,Seq,BadFunc}} + {auto_skip,{sequence_failed,Seq,BadFunc}} end end end. @@ -691,7 +691,7 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) -> ct_util:update_testdata(curr_tc, ClearCurrTC), case FinalResult of - {skip,{sequence_failed,_,_}} -> + {auto_skip,{sequence_failed,_,_}} -> %% ct_logs:init_tc is never called for a skipped test case %% in a failing sequence, so neither should end_tc ok; @@ -714,8 +714,13 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) -> %% {error,Reason} | {skip,Reason} | {timetrap_timeout,TVal} | %% {testcase_aborted,Reason} | testcase_aborted_or_killed | %% {'EXIT',Reason} | Other (ignored return value, e.g. 'ok') -tag({STag,Reason}) when STag == skip; STag == skipped -> - {skipped,Reason}; +tag({STag,Reason}) when STag == skip; STag == skipped -> + case Reason of + {failed,{_,init_per_testcase,_}} -> {auto_skipped,Reason}; + _ -> {skipped,Reason} + end; +tag({auto_skip,Reason}) -> + {auto_skipped,Reason}; tag(E = {ETag,_}) when ETag == error; ETag == 'EXIT'; ETag == timetrap_timeout; ETag == testcase_aborted -> @@ -725,13 +730,20 @@ tag(E = testcase_aborted_or_killed) -> tag(Other) -> Other. +tag_cth({skipped,Reason={failed,{_,init_per_testcase,_}}}) -> + {auto_skipped,Reason}; tag_cth({STag,Reason}) when STag == skip; STag == skipped -> - {skipped,Reason}; -tag_cth({fail, Reason}) -> - {failed, {error,Reason}}; + case Reason of + {failed,{_,init_per_testcase,_}} -> {auto_skipped,Reason}; + _ -> {skipped,Reason} + end; +tag_cth({auto_skip,Reason}) -> + {auto_skipped,Reason}; +tag_cth({fail,Reason}) -> + {failed,{error,Reason}}; tag_cth(E = {ETag,_}) when ETag == error; ETag == 'EXIT'; - ETag == timetrap_timeout; - ETag == testcase_aborted -> + ETag == timetrap_timeout; + ETag == testcase_aborted -> {failed,E}; tag_cth(E = testcase_aborted_or_killed) -> {failed,E}; diff --git a/lib/common_test/test/ct_config_SUITE.erl b/lib/common_test/test/ct_config_SUITE.erl index d92be9ec6e..284f836517 100644 --- a/lib/common_test/test/ct_config_SUITE.erl +++ b/lib/common_test/test/ct_config_SUITE.erl @@ -234,25 +234,25 @@ expected_events(config_static_SUITE)-> ?sok(test_get_config_deep_nested,{3,0,{0,0}}), ?sok(test_default_suitewide,{4,0,{0,0}}), ?snok(test_config_name_already_in_use1, - {skipped,{config_name_already_in_use,[x1]}},{4,0,{1,0}}), - ?sok(test_default_tclocal,{5,0,{1,0}}), + {failed,{error,{config_name_already_in_use,[x1]}}},{4,1,{0,0}}), + ?sok(test_default_tclocal,{5,1,{0,0}}), ?snok(test_config_name_already_in_use2, - {skipped,{config_name_already_in_use,[alias,x1]}},{5,0,{2,0}}), - ?sok(test_alias_tclocal,{6,0,{2,0}}), - ?sok(test_get_config_undefined,{7,0,{2,0}}), - ?sok(test_require_subvals,{8,0,{2,0}}), + {failed,{error,{config_name_already_in_use,[alias,x1]}}},{5,2,{0,0}}), + ?sok(test_alias_tclocal,{6,2,{0,0}}), + ?sok(test_get_config_undefined,{7,2,{0,0}}), + ?sok(test_require_subvals,{8,2,{0,0}}), ?snok(test_require_subvals2, - {skipped,{require_failed, - {not_available,{gen_cfg,[a,b,c,d]}}}},{8,0,{2,1}}), - ?sok(test_require_deep_config,{9,0,{2,1}}), - ?sok(test_shadow_all,{10,0,{2,1}}), - ?sok(test_element,{11,0,{2,1}}), - ?sok(test_shadow_all_element,{12,0,{2,1}}), - ?sok(test_internal_deep,{13,0,{2,1}}), - ?sok(test_alias_tclocal_nested,{14,0,{2,1}}), - ?sok(test_alias_tclocal_nested_backward_compat,{15,0,{2,1}}), - ?sok(test_alias_tclocal_nested_backward_compat_subvals,{16,0,{2,1}}), - ?sok(test_config_same_name_already_in_use,{17,0,{2,1}}), + {auto_skipped,{require_failed, + {not_available,{gen_cfg,[a,b,c,d]}}}},{8,2,{0,1}}), + ?sok(test_require_deep_config,{9,2,{0,1}}), + ?sok(test_shadow_all,{10,2,{0,1}}), + ?sok(test_element,{11,2,{0,1}}), + ?sok(test_shadow_all_element,{12,2,{0,1}}), + ?sok(test_internal_deep,{13,2,{0,1}}), + ?sok(test_alias_tclocal_nested,{14,2,{0,1}}), + ?sok(test_alias_tclocal_nested_backward_compat,{15,2,{0,1}}), + ?sok(test_alias_tclocal_nested_backward_compat_subvals,{16,2,{0,1}}), + ?sok(test_config_same_name_already_in_use,{17,2,{0,1}}), {?eh,tc_start,{config_static_SUITE,end_per_suite}}, {?eh,tc_done,{config_static_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index 3881ced17d..a59385c53b 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -622,41 +622,41 @@ test_events(cfg_error) -> {?eh,tc_done,{cfg_error_11_SUITE,init_per_suite,ok}}, {?eh,tc_start,{cfg_error_11_SUITE,tc1}}, {?eh,tc_done, {cfg_error_11_SUITE,tc1, - {skipped,{config_name_already_in_use,[dummy_alias]}}}}, - {?eh,test_stats,{12,6,{1,19}}}, + {failed,{error,{config_name_already_in_use,[dummy_alias]}}}}}, + {?eh,test_stats,{12,7,{0,19}}}, {?eh,tc_start,{cfg_error_11_SUITE,tc2}}, {?eh,tc_done,{cfg_error_11_SUITE,tc2,ok}}, - {?eh,test_stats,{13,6,{1,19}}}, + {?eh,test_stats,{13,7,{0,19}}}, {?eh,tc_start,{cfg_error_11_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_11_SUITE,end_per_suite,ok}}, {?eh,tc_start,{cfg_error_12_SUITE,tc1}}, {?eh,tc_done,{ct_framework,init_tc,{framework_error,{timetrap,500}}}}, - {?eh,test_stats,{13,7,{1,19}}}, + {?eh,test_stats,{13,8,{0,19}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc2}}, {?eh,tc_done,{cfg_error_12_SUITE,tc2,{failed, {cfg_error_12_SUITE,end_per_testcase, {timetrap_timeout,500}}}}}, - {?eh,test_stats,{14,7,{1,19}}}, + {?eh,test_stats,{14,8,{0,19}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc3}}, {?eh,tc_done,{cfg_error_12_SUITE,tc3,ok}}, - {?eh,test_stats,{15,7,{1,19}}}, + {?eh,test_stats,{15,8,{0,19}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc4}}, {?eh,tc_done,{cfg_error_12_SUITE,tc4,{failed, {cfg_error_12_SUITE,end_per_testcase, {timetrap_timeout,500}}}}}, - {?eh,test_stats,{16,7,{1,19}}}, + {?eh,test_stats,{16,8,{0,19}}}, {?eh,tc_start,{cfg_error_13_SUITE,init_per_suite}}, {?eh,tc_done,{cfg_error_13_SUITE,init_per_suite,ok}}, {?eh,tc_start,{cfg_error_13_SUITE,tc1}}, {?eh,tc_done,{cfg_error_13_SUITE,tc1,ok}}, - {?eh,test_stats,{17,7,{1,19}}}, + {?eh,test_stats,{17,8,{0,19}}}, {?eh,tc_start,{cfg_error_13_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_13_SUITE,end_per_suite,ok}}, {?eh,tc_start,{cfg_error_14_SUITE,init_per_suite}}, {?eh,tc_done,{cfg_error_14_SUITE,init_per_suite,ok}}, {?eh,tc_start,{cfg_error_14_SUITE,tc1}}, {?eh,tc_done,{cfg_error_14_SUITE,tc1,ok}}, - {?eh,test_stats,{18,7,{1,19}}}, + {?eh,test_stats,{18,8,{0,19}}}, {?eh,tc_start,{cfg_error_14_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_14_SUITE,end_per_suite, {comment, diff --git a/lib/common_test/test/ct_group_info_SUITE.erl b/lib/common_test/test/ct_group_info_SUITE.erl index c56fa952e8..c715142ff5 100644 --- a/lib/common_test/test/ct_group_info_SUITE.erl +++ b/lib/common_test/test/ct_group_info_SUITE.erl @@ -539,9 +539,11 @@ test_events(require) -> [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g4,[]}}}, {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g4,[]}, - {skipped,{require_failed,{name_in_use,common2_alias,common2}}}}}, + {auto_skipped,{require_failed, + {name_in_use,common2_alias,common2}}}}}, {?eh,tc_auto_skip,{group_require_1_SUITE,t41, - {require_failed,{name_in_use,common2_alias,common2}}}}, + {require_failed, + {name_in_use,common2_alias,common2}}}}, {?eh,test_stats,{4,0,{0,1}}}, {?eh,tc_auto_skip,{group_require_1_SUITE,end_per_group, {require_failed,{name_in_use,common2_alias,common2}}}}], @@ -566,7 +568,8 @@ test_events(require) -> [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g8,[]}}}, {?eh,tc_done,{group_require_1_SUITE, {init_per_group,g8,[]}, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,tc_auto_skip,{group_require_1_SUITE,t81, {require_failed,{not_available,non_existing}}}}, {?eh,test_stats,{8,0,{0,2}}}, @@ -575,7 +578,8 @@ test_events(require) -> [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g9,[]}}}, {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g9,[]},ok}}, {?eh,tc_done,{group_require_1_SUITE,t91, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,test_stats,{8,0,{0,3}}}, {?eh,tc_start,{group_require_1_SUITE,{end_per_group,g9,[]}}}, {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g9,[]},ok}}], @@ -627,7 +631,7 @@ test_events(require_default) -> [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g4,[]}}}, {?eh,tc_done,{group_require_1_SUITE, {init_per_group,g4,[]}, - {skipped,{require_failed,{not_available,common3}}}}}, + {auto_skipped,{require_failed,{not_available,common3}}}}}, {?eh,tc_auto_skip,{group_require_1_SUITE,t41, {require_failed,{not_available,common3}}}}, {?eh,test_stats,{4,0,{0,1}}}, @@ -654,7 +658,8 @@ test_events(require_default) -> [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g8,[]}}}, {?eh,tc_done,{group_require_1_SUITE, {init_per_group,g8,[]}, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,tc_auto_skip,{group_require_1_SUITE,t81, {require_failed,{not_available,non_existing}}}}, {?eh,test_stats,{8,0,{0,2}}}, @@ -664,7 +669,8 @@ test_events(require_default) -> [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g9,[]}}}, {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g9,[]},ok}}, {?eh,tc_done,{group_require_1_SUITE,t91, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,test_stats,{8,0,{0,3}}}, {?eh,tc_start,{group_require_1_SUITE,{end_per_group,g9,[]}}}, {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g9,[]},ok}}], @@ -714,7 +720,8 @@ test_events(require_no_ips) -> [{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g4,[]}}}, {?eh,tc_done,{group_require_2_SUITE,{init_per_group,g4,[]}, - {skipped,{require_failed,{name_in_use,common2_alias,common2}}}}}, + {auto_skipped,{require_failed, + {name_in_use,common2_alias,common2}}}}}, {?eh,tc_auto_skip,{group_require_2_SUITE,t41, {require_failed,{name_in_use,common2_alias,common2}}}}, {?eh,test_stats,{4,0,{0,1}}}, @@ -741,7 +748,8 @@ test_events(require_no_ips) -> [{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g8,[]}}}, {?eh,tc_done,{group_require_2_SUITE, {init_per_group,g8,[]}, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,tc_auto_skip,{group_require_2_SUITE,t81, {require_failed,{not_available,non_existing}}}}, {?eh,test_stats,{8,0,{0,2}}}, @@ -750,7 +758,8 @@ test_events(require_no_ips) -> [{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g9,[]}}}, {?eh,tc_done,{group_require_2_SUITE,{init_per_group,g9,[]},ok}}, {?eh,tc_done,{group_require_2_SUITE,t91, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,test_stats,{8,0,{0,3}}}, {?eh,tc_start,{group_require_2_SUITE,{end_per_group,g9,[]}}}, {?eh,tc_done,{group_require_2_SUITE,{end_per_group,g9,[]},ok}}], @@ -799,7 +808,7 @@ test_events(require_no_ipg) -> [{?eh,tc_start,{ct_framework,{init_per_group,g4,[{suite,group_require_3_SUITE}]}}}, {?eh,tc_done,{ct_framework,{init_per_group,g4,[{suite,group_require_3_SUITE}]}, - {skipped,{require_failed,{name_in_use,common2_alias,common2}}}}}, + {auto_skipped,{require_failed,{name_in_use,common2_alias,common2}}}}}, {?eh,tc_auto_skip,{group_require_3_SUITE,t41, {require_failed,{name_in_use,common2_alias,common2}}}}, {?eh,test_stats,{4,0,{0,1}}}, @@ -825,7 +834,7 @@ test_events(require_no_ipg) -> [{?eh,tc_start,{ct_framework,{init_per_group,g8,[{suite,group_require_3_SUITE}]}}}, {?eh,tc_done,{ct_framework,{init_per_group,g8,[{suite,group_require_3_SUITE}]}, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed,{not_available,non_existing}}}}}, {?eh,tc_auto_skip,{group_require_3_SUITE,t81, {require_failed,{not_available,non_existing}}}}, {?eh,test_stats,{8,0,{0,2}}}, @@ -834,7 +843,7 @@ test_events(require_no_ipg) -> [{?eh,tc_start,{ct_framework,{init_per_group,g9,[{suite,group_require_3_SUITE}]}}}, {?eh,tc_done,{ct_framework,{init_per_group,g9,[{suite,group_require_3_SUITE}]},ok}}, {?eh,tc_done,{group_require_3_SUITE,t91, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed,{not_available,non_existing}}}}}, {?eh,test_stats,{8,0,{0,3}}}, {?eh,tc_start,{ct_framework,{end_per_group,g9,[{suite,group_require_3_SUITE}]}}}, {?eh,tc_done,{ct_framework,{end_per_group,g9,[{suite,group_require_3_SUITE}]},ok}}], diff --git a/lib/common_test/test/ct_groups_test_2_SUITE.erl b/lib/common_test/test/ct_groups_test_2_SUITE.erl index 1b2ad12e2f..8b0de98709 100644 --- a/lib/common_test/test/ct_groups_test_2_SUITE.erl +++ b/lib/common_test/test/ct_groups_test_2_SUITE.erl @@ -171,16 +171,20 @@ test_events(missing_conf) -> {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, {?eh,start_info,{1,1,2}}, - {?eh,tc_start,{ct_framework,{init_per_group,group1,[]}}}, - {?eh,tc_done,{ct_framework,{init_per_group,group1,[]},ok}}, + {?eh,tc_start,{ct_framework,{init_per_group,group1, + [{suite,missing_conf_SUITE}]}}}, + {?eh,tc_done,{ct_framework,{init_per_group,group1, + [{suite,missing_conf_SUITE}]},ok}}, {?eh,tc_start,{missing_conf_SUITE,tc1}}, {?eh,tc_done,{missing_conf_SUITE,tc1,ok}}, {?eh,test_stats,{1,0,{0,0}}}, {?eh,tc_start,{missing_conf_SUITE,tc2}}, {?eh,tc_done,{missing_conf_SUITE,tc2,ok}}, {?eh,test_stats,{2,0,{0,0}}}, - {?eh,tc_start,{ct_framework,{end_per_group,group1,[]}}}, - {?eh,tc_done,{ct_framework,{end_per_group,group1,[]},ok}}, + {?eh,tc_start,{ct_framework,{end_per_group,group1, + [{suite,missing_conf_SUITE}]}}}, + {?eh,tc_done,{ct_framework,{end_per_group,group1, + [{suite,missing_conf_SUITE}]},ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} ]; diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl index 596bfe3ff0..083c87b49e 100644 --- a/lib/common_test/test/ct_hooks_SUITE.erl +++ b/lib/common_test/test/ct_hooks_SUITE.erl @@ -749,11 +749,10 @@ test_events(skip_pre_suite_cth) -> {?eh,cth,{'_',on_tc_skip, [init_per_suite,{tc_user_skip,{skipped,"Test skip"}},[]]}}, - {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, - {?eh,cth,{'_',on_tc_skip,[test_case,{tc_auto_skip,"Test skip"},[]]}}, + {?eh,tc_user_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, + {?eh,cth,{'_',on_tc_skip,[test_case,{tc_user_skip,"Test skip"},[]]}}, - {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, - {?eh,cth,{'_',on_tc_skip,[end_per_suite,{tc_auto_skip,"Test skip"},[]]}}, + {?eh,tc_user_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,cth, {'_',terminate,[[]]}}, @@ -773,11 +772,10 @@ test_events(skip_post_suite_cth) -> {?eh,cth,{'_',on_tc_skip, [init_per_suite,{tc_user_skip,{skipped,"Test skip"}},[]]}}, - {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, - {?eh,cth,{'_',on_tc_skip,[test_case,{tc_auto_skip,"Test skip"},[]]}}, + {?eh,tc_user_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, + {?eh,cth,{'_',on_tc_skip,[test_case,{tc_user_skip,"Test skip"},[]]}}, - {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, - {?eh,cth,{'_',on_tc_skip,[end_per_suite,{tc_auto_skip,"Test skip"},[]]}}, + {?eh,tc_user_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,cth,{'_',terminate,[[]]}}, @@ -1219,11 +1217,15 @@ test_events(cth_log) -> {?eh,tc_start,{cth_log_SUITE,init_per_suite}}, {parallel, - [{?eh,tc_start,{ct_framework,{init_per_group,g1,[parallel]}}}, - {?eh,tc_done,{ct_framework,{init_per_group,g1,[parallel]},ok}}, + [{?eh,tc_start,{ct_framework,{init_per_group,g1, + [{suite,cth_log_SUITE},parallel]}}}, + {?eh,tc_done,{ct_framework,{init_per_group,g1, + [{suite,cth_log_SUITE},parallel]},ok}}, {?eh,test_stats,{30,0,{0,0}}}, - {?eh,tc_start,{ct_framework,{end_per_group,g1,[parallel]}}}, - {?eh,tc_done,{ct_framework,{end_per_group,g1,[parallel]},ok}}]}, + {?eh,tc_start,{ct_framework,{end_per_group,g1, + [{suite,cth_log_SUITE},parallel]}}}, + {?eh,tc_done,{ct_framework,{end_per_group,g1, + [{suite,cth_log_SUITE},parallel]},ok}}]}, {?eh,tc_done,{cth_log_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, diff --git a/lib/common_test/test/ct_repeat_testrun_SUITE.erl b/lib/common_test/test/ct_repeat_testrun_SUITE.erl index 35d67a10f2..9d5216449b 100644 --- a/lib/common_test/test/ct_repeat_testrun_SUITE.erl +++ b/lib/common_test/test/ct_repeat_testrun_SUITE.erl @@ -78,7 +78,7 @@ init_per_suite(Config0) -> {1,0,{0,0}} = ct_test_support:run(ct,run_test,[Opts2],Config), %% Time the shortest testcase to use for offset - {T0,{1,0,{0,0}}} = timer:tc(ct_test_support,run,[ct,run_test,[Opts1],Config]), + {_T0,{1,0,{0,0}}} = timer:tc(ct_test_support,run,[ct,run_test,[Opts1],Config]), %% -2 is to ensure we hit inside the target test case and not after % T = round(T0/1000000)-2, @@ -343,13 +343,13 @@ skip_first_tc1(Suite) -> {?eh,tc_done,{Suite,tc2,?skipped}}, {?eh,test_stats,{'_',0,{1,0}}}, {?eh,tc_done,{Suite,{init_per_group,g,[]},?skipped}}, - {?eh,tc_auto_skip,{Suite,tc1,?skip_reason}}, - {?eh,test_stats,{'_',0,{1,1}}}, - {?eh,tc_auto_skip,{Suite,tc2,?skip_reason}}, - {?eh,test_stats,{'_',0,{1,2}}}, - {?eh,tc_auto_skip,{Suite,end_per_group,?skip_reason}}, + {?eh,tc_user_skip,{Suite,tc1,?skip_reason}}, + {?eh,test_stats,{'_',0,{2,0}}}, + {?eh,tc_user_skip,{Suite,tc2,?skip_reason}}, + {?eh,test_stats,{'_',0,{3,0}}}, + {?eh,tc_user_skip,{Suite,end_per_group,?skip_reason}}, {?eh,tc_done,{Suite,tc2,?skipped}}, - {?eh,test_stats,{'_',0,{2,2}}}, + {?eh,test_stats,{'_',0,{4,0}}}, {?eh,tc_start,{Suite,end_per_suite}}, {?eh,tc_done,{Suite,end_per_suite,ok}}]. diff --git a/lib/common_test/test/ct_skip_SUITE.erl b/lib/common_test/test/ct_skip_SUITE.erl index 6f6ff754aa..f7cdc90a4d 100644 --- a/lib/common_test/test/ct_skip_SUITE.erl +++ b/lib/common_test/test/ct_skip_SUITE.erl @@ -164,7 +164,7 @@ test_events(auto_skip) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, - {?eh,start_info,{11,11,34}}, + {?eh,start_info,{12,12,43}}, {?eh,tc_start,{auto_skip_1_SUITE,init_per_suite}}, {?eh,tc_done, @@ -384,26 +384,26 @@ test_events(auto_skip) -> {?eh,tc_start,{auto_skip_10_SUITE,init_per_suite}}, {?eh,tc_done,{auto_skip_10_SUITE,init_per_suite, - {failed,{error, - {require_failed_in_suite0, - {not_available,undefined_config_variable}}}}}}, - {?eh,tc_auto_skip, - {auto_skip_10_SUITE,tc1, - {require_failed_in_suite0,{not_available,undefined_config_variable}}}}, + {auto_skipped,{require_failed_in_suite0, + {not_available,undefined_config_variable}}}}}, + {?eh,tc_auto_skip,{auto_skip_10_SUITE,tc1, + {require_failed_in_suite0, + {not_available,undefined_config_variable}}}}, {?eh,test_stats,{9,0,{0,21}}}, - {?eh,tc_auto_skip, - {auto_skip_10_SUITE,tc2, - {require_failed_in_suite0,{not_available,undefined_config_variable}}}}, + {?eh,tc_auto_skip,{auto_skip_10_SUITE,tc2, + {require_failed_in_suite0, + {not_available,undefined_config_variable}}}}, {?eh,test_stats,{9,0,{0,22}}}, - {?eh,tc_auto_skip, - {auto_skip_10_SUITE,end_per_suite, - {require_failed_in_suite0,{not_available,undefined_config_variable}}}}, + {?eh,tc_auto_skip,{auto_skip_10_SUITE,end_per_suite, + {require_failed_in_suite0, + {not_available,undefined_config_variable}}}}, {?eh,tc_start,{auto_skip_11_SUITE,init_per_suite}}, {?eh,tc_done,{auto_skip_11_SUITE,init_per_suite,ok}}, {?eh,tc_start,{auto_skip_11_SUITE,tc1}}, {?eh,tc_done,{auto_skip_11_SUITE,tc1, - {skipped,{require_failed,{not_available,undefined_config_variable}}}}}, + {auto_skipped,{require_failed, + {not_available,undefined_config_variable}}}}}, {?eh,test_stats,{9,0,{0,23}}}, {?eh,tc_start,{auto_skip_11_SUITE,tc2}}, {?eh,tc_done,{auto_skip_11_SUITE,tc2,ok}}, @@ -422,7 +422,8 @@ test_events(auto_skip) -> {?eh,tc_start,{auto_skip_11_SUITE,tc3}}, {?eh,tc_done, {auto_skip_11_SUITE,tc3, - {skipped,{require_failed,{not_available,undefined_config_variable}}}}}, + {auto_skipped,{require_failed, + {not_available,undefined_config_variable}}}}}, {?eh,test_stats,{10,0,{0,24}}}, {?eh,tc_start, {auto_skip_11_SUITE,{end_per_group,g2,[parallel]}}}, @@ -435,8 +436,75 @@ test_events(auto_skip) -> {?eh,tc_start,{auto_skip_11_SUITE,end_per_suite}}, {?eh,tc_done,{auto_skip_11_SUITE,end_per_suite,ok}}, - - %%! HERE: auto_skip_12_SUITE terms!! + + {?eh,tc_start,{auto_skip_12_SUITE,init_per_suite}}, + {?eh,tc_done,{auto_skip_12_SUITE,init_per_suite,ok}}, + + [{?eh,tc_start,{ct_framework,{init_per_group,g1, + [{suite,auto_skip_12_SUITE}]}}}, + {?eh,tc_done,{ct_framework,{init_per_group,g1, + [{suite,auto_skip_12_SUITE}]}, + {auto_skipped, + {require_failed,{not_available,unknown_variable_g1}}}}}, + {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc1, + {require_failed,{not_available,unknown_variable_g1}}}}, + {?eh,test_stats,{10,0,{0,25}}}, + {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc2, + {require_failed,{not_available,unknown_variable_g1}}}}, + {?eh,test_stats,{10,0,{0,26}}}, + {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3, + {require_failed,{not_available,unknown_variable_g1}}}}, + {?eh,test_stats,{10,0,{0,27}}}, + {?eh,tc_auto_skip,{ct_framework,end_per_group, + {require_failed,{not_available,unknown_variable_g1}}}}], + + [{?eh,tc_start,{ct_framework,{init_per_group,g1, + [{suite,auto_skip_12_SUITE}]}}}, + {?eh,tc_done,{ct_framework,{init_per_group,g1, + [{suite,auto_skip_12_SUITE}]}, + {auto_skipped, + {require_failed,{not_available,unknown_variable_g1}}}}}, + {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc1, + {require_failed,{not_available,unknown_variable_g1}}}}, + {?eh,test_stats,{10,0,{0,28}}}, + {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc2, + {require_failed,{not_available,unknown_variable_g1}}}}, + {?eh,test_stats,{10,0,{0,29}}}, + {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3, + {require_failed,{not_available,unknown_variable_g1}}}}, + {?eh,test_stats,{10,0,{0,30}}}, + {?eh,tc_auto_skip,{ct_framework,end_per_group, + {require_failed,{not_available,unknown_variable_g1}}}}], + + [{?eh,tc_start,{ct_framework,{init_per_group,g3, + [{suite,auto_skip_12_SUITE}]}}}, + {?eh,tc_done,{ct_framework,{init_per_group,g3, + [{suite,auto_skip_12_SUITE}]},ok}}, + {?eh,tc_start,{auto_skip_12_SUITE,tc1}}, + {?eh,tc_done,{auto_skip_12_SUITE,tc1,ok}}, + {?eh,test_stats,{11,0,{0,30}}}, + {?eh,tc_start,{auto_skip_12_SUITE,tc2}}, + {?eh,tc_done,{auto_skip_12_SUITE,tc2,ok}}, + {?eh,test_stats,{12,0,{0,30}}}, + [{?eh,tc_start,{ct_framework,{init_per_group,g4, + [{suite,auto_skip_12_SUITE}]}}}, + {?eh,tc_done,{ct_framework,{init_per_group,g4, + [{suite,auto_skip_12_SUITE}]}, + {auto_skipped, + {require_failed,{not_available,unknown_variable_g4}}}}}, + {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3, + {require_failed,{not_available,unknown_variable_g4}}}}, + {?eh,test_stats,{12,0,{0,31}}}, + {?eh,tc_auto_skip,{ct_framework,end_per_group, + {require_failed,{not_available,unknown_variable_g4}}}}], + + {?eh,tc_start,{ct_framework,{end_per_group,g3, + [{suite,auto_skip_12_SUITE}]}}}, + {?eh,tc_done,{ct_framework,{end_per_group,g3, + [{suite,auto_skip_12_SUITE}]},ok}}], + + {?eh,tc_start,{auto_skip_12_SUITE,end_per_suite}}, + {?eh,tc_done,{auto_skip_12_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} diff --git a/lib/common_test/test/ct_surefire_SUITE.erl b/lib/common_test/test/ct_surefire_SUITE.erl index b86b47f0a2..1a370de44a 100644 --- a/lib/common_test/test/ct_surefire_SUITE.erl +++ b/lib/common_test/test/ct_surefire_SUITE.erl @@ -182,7 +182,7 @@ test_events(_) -> {?eh,test_stats,{1,1,{1,0}}}, {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, - {skipped,{require_failed,'_'}}}}, + {auto_skipped,{require_failed,'_'}}}}, {?eh,test_stats,{1,1,{1,1}}}, [{?eh,tc_start,{surefire_SUITE,{init_per_group,g,[]}}}, {?eh,tc_done,{surefire_SUITE,{init_per_group,g,[]},ok}}, @@ -198,7 +198,7 @@ test_events(_) -> {?eh,test_stats,{2,2,{2,1}}}, {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, - {skipped,{require_failed,'_'}}}}, + {auto_skipped,{require_failed,'_'}}}}, {?eh,test_stats,{2,2,{2,2}}}, {?eh,tc_start,{surefire_SUITE,{end_per_group,g,[]}}}, {?eh,tc_done,{surefire_SUITE,{end_per_group,g,[]},ok}}], diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE.erl index 8e4852369d..64957a76c9 100644 --- a/lib/common_test/test/ct_test_server_if_1_SUITE.erl +++ b/lib/common_test/test/ct_test_server_if_1_SUITE.erl @@ -144,7 +144,7 @@ test_events(ts_if_1) -> {?eh,tc_start,{ts_if_1_SUITE,init_per_suite}}, {?eh,tc_done,{ts_if_1_SUITE,init_per_suite,ok}}, {?eh,tc_start,{ts_if_1_SUITE,tc1}}, - {?eh,tc_done,{ts_if_1_SUITE,tc1,{skipped, + {?eh,tc_done,{ts_if_1_SUITE,tc1,{auto_skipped, {failed, {ts_if_1_SUITE,init_per_testcase, {timetrap_timeout,2000}}}}}}, @@ -159,62 +159,65 @@ test_events(ts_if_1) -> {?eh,tc_start,{ts_if_1_SUITE,tc4}}, {?eh,tc_done,{ts_if_1_SUITE,tc4,{failed,{error,failed_on_purpose}}}}, {?eh,test_stats,{1,2,{0,1}}}, - {?eh,tc_done,{ts_if_1_SUITE,tc5,{skipped,{sequence_failed,seq1,tc4}}}}, - {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_done,{ts_if_1_SUITE,tc5,{auto_skipped,{sequence_failed,seq1,tc4}}}}, + {?eh,test_stats,{1,2,{0,2}}}, [{?eh,tc_start,{ts_if_1_SUITE,{init_per_group,seq2,[sequence]}}}, {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,seq2,[sequence]},ok}}, {?eh,tc_start,{ts_if_1_SUITE,tc4}}, {?eh,tc_done,{ts_if_1_SUITE,tc4,{failed,{error,failed_on_purpose}}}}, - {?eh,test_stats,{1,3,{1,1}}}, + {?eh,test_stats,{1,3,{0,2}}}, {?eh,tc_auto_skip,{ts_if_1_SUITE,tc5,{failed,{ts_if_1_SUITE,tc4}}}}, - {?eh,test_stats,{1,3,{1,2}}}, + {?eh,test_stats,{1,3,{0,3}}}, {?eh,tc_start,{ts_if_1_SUITE,{end_per_group,seq2,[sequence]}}}, {?eh,tc_done,{ts_if_1_SUITE,{end_per_group,seq2,[sequence]},ok}}], {?eh,tc_start,{ts_if_1_SUITE,tc6}}, - {?eh,tc_done,{ts_if_1_SUITE,tc6,{skipped,{require_failed,{not_available,void}}}}}, - {?eh,test_stats,{1,3,{1,3}}}, + {?eh,tc_done,{ts_if_1_SUITE,tc6,{auto_skipped, + {require_failed,{not_available,void}}}}}, + {?eh,test_stats,{1,3,{0,4}}}, {?eh,tc_start,{ts_if_1_SUITE,tc7}}, {?eh,tc_done,{ts_if_1_SUITE,tc7,ok}}, - {?eh,test_stats,{2,3,{1,3}}}, + {?eh,test_stats,{2,3,{0,4}}}, {?eh,tc_start,{ts_if_1_SUITE,tc8}}, {?eh,tc_done,{ts_if_1_SUITE,tc8,{skipped,"tc8 skipped"}}}, - {?eh,test_stats,{2,3,{2,3}}}, + {?eh,test_stats,{2,3,{1,4}}}, {?eh,tc_start,{ts_if_1_SUITE,tc9}}, {?eh,tc_done,{ts_if_1_SUITE,tc9,{skipped,'tc9 skipped'}}}, - {?eh,test_stats,{2,3,{3,3}}}, + {?eh,test_stats,{2,3,{2,4}}}, {?eh,tc_start,{ts_if_1_SUITE,tc10}}, {?eh,tc_done,{ts_if_1_SUITE,tc10,{failed,{error,{function_clause,'_'}}}}}, - {?eh,test_stats,{2,4,{3,3}}}, + {?eh,test_stats,{2,4,{2,4}}}, {?eh,tc_start,{ts_if_1_SUITE,tc11}}, {?eh,tc_done,{ts_if_1_SUITE,tc11, - {skipped,{failed,{ts_if_1_SUITE,init_per_testcase,bad_return}}}}}, - {?eh,test_stats,{2,4,{3,4}}}, + {auto_skipped,{failed,{ts_if_1_SUITE,init_per_testcase,bad_return}}}}}, + {?eh,test_stats,{2,4,{2,5}}}, [{?eh,tc_start,{ts_if_1_SUITE,{init_per_group,g1,[]}}}, - {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g1,[]},{skipped,g1_got_skipped}}}, - {?eh,tc_auto_skip,{ts_if_1_SUITE,gtc1,g1_got_skipped}}, + {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g1,[]}, + {skipped,g1_got_skipped}}}, + {?eh,tc_user_skip,{ts_if_1_SUITE,gtc1,g1_got_skipped}}, {?eh,test_stats,{2,4,{3,5}}}, - {?eh,tc_auto_skip,{ts_if_1_SUITE,end_per_group,g1_got_skipped}}], + {?eh,tc_user_skip,{ts_if_1_SUITE,end_per_group,g1_got_skipped}}], {parallel, [{?eh,tc_start,{ts_if_1_SUITE,{init_per_group,g2,[parallel]}}}, {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g2,[parallel]},ok}}, [{?eh,tc_start,{ts_if_1_SUITE,{init_per_group,g3,[]}}}, - {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g3,[]},{skipped,g3_got_skipped}}}, - {?eh,tc_auto_skip,{ts_if_1_SUITE,gtc2,g3_got_skipped}}, - {?eh,test_stats,{2,4,{3,6}}}, - {?eh,tc_auto_skip,{ts_if_1_SUITE,end_per_group,g3_got_skipped}}], + {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g3,[]}, + {skipped,g3_got_skipped}}}, + {?eh,tc_user_skip,{ts_if_1_SUITE,gtc2,g3_got_skipped}}, + {?eh,test_stats,{2,4,{4,5}}}, + {?eh,tc_user_skip,{ts_if_1_SUITE,end_per_group,g3_got_skipped}}], {?eh,tc_start,{ts_if_1_SUITE,{end_per_group,g2,[parallel]}}}, {?eh,tc_done,{ts_if_1_SUITE,{end_per_group,g2,[parallel]},ok}}]}, {?eh,tc_start,{ts_if_1_SUITE,tc12}}, {?eh,tc_done,{ts_if_1_SUITE,tc12,{failed,{testcase_aborted,'stopping tc12'}}}}, - {?eh,test_stats,{2,5,{3,6}}}, + {?eh,test_stats,{2,5,{4,5}}}, {?eh,tc_start,{ts_if_1_SUITE,tc13}}, {?eh,tc_done,{ts_if_1_SUITE,tc13,ok}}, - {?eh,test_stats,{3,5,{3,6}}}, + {?eh,test_stats,{3,5,{4,5}}}, {?eh,tc_start,{ts_if_1_SUITE,end_per_suite}}, {?eh,tc_done,{ts_if_1_SUITE,end_per_suite,ok}}, @@ -223,22 +226,23 @@ test_events(ts_if_1) -> {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}}, {?eh,tc_auto_skip,{ts_if_2_SUITE,my_test_case, {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}}, - {?eh,test_stats,{3,5,{3,7}}}, + {?eh,test_stats,{3,5,{4,6}}}, {?eh,tc_auto_skip,{ts_if_2_SUITE,end_per_suite, {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}}, {?eh,tc_start,{ct_framework,error_in_suite}}, - {?eh,test_stats,{3,5,{4,7}}}, + {?eh,test_stats,{3,6,{4,6}}}, {?eh,tc_start,{ct_framework,error_in_suite}}, - {?eh,test_stats,{3,5,{5,7}}}, + {?eh,test_stats,{3,7,{4,6}}}, {?eh,tc_start,{ts_if_5_SUITE,init_per_suite}}, {?eh,tc_done,{ts_if_5_SUITE,init_per_suite, - {skipped,{require_failed_in_suite0,{not_available,undef_variable}}}}}, + {auto_skipped,{require_failed_in_suite0, + {not_available,undef_variable}}}}}, {?eh,tc_auto_skip,{ts_if_5_SUITE,my_test_case, {require_failed_in_suite0,{not_available,undef_variable}}}}, - {?eh,test_stats,{3,5,{5,8}}}, + {?eh,test_stats,{3,7,{4,7}}}, {?eh,tc_auto_skip,{ts_if_5_SUITE,end_per_suite, {require_failed_in_suite0,{not_available,undef_variable}}}}, @@ -247,23 +251,25 @@ test_events(ts_if_1) -> {failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}}, {?eh,tc_auto_skip,{ts_if_6_SUITE,tc1, {failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}}, - {?eh,test_stats,{3,5,{5,9}}}, + {?eh,test_stats,{3,7,{4,8}}}, + {?eh,tc_auto_skip,{ct_framework,end_per_suite, + {failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}}, {?eh,tc_start,{ts_if_7_SUITE,tc1}}, {?eh,tc_done,{ts_if_7_SUITE,tc1,ok}}, - {?eh,test_stats,{4,5,{5,9}}}, + {?eh,test_stats,{4,7,{4,8}}}, {?eh,tc_start,{ts_if_8_SUITE,tc1}}, {?eh,tc_done,{ts_if_8_SUITE,tc1,{failed,{error,failed_on_purpose}}}}, - {?eh,test_stats,{4,6,{5,9}}}, + {?eh,test_stats,{4,8,{4,8}}}, {?eh,tc_user_skip,{skipped_by_spec_1_SUITE,all,"should be skipped"}}, - {?eh,test_stats,{4,6,{6,9}}}, + {?eh,test_stats,{4,8,{5,8}}}, {?eh,tc_start,{skipped_by_spec_2_SUITE,init_per_suite}}, {?eh,tc_done,{skipped_by_spec_2_SUITE,init_per_suite,ok}}, {?eh,tc_user_skip,{skipped_by_spec_2_SUITE,tc1,"should be skipped"}}, - {?eh,test_stats,{4,6,{7,9}}}, + {?eh,test_stats,{4,8,{6,8}}}, {?eh,tc_start,{skipped_by_spec_2_SUITE,end_per_suite}}, {?eh,tc_done,{skipped_by_spec_2_SUITE,end_per_suite,ok}}, diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl index 67e430f821..772274ce7e 100644 --- a/lib/common_test/test/ct_test_support.erl +++ b/lib/common_test/test/ct_test_support.erl @@ -692,8 +692,10 @@ locate({parallel,TEvs}, Node, Evs, Config) -> test_server:format("Found ~p!", [TEv]), {Done,RemEvs2,length(RemEvs2)} end; - %% end_per_group auto skipped - (TEv={TEH,tc_auto_skip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize}) -> + %% end_per_group auto- or user skipped + (TEv={TEH,AutoOrUserSkip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize}) + when AutoOrUserSkip == tc_auto_skip; + AutoOrUserSkip == tc_user_skip -> RemEvs1 = lists:dropwhile( fun({EH,#event{name=tc_auto_skip, @@ -704,10 +706,18 @@ locate({parallel,TEvs}, Node, Evs, Config) -> match -> false; _ -> true end; + ({EH,#event{name=tc_user_skip, + node=EvNode, + data={Mod,end_per_group,Reason}}}) when + EH == TEH, EvNode == Node, Mod == M -> + case match_data(R, Reason) of + match -> false; + _ -> true + end; ({EH,#event{name=stop_logging, node=EvNode,data=_}}) when EH == TEH, EvNode == Node -> - exit({tc_auto_skip_not_found,TEv}); + exit({tc_auto_or_user_skip_not_found,TEv}); (_) -> true end, RemEvs), @@ -925,8 +935,10 @@ locate({shuffle,TEvs}, Node, Evs, Config) -> test_server:format("Found ~p!", [TEv]), {Done,RemEvs2,length(RemEvs2)} end; - %% end_per_group auto skipped - (TEv={TEH,tc_auto_skip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize}) -> + %% end_per_group auto-or user skipped + (TEv={TEH,AutoOrUserSkip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize}) + when AutoOrUserSkip == tc_auto_skip; + AutoOrUserSkip == tc_user_skip -> RemEvs1 = lists:dropwhile( fun({EH,#event{name=tc_auto_skip, @@ -934,6 +946,11 @@ locate({shuffle,TEvs}, Node, Evs, Config) -> data={Mod,end_per_group,Reason}}}) when EH == TEH, EvNode == Node, Mod == M, Reason == R -> false; + ({EH,#event{name=tc_user_skip, + node=EvNode, + data={Mod,end_per_group,Reason}}}) when + EH == TEH, EvNode == Node, Mod == M, Reason == R -> + false; ({EH,#event{name=stop_logging, node=EvNode,data=_}}) when EH == TEH, EvNode == Node -> @@ -1178,6 +1195,9 @@ log_events1([E={_EH,tc_done,{_M,{end_per_group,_GrName,Props},_R}} | Evs], Dev, log_events1([E={_EH,tc_auto_skip,{_M,end_per_group,_Reason}} | Evs], Dev, Ind) -> io:format(Dev, "~s~p],~n", [Ind,E]), log_events1(Evs, Dev, Ind--" "); +log_events1([E={_EH,tc_user_skip,{_M,end_per_group,_Reason}} | Evs], Dev, Ind) -> + io:format(Dev, "~s~p],~n", [Ind,E]), + log_events1(Evs, Dev, Ind--" "); log_events1([E], Dev, Ind) -> io:format(Dev, "~s~p~n].~n", [Ind,E]), ok; diff --git a/lib/test_server/src/test_server.erl b/lib/test_server/src/test_server.erl index b2cfe5b440..54be6d4c72 100644 --- a/lib/test_server/src/test_server.erl +++ b/lib/test_server/src/test_server.erl @@ -925,7 +925,7 @@ run_test_case_eval(Mod, Func, Args0, Name, Ref, RunInit, run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback); Error = {error,_Reason} -> NewResult = do_end_tc_call(Mod,Func, {Error,Args0}, - {skip,{failed,Error}}), + {auto_skip,{failed,Error}}), {{0,NewResult},Where,[]}; {fail,Reason} -> Conf = [{tc_status,{failed,Reason}} | hd(Args0)], diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index bef967ff12..353559df03 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -2210,13 +2210,19 @@ run_test_cases(TestSpec, Config, TimetrapData) -> %% group1_end | ---> %% -run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], - Config, TimetrapData, Mode, Status) when Type==conf; - Type==make -> +run_test_cases_loop([{SkipTag,{Type,Ref,Case,Comment},SkipMode}|Cases], + Config, TimetrapData, Mode, Status) when + ((SkipTag==auto_skip_case) or (SkipTag==skip_case)) and + ((Type==conf) or (Type==make)) -> file:set_cwd(filename:dirname(get(test_server_dir))), CurrIOHandler = get(test_server_common_io_handler), ParentMode = tl(Mode), + {AutoOrUser,ReportTag} = + if SkipTag == auto_skip_case -> {auto,tc_auto_skip}; + SkipTag == skip_case -> {user,tc_user_skip} + end, + %% check and update the mode for test case execution and io msg handling case {curr_ref(Mode),check_props(parallel, Mode)} of {Ref,Ref} -> @@ -2226,9 +2232,9 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], %% group, buffered io can be flushed handle_test_case_io_and_status(), set_io_buffering(undefined), - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, + {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, false, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip, + test_server_sup:framework_call(report, [ReportTag, {Mod,Func,Comment}]), run_test_cases_loop(Cases, Config, TimetrapData, ParentMode, delete_status(Ref, Status)); @@ -2236,9 +2242,9 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], %% this is a skipped end conf for a parallel group nested %% under a parallel group (io buffering is active) wait_for_cases(Ref), - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, + {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, true, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip, + test_server_sup:framework_call(report, [ReportTag, {Mod,Func,Comment}]), case CurrIOHandler of {Ref,_} -> @@ -2256,9 +2262,9 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], {Ref,false} -> %% this is a skipped end conf for a non-parallel group that's not %% nested under a parallel group - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, + {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, false, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip, + test_server_sup:framework_call(report, [ReportTag, {Mod,Func,Comment}]), %% Check if this group is auto skipped because of error in the @@ -2277,7 +2283,7 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], Reason = {group_result,GrName,failed}, skip_cases_upto(ParentRef, Cases, Reason, tc, Mode, - auto_skip_case) + SkipTag) end; false -> Cases @@ -2290,9 +2296,9 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], {Ref,_} -> %% this is a skipped end conf for a non-parallel group nested under %% a parallel group (io buffering is active) - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, + {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, true, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip, + test_server_sup:framework_call(report, [ReportTag, {Mod,Func,Comment}]), case CurrIOHandler of {Ref,_} -> @@ -2308,9 +2314,9 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], {_,false} -> %% this is a skipped start conf for a group which is not nested %% under a parallel group - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, + {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, false, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip, + test_server_sup:framework_call(report, [ReportTag, {Mod,Func,Comment}]), run_test_cases_loop(Cases, Config, TimetrapData, [conf(Ref,[])|Mode], Status); @@ -2323,9 +2329,9 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], true -> ok end, - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, + {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, true, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip, + test_server_sup:framework_call(report, [ReportTag, {Mod,Func,Comment}]), run_test_cases_loop(Cases, Config, TimetrapData, [conf(Ref,[])|Mode], Status) @@ -2339,23 +2345,6 @@ run_test_cases_loop([{auto_skip_case,{Case,Comment},SkipMode}|Cases], run_test_cases_loop(Cases, Config, TimetrapData, Mode, update_status(skipped, Mod, Func, Status)); -run_test_cases_loop([{skip_case,{conf,Ref,Case,Comment}}|Cases0], - Config, TimetrapData, Mode, Status) -> - {Mod,Func} = skip_case(user, Ref, 0, Case, Comment, is_io_buffered()), - {Cases,Config1} = - case curr_ref(Mode) of - Ref -> - %% skipped end conf - {Cases0,tl(Config)}; - _ -> - %% skipped start conf - {skip_cases_upto(Ref, Cases0, Comment, conf, Mode, skip_case), - Config} - end, - test_server_sup:framework_call(report, [tc_user_skip,{Mod,Func,Comment}]), - run_test_cases_loop(Cases, Config1, TimetrapData, Mode, - update_status(skipped, Mod, Func, Status)); - run_test_cases_loop([{skip_case,{Case,Comment}}|Cases], Config, TimetrapData, Mode, Status) -> {Mod,Func} = skip_case(user, undefined, get(test_server_case_num)+1, |