diff options
Diffstat (limited to 'lib/common_test/test')
| -rw-r--r-- | lib/common_test/test/ct_config_SUITE.erl | 34 | ||||
| -rw-r--r-- | lib/common_test/test/ct_error_SUITE.erl | 18 | ||||
| -rw-r--r-- | lib/common_test/test/ct_group_info_SUITE.erl | 35 | ||||
| -rw-r--r-- | lib/common_test/test/ct_groups_test_2_SUITE.erl | 12 | ||||
| -rw-r--r-- | lib/common_test/test/ct_hooks_SUITE.erl | 26 | ||||
| -rw-r--r-- | lib/common_test/test/ct_repeat_testrun_SUITE.erl | 14 | ||||
| -rw-r--r-- | lib/common_test/test/ct_skip_SUITE.erl | 102 | ||||
| -rw-r--r-- | lib/common_test/test/ct_surefire_SUITE.erl | 4 | ||||
| -rw-r--r-- | lib/common_test/test/ct_test_server_if_1_SUITE.erl | 70 | ||||
| -rw-r--r-- | lib/common_test/test/ct_test_support.erl | 30 |
10 files changed, 227 insertions, 118 deletions
diff --git a/lib/common_test/test/ct_config_SUITE.erl b/lib/common_test/test/ct_config_SUITE.erl index d92be9ec6e..284f836517 100644 --- a/lib/common_test/test/ct_config_SUITE.erl +++ b/lib/common_test/test/ct_config_SUITE.erl @@ -234,25 +234,25 @@ expected_events(config_static_SUITE)-> ?sok(test_get_config_deep_nested,{3,0,{0,0}}), ?sok(test_default_suitewide,{4,0,{0,0}}), ?snok(test_config_name_already_in_use1, - {skipped,{config_name_already_in_use,[x1]}},{4,0,{1,0}}), - ?sok(test_default_tclocal,{5,0,{1,0}}), + {failed,{error,{config_name_already_in_use,[x1]}}},{4,1,{0,0}}), + ?sok(test_default_tclocal,{5,1,{0,0}}), ?snok(test_config_name_already_in_use2, - {skipped,{config_name_already_in_use,[alias,x1]}},{5,0,{2,0}}), - ?sok(test_alias_tclocal,{6,0,{2,0}}), - ?sok(test_get_config_undefined,{7,0,{2,0}}), - ?sok(test_require_subvals,{8,0,{2,0}}), + {failed,{error,{config_name_already_in_use,[alias,x1]}}},{5,2,{0,0}}), + ?sok(test_alias_tclocal,{6,2,{0,0}}), + ?sok(test_get_config_undefined,{7,2,{0,0}}), + ?sok(test_require_subvals,{8,2,{0,0}}), ?snok(test_require_subvals2, - {skipped,{require_failed, - {not_available,{gen_cfg,[a,b,c,d]}}}},{8,0,{2,1}}), - ?sok(test_require_deep_config,{9,0,{2,1}}), - ?sok(test_shadow_all,{10,0,{2,1}}), - ?sok(test_element,{11,0,{2,1}}), - ?sok(test_shadow_all_element,{12,0,{2,1}}), - ?sok(test_internal_deep,{13,0,{2,1}}), - ?sok(test_alias_tclocal_nested,{14,0,{2,1}}), - ?sok(test_alias_tclocal_nested_backward_compat,{15,0,{2,1}}), - ?sok(test_alias_tclocal_nested_backward_compat_subvals,{16,0,{2,1}}), - ?sok(test_config_same_name_already_in_use,{17,0,{2,1}}), + {auto_skipped,{require_failed, + {not_available,{gen_cfg,[a,b,c,d]}}}},{8,2,{0,1}}), + ?sok(test_require_deep_config,{9,2,{0,1}}), + ?sok(test_shadow_all,{10,2,{0,1}}), + ?sok(test_element,{11,2,{0,1}}), + ?sok(test_shadow_all_element,{12,2,{0,1}}), + ?sok(test_internal_deep,{13,2,{0,1}}), + ?sok(test_alias_tclocal_nested,{14,2,{0,1}}), + ?sok(test_alias_tclocal_nested_backward_compat,{15,2,{0,1}}), + ?sok(test_alias_tclocal_nested_backward_compat_subvals,{16,2,{0,1}}), + ?sok(test_config_same_name_already_in_use,{17,2,{0,1}}), {?eh,tc_start,{config_static_SUITE,end_per_suite}}, {?eh,tc_done,{config_static_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index 3881ced17d..a59385c53b 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -622,41 +622,41 @@ test_events(cfg_error) -> {?eh,tc_done,{cfg_error_11_SUITE,init_per_suite,ok}}, {?eh,tc_start,{cfg_error_11_SUITE,tc1}}, {?eh,tc_done, {cfg_error_11_SUITE,tc1, - {skipped,{config_name_already_in_use,[dummy_alias]}}}}, - {?eh,test_stats,{12,6,{1,19}}}, + {failed,{error,{config_name_already_in_use,[dummy_alias]}}}}}, + {?eh,test_stats,{12,7,{0,19}}}, {?eh,tc_start,{cfg_error_11_SUITE,tc2}}, {?eh,tc_done,{cfg_error_11_SUITE,tc2,ok}}, - {?eh,test_stats,{13,6,{1,19}}}, + {?eh,test_stats,{13,7,{0,19}}}, {?eh,tc_start,{cfg_error_11_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_11_SUITE,end_per_suite,ok}}, {?eh,tc_start,{cfg_error_12_SUITE,tc1}}, {?eh,tc_done,{ct_framework,init_tc,{framework_error,{timetrap,500}}}}, - {?eh,test_stats,{13,7,{1,19}}}, + {?eh,test_stats,{13,8,{0,19}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc2}}, {?eh,tc_done,{cfg_error_12_SUITE,tc2,{failed, {cfg_error_12_SUITE,end_per_testcase, {timetrap_timeout,500}}}}}, - {?eh,test_stats,{14,7,{1,19}}}, + {?eh,test_stats,{14,8,{0,19}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc3}}, {?eh,tc_done,{cfg_error_12_SUITE,tc3,ok}}, - {?eh,test_stats,{15,7,{1,19}}}, + {?eh,test_stats,{15,8,{0,19}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc4}}, {?eh,tc_done,{cfg_error_12_SUITE,tc4,{failed, {cfg_error_12_SUITE,end_per_testcase, {timetrap_timeout,500}}}}}, - {?eh,test_stats,{16,7,{1,19}}}, + {?eh,test_stats,{16,8,{0,19}}}, {?eh,tc_start,{cfg_error_13_SUITE,init_per_suite}}, {?eh,tc_done,{cfg_error_13_SUITE,init_per_suite,ok}}, {?eh,tc_start,{cfg_error_13_SUITE,tc1}}, {?eh,tc_done,{cfg_error_13_SUITE,tc1,ok}}, - {?eh,test_stats,{17,7,{1,19}}}, + {?eh,test_stats,{17,8,{0,19}}}, {?eh,tc_start,{cfg_error_13_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_13_SUITE,end_per_suite,ok}}, {?eh,tc_start,{cfg_error_14_SUITE,init_per_suite}}, {?eh,tc_done,{cfg_error_14_SUITE,init_per_suite,ok}}, {?eh,tc_start,{cfg_error_14_SUITE,tc1}}, {?eh,tc_done,{cfg_error_14_SUITE,tc1,ok}}, - {?eh,test_stats,{18,7,{1,19}}}, + {?eh,test_stats,{18,8,{0,19}}}, {?eh,tc_start,{cfg_error_14_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_14_SUITE,end_per_suite, {comment, diff --git a/lib/common_test/test/ct_group_info_SUITE.erl b/lib/common_test/test/ct_group_info_SUITE.erl index c56fa952e8..c715142ff5 100644 --- a/lib/common_test/test/ct_group_info_SUITE.erl +++ b/lib/common_test/test/ct_group_info_SUITE.erl @@ -539,9 +539,11 @@ test_events(require) -> [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g4,[]}}}, {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g4,[]}, - {skipped,{require_failed,{name_in_use,common2_alias,common2}}}}}, + {auto_skipped,{require_failed, + {name_in_use,common2_alias,common2}}}}}, {?eh,tc_auto_skip,{group_require_1_SUITE,t41, - {require_failed,{name_in_use,common2_alias,common2}}}}, + {require_failed, + {name_in_use,common2_alias,common2}}}}, {?eh,test_stats,{4,0,{0,1}}}, {?eh,tc_auto_skip,{group_require_1_SUITE,end_per_group, {require_failed,{name_in_use,common2_alias,common2}}}}], @@ -566,7 +568,8 @@ test_events(require) -> [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g8,[]}}}, {?eh,tc_done,{group_require_1_SUITE, {init_per_group,g8,[]}, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,tc_auto_skip,{group_require_1_SUITE,t81, {require_failed,{not_available,non_existing}}}}, {?eh,test_stats,{8,0,{0,2}}}, @@ -575,7 +578,8 @@ test_events(require) -> [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g9,[]}}}, {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g9,[]},ok}}, {?eh,tc_done,{group_require_1_SUITE,t91, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,test_stats,{8,0,{0,3}}}, {?eh,tc_start,{group_require_1_SUITE,{end_per_group,g9,[]}}}, {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g9,[]},ok}}], @@ -627,7 +631,7 @@ test_events(require_default) -> [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g4,[]}}}, {?eh,tc_done,{group_require_1_SUITE, {init_per_group,g4,[]}, - {skipped,{require_failed,{not_available,common3}}}}}, + {auto_skipped,{require_failed,{not_available,common3}}}}}, {?eh,tc_auto_skip,{group_require_1_SUITE,t41, {require_failed,{not_available,common3}}}}, {?eh,test_stats,{4,0,{0,1}}}, @@ -654,7 +658,8 @@ test_events(require_default) -> [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g8,[]}}}, {?eh,tc_done,{group_require_1_SUITE, {init_per_group,g8,[]}, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,tc_auto_skip,{group_require_1_SUITE,t81, {require_failed,{not_available,non_existing}}}}, {?eh,test_stats,{8,0,{0,2}}}, @@ -664,7 +669,8 @@ test_events(require_default) -> [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g9,[]}}}, {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g9,[]},ok}}, {?eh,tc_done,{group_require_1_SUITE,t91, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,test_stats,{8,0,{0,3}}}, {?eh,tc_start,{group_require_1_SUITE,{end_per_group,g9,[]}}}, {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g9,[]},ok}}], @@ -714,7 +720,8 @@ test_events(require_no_ips) -> [{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g4,[]}}}, {?eh,tc_done,{group_require_2_SUITE,{init_per_group,g4,[]}, - {skipped,{require_failed,{name_in_use,common2_alias,common2}}}}}, + {auto_skipped,{require_failed, + {name_in_use,common2_alias,common2}}}}}, {?eh,tc_auto_skip,{group_require_2_SUITE,t41, {require_failed,{name_in_use,common2_alias,common2}}}}, {?eh,test_stats,{4,0,{0,1}}}, @@ -741,7 +748,8 @@ test_events(require_no_ips) -> [{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g8,[]}}}, {?eh,tc_done,{group_require_2_SUITE, {init_per_group,g8,[]}, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,tc_auto_skip,{group_require_2_SUITE,t81, {require_failed,{not_available,non_existing}}}}, {?eh,test_stats,{8,0,{0,2}}}, @@ -750,7 +758,8 @@ test_events(require_no_ips) -> [{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g9,[]}}}, {?eh,tc_done,{group_require_2_SUITE,{init_per_group,g9,[]},ok}}, {?eh,tc_done,{group_require_2_SUITE,t91, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,test_stats,{8,0,{0,3}}}, {?eh,tc_start,{group_require_2_SUITE,{end_per_group,g9,[]}}}, {?eh,tc_done,{group_require_2_SUITE,{end_per_group,g9,[]},ok}}], @@ -799,7 +808,7 @@ test_events(require_no_ipg) -> [{?eh,tc_start,{ct_framework,{init_per_group,g4,[{suite,group_require_3_SUITE}]}}}, {?eh,tc_done,{ct_framework,{init_per_group,g4,[{suite,group_require_3_SUITE}]}, - {skipped,{require_failed,{name_in_use,common2_alias,common2}}}}}, + {auto_skipped,{require_failed,{name_in_use,common2_alias,common2}}}}}, {?eh,tc_auto_skip,{group_require_3_SUITE,t41, {require_failed,{name_in_use,common2_alias,common2}}}}, {?eh,test_stats,{4,0,{0,1}}}, @@ -825,7 +834,7 @@ test_events(require_no_ipg) -> [{?eh,tc_start,{ct_framework,{init_per_group,g8,[{suite,group_require_3_SUITE}]}}}, {?eh,tc_done,{ct_framework,{init_per_group,g8,[{suite,group_require_3_SUITE}]}, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed,{not_available,non_existing}}}}}, {?eh,tc_auto_skip,{group_require_3_SUITE,t81, {require_failed,{not_available,non_existing}}}}, {?eh,test_stats,{8,0,{0,2}}}, @@ -834,7 +843,7 @@ test_events(require_no_ipg) -> [{?eh,tc_start,{ct_framework,{init_per_group,g9,[{suite,group_require_3_SUITE}]}}}, {?eh,tc_done,{ct_framework,{init_per_group,g9,[{suite,group_require_3_SUITE}]},ok}}, {?eh,tc_done,{group_require_3_SUITE,t91, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed,{not_available,non_existing}}}}}, {?eh,test_stats,{8,0,{0,3}}}, {?eh,tc_start,{ct_framework,{end_per_group,g9,[{suite,group_require_3_SUITE}]}}}, {?eh,tc_done,{ct_framework,{end_per_group,g9,[{suite,group_require_3_SUITE}]},ok}}], diff --git a/lib/common_test/test/ct_groups_test_2_SUITE.erl b/lib/common_test/test/ct_groups_test_2_SUITE.erl index 1b2ad12e2f..8b0de98709 100644 --- a/lib/common_test/test/ct_groups_test_2_SUITE.erl +++ b/lib/common_test/test/ct_groups_test_2_SUITE.erl @@ -171,16 +171,20 @@ test_events(missing_conf) -> {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, {?eh,start_info,{1,1,2}}, - {?eh,tc_start,{ct_framework,{init_per_group,group1,[]}}}, - {?eh,tc_done,{ct_framework,{init_per_group,group1,[]},ok}}, + {?eh,tc_start,{ct_framework,{init_per_group,group1, + [{suite,missing_conf_SUITE}]}}}, + {?eh,tc_done,{ct_framework,{init_per_group,group1, + [{suite,missing_conf_SUITE}]},ok}}, {?eh,tc_start,{missing_conf_SUITE,tc1}}, {?eh,tc_done,{missing_conf_SUITE,tc1,ok}}, {?eh,test_stats,{1,0,{0,0}}}, {?eh,tc_start,{missing_conf_SUITE,tc2}}, {?eh,tc_done,{missing_conf_SUITE,tc2,ok}}, {?eh,test_stats,{2,0,{0,0}}}, - {?eh,tc_start,{ct_framework,{end_per_group,group1,[]}}}, - {?eh,tc_done,{ct_framework,{end_per_group,group1,[]},ok}}, + {?eh,tc_start,{ct_framework,{end_per_group,group1, + [{suite,missing_conf_SUITE}]}}}, + {?eh,tc_done,{ct_framework,{end_per_group,group1, + [{suite,missing_conf_SUITE}]},ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} ]; diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl index 596bfe3ff0..083c87b49e 100644 --- a/lib/common_test/test/ct_hooks_SUITE.erl +++ b/lib/common_test/test/ct_hooks_SUITE.erl @@ -749,11 +749,10 @@ test_events(skip_pre_suite_cth) -> {?eh,cth,{'_',on_tc_skip, [init_per_suite,{tc_user_skip,{skipped,"Test skip"}},[]]}}, - {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, - {?eh,cth,{'_',on_tc_skip,[test_case,{tc_auto_skip,"Test skip"},[]]}}, + {?eh,tc_user_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, + {?eh,cth,{'_',on_tc_skip,[test_case,{tc_user_skip,"Test skip"},[]]}}, - {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, - {?eh,cth,{'_',on_tc_skip,[end_per_suite,{tc_auto_skip,"Test skip"},[]]}}, + {?eh,tc_user_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,cth, {'_',terminate,[[]]}}, @@ -773,11 +772,10 @@ test_events(skip_post_suite_cth) -> {?eh,cth,{'_',on_tc_skip, [init_per_suite,{tc_user_skip,{skipped,"Test skip"}},[]]}}, - {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, - {?eh,cth,{'_',on_tc_skip,[test_case,{tc_auto_skip,"Test skip"},[]]}}, + {?eh,tc_user_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, + {?eh,cth,{'_',on_tc_skip,[test_case,{tc_user_skip,"Test skip"},[]]}}, - {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, - {?eh,cth,{'_',on_tc_skip,[end_per_suite,{tc_auto_skip,"Test skip"},[]]}}, + {?eh,tc_user_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,cth,{'_',terminate,[[]]}}, @@ -1219,11 +1217,15 @@ test_events(cth_log) -> {?eh,tc_start,{cth_log_SUITE,init_per_suite}}, {parallel, - [{?eh,tc_start,{ct_framework,{init_per_group,g1,[parallel]}}}, - {?eh,tc_done,{ct_framework,{init_per_group,g1,[parallel]},ok}}, + [{?eh,tc_start,{ct_framework,{init_per_group,g1, + [{suite,cth_log_SUITE},parallel]}}}, + {?eh,tc_done,{ct_framework,{init_per_group,g1, + [{suite,cth_log_SUITE},parallel]},ok}}, {?eh,test_stats,{30,0,{0,0}}}, - {?eh,tc_start,{ct_framework,{end_per_group,g1,[parallel]}}}, - {?eh,tc_done,{ct_framework,{end_per_group,g1,[parallel]},ok}}]}, + {?eh,tc_start,{ct_framework,{end_per_group,g1, + [{suite,cth_log_SUITE},parallel]}}}, + {?eh,tc_done,{ct_framework,{end_per_group,g1, + [{suite,cth_log_SUITE},parallel]},ok}}]}, {?eh,tc_done,{cth_log_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, diff --git a/lib/common_test/test/ct_repeat_testrun_SUITE.erl b/lib/common_test/test/ct_repeat_testrun_SUITE.erl index 35d67a10f2..9d5216449b 100644 --- a/lib/common_test/test/ct_repeat_testrun_SUITE.erl +++ b/lib/common_test/test/ct_repeat_testrun_SUITE.erl @@ -78,7 +78,7 @@ init_per_suite(Config0) -> {1,0,{0,0}} = ct_test_support:run(ct,run_test,[Opts2],Config), %% Time the shortest testcase to use for offset - {T0,{1,0,{0,0}}} = timer:tc(ct_test_support,run,[ct,run_test,[Opts1],Config]), + {_T0,{1,0,{0,0}}} = timer:tc(ct_test_support,run,[ct,run_test,[Opts1],Config]), %% -2 is to ensure we hit inside the target test case and not after % T = round(T0/1000000)-2, @@ -343,13 +343,13 @@ skip_first_tc1(Suite) -> {?eh,tc_done,{Suite,tc2,?skipped}}, {?eh,test_stats,{'_',0,{1,0}}}, {?eh,tc_done,{Suite,{init_per_group,g,[]},?skipped}}, - {?eh,tc_auto_skip,{Suite,tc1,?skip_reason}}, - {?eh,test_stats,{'_',0,{1,1}}}, - {?eh,tc_auto_skip,{Suite,tc2,?skip_reason}}, - {?eh,test_stats,{'_',0,{1,2}}}, - {?eh,tc_auto_skip,{Suite,end_per_group,?skip_reason}}, + {?eh,tc_user_skip,{Suite,tc1,?skip_reason}}, + {?eh,test_stats,{'_',0,{2,0}}}, + {?eh,tc_user_skip,{Suite,tc2,?skip_reason}}, + {?eh,test_stats,{'_',0,{3,0}}}, + {?eh,tc_user_skip,{Suite,end_per_group,?skip_reason}}, {?eh,tc_done,{Suite,tc2,?skipped}}, - {?eh,test_stats,{'_',0,{2,2}}}, + {?eh,test_stats,{'_',0,{4,0}}}, {?eh,tc_start,{Suite,end_per_suite}}, {?eh,tc_done,{Suite,end_per_suite,ok}}]. diff --git a/lib/common_test/test/ct_skip_SUITE.erl b/lib/common_test/test/ct_skip_SUITE.erl index 6f6ff754aa..f7cdc90a4d 100644 --- a/lib/common_test/test/ct_skip_SUITE.erl +++ b/lib/common_test/test/ct_skip_SUITE.erl @@ -164,7 +164,7 @@ test_events(auto_skip) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, - {?eh,start_info,{11,11,34}}, + {?eh,start_info,{12,12,43}}, {?eh,tc_start,{auto_skip_1_SUITE,init_per_suite}}, {?eh,tc_done, @@ -384,26 +384,26 @@ test_events(auto_skip) -> {?eh,tc_start,{auto_skip_10_SUITE,init_per_suite}}, {?eh,tc_done,{auto_skip_10_SUITE,init_per_suite, - {failed,{error, - {require_failed_in_suite0, - {not_available,undefined_config_variable}}}}}}, - {?eh,tc_auto_skip, - {auto_skip_10_SUITE,tc1, - {require_failed_in_suite0,{not_available,undefined_config_variable}}}}, + {auto_skipped,{require_failed_in_suite0, + {not_available,undefined_config_variable}}}}}, + {?eh,tc_auto_skip,{auto_skip_10_SUITE,tc1, + {require_failed_in_suite0, + {not_available,undefined_config_variable}}}}, {?eh,test_stats,{9,0,{0,21}}}, - {?eh,tc_auto_skip, - {auto_skip_10_SUITE,tc2, - {require_failed_in_suite0,{not_available,undefined_config_variable}}}}, + {?eh,tc_auto_skip,{auto_skip_10_SUITE,tc2, + {require_failed_in_suite0, + {not_available,undefined_config_variable}}}}, {?eh,test_stats,{9,0,{0,22}}}, - {?eh,tc_auto_skip, - {auto_skip_10_SUITE,end_per_suite, - {require_failed_in_suite0,{not_available,undefined_config_variable}}}}, + {?eh,tc_auto_skip,{auto_skip_10_SUITE,end_per_suite, + {require_failed_in_suite0, + {not_available,undefined_config_variable}}}}, {?eh,tc_start,{auto_skip_11_SUITE,init_per_suite}}, {?eh,tc_done,{auto_skip_11_SUITE,init_per_suite,ok}}, {?eh,tc_start,{auto_skip_11_SUITE,tc1}}, {?eh,tc_done,{auto_skip_11_SUITE,tc1, - {skipped,{require_failed,{not_available,undefined_config_variable}}}}}, + {auto_skipped,{require_failed, + {not_available,undefined_config_variable}}}}}, {?eh,test_stats,{9,0,{0,23}}}, {?eh,tc_start,{auto_skip_11_SUITE,tc2}}, {?eh,tc_done,{auto_skip_11_SUITE,tc2,ok}}, @@ -422,7 +422,8 @@ test_events(auto_skip) -> {?eh,tc_start,{auto_skip_11_SUITE,tc3}}, {?eh,tc_done, {auto_skip_11_SUITE,tc3, - {skipped,{require_failed,{not_available,undefined_config_variable}}}}}, + {auto_skipped,{require_failed, + {not_available,undefined_config_variable}}}}}, {?eh,test_stats,{10,0,{0,24}}}, {?eh,tc_start, {auto_skip_11_SUITE,{end_per_group,g2,[parallel]}}}, @@ -435,8 +436,75 @@ test_events(auto_skip) -> {?eh,tc_start,{auto_skip_11_SUITE,end_per_suite}}, {?eh,tc_done,{auto_skip_11_SUITE,end_per_suite,ok}}, - - %%! HERE: auto_skip_12_SUITE terms!! + + {?eh,tc_start,{auto_skip_12_SUITE,init_per_suite}}, + {?eh,tc_done,{auto_skip_12_SUITE,init_per_suite,ok}}, + + [{?eh,tc_start,{ct_framework,{init_per_group,g1, + [{suite,auto_skip_12_SUITE}]}}}, + {?eh,tc_done,{ct_framework,{init_per_group,g1, + [{suite,auto_skip_12_SUITE}]}, + {auto_skipped, + {require_failed,{not_available,unknown_variable_g1}}}}}, + {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc1, + {require_failed,{not_available,unknown_variable_g1}}}}, + {?eh,test_stats,{10,0,{0,25}}}, + {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc2, + {require_failed,{not_available,unknown_variable_g1}}}}, + {?eh,test_stats,{10,0,{0,26}}}, + {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3, + {require_failed,{not_available,unknown_variable_g1}}}}, + {?eh,test_stats,{10,0,{0,27}}}, + {?eh,tc_auto_skip,{ct_framework,end_per_group, + {require_failed,{not_available,unknown_variable_g1}}}}], + + [{?eh,tc_start,{ct_framework,{init_per_group,g1, + [{suite,auto_skip_12_SUITE}]}}}, + {?eh,tc_done,{ct_framework,{init_per_group,g1, + [{suite,auto_skip_12_SUITE}]}, + {auto_skipped, + {require_failed,{not_available,unknown_variable_g1}}}}}, + {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc1, + {require_failed,{not_available,unknown_variable_g1}}}}, + {?eh,test_stats,{10,0,{0,28}}}, + {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc2, + {require_failed,{not_available,unknown_variable_g1}}}}, + {?eh,test_stats,{10,0,{0,29}}}, + {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3, + {require_failed,{not_available,unknown_variable_g1}}}}, + {?eh,test_stats,{10,0,{0,30}}}, + {?eh,tc_auto_skip,{ct_framework,end_per_group, + {require_failed,{not_available,unknown_variable_g1}}}}], + + [{?eh,tc_start,{ct_framework,{init_per_group,g3, + [{suite,auto_skip_12_SUITE}]}}}, + {?eh,tc_done,{ct_framework,{init_per_group,g3, + [{suite,auto_skip_12_SUITE}]},ok}}, + {?eh,tc_start,{auto_skip_12_SUITE,tc1}}, + {?eh,tc_done,{auto_skip_12_SUITE,tc1,ok}}, + {?eh,test_stats,{11,0,{0,30}}}, + {?eh,tc_start,{auto_skip_12_SUITE,tc2}}, + {?eh,tc_done,{auto_skip_12_SUITE,tc2,ok}}, + {?eh,test_stats,{12,0,{0,30}}}, + [{?eh,tc_start,{ct_framework,{init_per_group,g4, + [{suite,auto_skip_12_SUITE}]}}}, + {?eh,tc_done,{ct_framework,{init_per_group,g4, + [{suite,auto_skip_12_SUITE}]}, + {auto_skipped, + {require_failed,{not_available,unknown_variable_g4}}}}}, + {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3, + {require_failed,{not_available,unknown_variable_g4}}}}, + {?eh,test_stats,{12,0,{0,31}}}, + {?eh,tc_auto_skip,{ct_framework,end_per_group, + {require_failed,{not_available,unknown_variable_g4}}}}], + + {?eh,tc_start,{ct_framework,{end_per_group,g3, + [{suite,auto_skip_12_SUITE}]}}}, + {?eh,tc_done,{ct_framework,{end_per_group,g3, + [{suite,auto_skip_12_SUITE}]},ok}}], + + {?eh,tc_start,{auto_skip_12_SUITE,end_per_suite}}, + {?eh,tc_done,{auto_skip_12_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} diff --git a/lib/common_test/test/ct_surefire_SUITE.erl b/lib/common_test/test/ct_surefire_SUITE.erl index b86b47f0a2..1a370de44a 100644 --- a/lib/common_test/test/ct_surefire_SUITE.erl +++ b/lib/common_test/test/ct_surefire_SUITE.erl @@ -182,7 +182,7 @@ test_events(_) -> {?eh,test_stats,{1,1,{1,0}}}, {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, - {skipped,{require_failed,'_'}}}}, + {auto_skipped,{require_failed,'_'}}}}, {?eh,test_stats,{1,1,{1,1}}}, [{?eh,tc_start,{surefire_SUITE,{init_per_group,g,[]}}}, {?eh,tc_done,{surefire_SUITE,{init_per_group,g,[]},ok}}, @@ -198,7 +198,7 @@ test_events(_) -> {?eh,test_stats,{2,2,{2,1}}}, {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, - {skipped,{require_failed,'_'}}}}, + {auto_skipped,{require_failed,'_'}}}}, {?eh,test_stats,{2,2,{2,2}}}, {?eh,tc_start,{surefire_SUITE,{end_per_group,g,[]}}}, {?eh,tc_done,{surefire_SUITE,{end_per_group,g,[]},ok}}], diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE.erl index 8e4852369d..64957a76c9 100644 --- a/lib/common_test/test/ct_test_server_if_1_SUITE.erl +++ b/lib/common_test/test/ct_test_server_if_1_SUITE.erl @@ -144,7 +144,7 @@ test_events(ts_if_1) -> {?eh,tc_start,{ts_if_1_SUITE,init_per_suite}}, {?eh,tc_done,{ts_if_1_SUITE,init_per_suite,ok}}, {?eh,tc_start,{ts_if_1_SUITE,tc1}}, - {?eh,tc_done,{ts_if_1_SUITE,tc1,{skipped, + {?eh,tc_done,{ts_if_1_SUITE,tc1,{auto_skipped, {failed, {ts_if_1_SUITE,init_per_testcase, {timetrap_timeout,2000}}}}}}, @@ -159,62 +159,65 @@ test_events(ts_if_1) -> {?eh,tc_start,{ts_if_1_SUITE,tc4}}, {?eh,tc_done,{ts_if_1_SUITE,tc4,{failed,{error,failed_on_purpose}}}}, {?eh,test_stats,{1,2,{0,1}}}, - {?eh,tc_done,{ts_if_1_SUITE,tc5,{skipped,{sequence_failed,seq1,tc4}}}}, - {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_done,{ts_if_1_SUITE,tc5,{auto_skipped,{sequence_failed,seq1,tc4}}}}, + {?eh,test_stats,{1,2,{0,2}}}, [{?eh,tc_start,{ts_if_1_SUITE,{init_per_group,seq2,[sequence]}}}, {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,seq2,[sequence]},ok}}, {?eh,tc_start,{ts_if_1_SUITE,tc4}}, {?eh,tc_done,{ts_if_1_SUITE,tc4,{failed,{error,failed_on_purpose}}}}, - {?eh,test_stats,{1,3,{1,1}}}, + {?eh,test_stats,{1,3,{0,2}}}, {?eh,tc_auto_skip,{ts_if_1_SUITE,tc5,{failed,{ts_if_1_SUITE,tc4}}}}, - {?eh,test_stats,{1,3,{1,2}}}, + {?eh,test_stats,{1,3,{0,3}}}, {?eh,tc_start,{ts_if_1_SUITE,{end_per_group,seq2,[sequence]}}}, {?eh,tc_done,{ts_if_1_SUITE,{end_per_group,seq2,[sequence]},ok}}], {?eh,tc_start,{ts_if_1_SUITE,tc6}}, - {?eh,tc_done,{ts_if_1_SUITE,tc6,{skipped,{require_failed,{not_available,void}}}}}, - {?eh,test_stats,{1,3,{1,3}}}, + {?eh,tc_done,{ts_if_1_SUITE,tc6,{auto_skipped, + {require_failed,{not_available,void}}}}}, + {?eh,test_stats,{1,3,{0,4}}}, {?eh,tc_start,{ts_if_1_SUITE,tc7}}, {?eh,tc_done,{ts_if_1_SUITE,tc7,ok}}, - {?eh,test_stats,{2,3,{1,3}}}, + {?eh,test_stats,{2,3,{0,4}}}, {?eh,tc_start,{ts_if_1_SUITE,tc8}}, {?eh,tc_done,{ts_if_1_SUITE,tc8,{skipped,"tc8 skipped"}}}, - {?eh,test_stats,{2,3,{2,3}}}, + {?eh,test_stats,{2,3,{1,4}}}, {?eh,tc_start,{ts_if_1_SUITE,tc9}}, {?eh,tc_done,{ts_if_1_SUITE,tc9,{skipped,'tc9 skipped'}}}, - {?eh,test_stats,{2,3,{3,3}}}, + {?eh,test_stats,{2,3,{2,4}}}, {?eh,tc_start,{ts_if_1_SUITE,tc10}}, {?eh,tc_done,{ts_if_1_SUITE,tc10,{failed,{error,{function_clause,'_'}}}}}, - {?eh,test_stats,{2,4,{3,3}}}, + {?eh,test_stats,{2,4,{2,4}}}, {?eh,tc_start,{ts_if_1_SUITE,tc11}}, {?eh,tc_done,{ts_if_1_SUITE,tc11, - {skipped,{failed,{ts_if_1_SUITE,init_per_testcase,bad_return}}}}}, - {?eh,test_stats,{2,4,{3,4}}}, + {auto_skipped,{failed,{ts_if_1_SUITE,init_per_testcase,bad_return}}}}}, + {?eh,test_stats,{2,4,{2,5}}}, [{?eh,tc_start,{ts_if_1_SUITE,{init_per_group,g1,[]}}}, - {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g1,[]},{skipped,g1_got_skipped}}}, - {?eh,tc_auto_skip,{ts_if_1_SUITE,gtc1,g1_got_skipped}}, + {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g1,[]}, + {skipped,g1_got_skipped}}}, + {?eh,tc_user_skip,{ts_if_1_SUITE,gtc1,g1_got_skipped}}, {?eh,test_stats,{2,4,{3,5}}}, - {?eh,tc_auto_skip,{ts_if_1_SUITE,end_per_group,g1_got_skipped}}], + {?eh,tc_user_skip,{ts_if_1_SUITE,end_per_group,g1_got_skipped}}], {parallel, [{?eh,tc_start,{ts_if_1_SUITE,{init_per_group,g2,[parallel]}}}, {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g2,[parallel]},ok}}, [{?eh,tc_start,{ts_if_1_SUITE,{init_per_group,g3,[]}}}, - {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g3,[]},{skipped,g3_got_skipped}}}, - {?eh,tc_auto_skip,{ts_if_1_SUITE,gtc2,g3_got_skipped}}, - {?eh,test_stats,{2,4,{3,6}}}, - {?eh,tc_auto_skip,{ts_if_1_SUITE,end_per_group,g3_got_skipped}}], + {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g3,[]}, + {skipped,g3_got_skipped}}}, + {?eh,tc_user_skip,{ts_if_1_SUITE,gtc2,g3_got_skipped}}, + {?eh,test_stats,{2,4,{4,5}}}, + {?eh,tc_user_skip,{ts_if_1_SUITE,end_per_group,g3_got_skipped}}], {?eh,tc_start,{ts_if_1_SUITE,{end_per_group,g2,[parallel]}}}, {?eh,tc_done,{ts_if_1_SUITE,{end_per_group,g2,[parallel]},ok}}]}, {?eh,tc_start,{ts_if_1_SUITE,tc12}}, {?eh,tc_done,{ts_if_1_SUITE,tc12,{failed,{testcase_aborted,'stopping tc12'}}}}, - {?eh,test_stats,{2,5,{3,6}}}, + {?eh,test_stats,{2,5,{4,5}}}, {?eh,tc_start,{ts_if_1_SUITE,tc13}}, {?eh,tc_done,{ts_if_1_SUITE,tc13,ok}}, - {?eh,test_stats,{3,5,{3,6}}}, + {?eh,test_stats,{3,5,{4,5}}}, {?eh,tc_start,{ts_if_1_SUITE,end_per_suite}}, {?eh,tc_done,{ts_if_1_SUITE,end_per_suite,ok}}, @@ -223,22 +226,23 @@ test_events(ts_if_1) -> {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}}, {?eh,tc_auto_skip,{ts_if_2_SUITE,my_test_case, {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}}, - {?eh,test_stats,{3,5,{3,7}}}, + {?eh,test_stats,{3,5,{4,6}}}, {?eh,tc_auto_skip,{ts_if_2_SUITE,end_per_suite, {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}}, {?eh,tc_start,{ct_framework,error_in_suite}}, - {?eh,test_stats,{3,5,{4,7}}}, + {?eh,test_stats,{3,6,{4,6}}}, {?eh,tc_start,{ct_framework,error_in_suite}}, - {?eh,test_stats,{3,5,{5,7}}}, + {?eh,test_stats,{3,7,{4,6}}}, {?eh,tc_start,{ts_if_5_SUITE,init_per_suite}}, {?eh,tc_done,{ts_if_5_SUITE,init_per_suite, - {skipped,{require_failed_in_suite0,{not_available,undef_variable}}}}}, + {auto_skipped,{require_failed_in_suite0, + {not_available,undef_variable}}}}}, {?eh,tc_auto_skip,{ts_if_5_SUITE,my_test_case, {require_failed_in_suite0,{not_available,undef_variable}}}}, - {?eh,test_stats,{3,5,{5,8}}}, + {?eh,test_stats,{3,7,{4,7}}}, {?eh,tc_auto_skip,{ts_if_5_SUITE,end_per_suite, {require_failed_in_suite0,{not_available,undef_variable}}}}, @@ -247,23 +251,25 @@ test_events(ts_if_1) -> {failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}}, {?eh,tc_auto_skip,{ts_if_6_SUITE,tc1, {failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}}, - {?eh,test_stats,{3,5,{5,9}}}, + {?eh,test_stats,{3,7,{4,8}}}, + {?eh,tc_auto_skip,{ct_framework,end_per_suite, + {failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}}, {?eh,tc_start,{ts_if_7_SUITE,tc1}}, {?eh,tc_done,{ts_if_7_SUITE,tc1,ok}}, - {?eh,test_stats,{4,5,{5,9}}}, + {?eh,test_stats,{4,7,{4,8}}}, {?eh,tc_start,{ts_if_8_SUITE,tc1}}, {?eh,tc_done,{ts_if_8_SUITE,tc1,{failed,{error,failed_on_purpose}}}}, - {?eh,test_stats,{4,6,{5,9}}}, + {?eh,test_stats,{4,8,{4,8}}}, {?eh,tc_user_skip,{skipped_by_spec_1_SUITE,all,"should be skipped"}}, - {?eh,test_stats,{4,6,{6,9}}}, + {?eh,test_stats,{4,8,{5,8}}}, {?eh,tc_start,{skipped_by_spec_2_SUITE,init_per_suite}}, {?eh,tc_done,{skipped_by_spec_2_SUITE,init_per_suite,ok}}, {?eh,tc_user_skip,{skipped_by_spec_2_SUITE,tc1,"should be skipped"}}, - {?eh,test_stats,{4,6,{7,9}}}, + {?eh,test_stats,{4,8,{6,8}}}, {?eh,tc_start,{skipped_by_spec_2_SUITE,end_per_suite}}, {?eh,tc_done,{skipped_by_spec_2_SUITE,end_per_suite,ok}}, diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl index 67e430f821..772274ce7e 100644 --- a/lib/common_test/test/ct_test_support.erl +++ b/lib/common_test/test/ct_test_support.erl @@ -692,8 +692,10 @@ locate({parallel,TEvs}, Node, Evs, Config) -> test_server:format("Found ~p!", [TEv]), {Done,RemEvs2,length(RemEvs2)} end; - %% end_per_group auto skipped - (TEv={TEH,tc_auto_skip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize}) -> + %% end_per_group auto- or user skipped + (TEv={TEH,AutoOrUserSkip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize}) + when AutoOrUserSkip == tc_auto_skip; + AutoOrUserSkip == tc_user_skip -> RemEvs1 = lists:dropwhile( fun({EH,#event{name=tc_auto_skip, @@ -704,10 +706,18 @@ locate({parallel,TEvs}, Node, Evs, Config) -> match -> false; _ -> true end; + ({EH,#event{name=tc_user_skip, + node=EvNode, + data={Mod,end_per_group,Reason}}}) when + EH == TEH, EvNode == Node, Mod == M -> + case match_data(R, Reason) of + match -> false; + _ -> true + end; ({EH,#event{name=stop_logging, node=EvNode,data=_}}) when EH == TEH, EvNode == Node -> - exit({tc_auto_skip_not_found,TEv}); + exit({tc_auto_or_user_skip_not_found,TEv}); (_) -> true end, RemEvs), @@ -925,8 +935,10 @@ locate({shuffle,TEvs}, Node, Evs, Config) -> test_server:format("Found ~p!", [TEv]), {Done,RemEvs2,length(RemEvs2)} end; - %% end_per_group auto skipped - (TEv={TEH,tc_auto_skip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize}) -> + %% end_per_group auto-or user skipped + (TEv={TEH,AutoOrUserSkip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize}) + when AutoOrUserSkip == tc_auto_skip; + AutoOrUserSkip == tc_user_skip -> RemEvs1 = lists:dropwhile( fun({EH,#event{name=tc_auto_skip, @@ -934,6 +946,11 @@ locate({shuffle,TEvs}, Node, Evs, Config) -> data={Mod,end_per_group,Reason}}}) when EH == TEH, EvNode == Node, Mod == M, Reason == R -> false; + ({EH,#event{name=tc_user_skip, + node=EvNode, + data={Mod,end_per_group,Reason}}}) when + EH == TEH, EvNode == Node, Mod == M, Reason == R -> + false; ({EH,#event{name=stop_logging, node=EvNode,data=_}}) when EH == TEH, EvNode == Node -> @@ -1178,6 +1195,9 @@ log_events1([E={_EH,tc_done,{_M,{end_per_group,_GrName,Props},_R}} | Evs], Dev, log_events1([E={_EH,tc_auto_skip,{_M,end_per_group,_Reason}} | Evs], Dev, Ind) -> io:format(Dev, "~s~p],~n", [Ind,E]), log_events1(Evs, Dev, Ind--" "); +log_events1([E={_EH,tc_user_skip,{_M,end_per_group,_Reason}} | Evs], Dev, Ind) -> + io:format(Dev, "~s~p],~n", [Ind,E]), + log_events1(Evs, Dev, Ind--" "); log_events1([E], Dev, Ind) -> io:format(Dev, "~s~p~n].~n", [Ind,E]), ok; |