diff options
author | Peter Andersson <[email protected]> | 2013-11-28 17:37:25 +0100 |
---|---|---|
committer | Peter Andersson <[email protected]> | 2013-11-28 17:37:33 +0100 |
commit | ac7902caa784d4f4c73eb7291ab4736a75ad13a4 (patch) | |
tree | 7c292d5ffd0907a42efb49e8ab1c60f4ab515649 /lib/common_test/test/ct_group_info_SUITE.erl | |
parent | 9f998969014095333545cb118467e1ff8d450c31 (diff) | |
parent | 093890dc793e85a09b40f1eca878f410c73cf625 (diff) | |
download | otp-ac7902caa784d4f4c73eb7291ab4736a75ad13a4.tar.gz otp-ac7902caa784d4f4c73eb7291ab4736a75ad13a4.tar.bz2 otp-ac7902caa784d4f4c73eb7291ab4736a75ad13a4.zip |
Merge remote branch 'origin/peppe/common_test/change_skip_behaviour' into maint
* origin/peppe/common_test/change_skip_behaviour:
Modify the auto_skip report for group config funcs to include group name
Fix problems with info functions and add more tests
Add test cases for new and modified functionality
Fix problem with suites and groups skipped from test specification
Change report tag for failed init_per_testcase from skipped to auto_skipped
Add tests for skipping parallel groups
Correct tests cases that fail because of modified events
Correct various bugs related to auto_skip and groups
Change status from skip to auto_skip for config func that fails due to require
Fix problem with handling Config and FW reports correctly
Change skip vs auto_skip behaviour
OTP-11305
Diffstat (limited to 'lib/common_test/test/ct_group_info_SUITE.erl')
-rw-r--r-- | lib/common_test/test/ct_group_info_SUITE.erl | 156 |
1 files changed, 94 insertions, 62 deletions
diff --git a/lib/common_test/test/ct_group_info_SUITE.erl b/lib/common_test/test/ct_group_info_SUITE.erl index c56fa952e8..e7bc5baaa1 100644 --- a/lib/common_test/test/ct_group_info_SUITE.erl +++ b/lib/common_test/test/ct_group_info_SUITE.erl @@ -269,13 +269,19 @@ test_events(timetrap_all) -> {?eh,tc_done,{group_timetrap_1_SUITE,{end_per_group,g10,[]},ok}}], [{?eh,tc_start,{group_timetrap_1_SUITE,{init_per_group,g11,[]}}}, - {?eh,tc_done,{group_timetrap_1_SUITE,{init_per_group,g11,[]},ok}}, - {?eh,tc_done,{group_timetrap_1_SUITE,t111,{failed,{timetrap_timeout,1000}}}}, - {?eh,test_stats,{0,14,{0,0}}}, - {?eh,tc_start,{group_timetrap_1_SUITE,{end_per_group,g11,[]}}}, - {?eh,tc_done,{group_timetrap_1_SUITE,{end_per_group,g11,[]},ok}}], - + {?eh,tc_done,{group_timetrap_1_SUITE, + {init_per_group,g11,[]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,tc_auto_skip, + {group_timetrap_1_SUITE,t111,{group0_failed,bad_return_value}}}, + {?eh,test_stats,{0,13,{0,1}}}, + {?eh,tc_auto_skip,{group_timetrap_1_SUITE, + {end_per_group,g11}, + {group0_failed,bad_return_value}}}], + + {?eh,tc_start,{group_timetrap_1_SUITE,end_per_suite}}, {?eh,tc_done,{group_timetrap_1_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} ]; @@ -422,13 +428,15 @@ test_events(timetrap_all_no_ips) -> {?eh,tc_done,{group_timetrap_2_SUITE,{end_per_group,g10,[]},ok}}], [{?eh,tc_start,{group_timetrap_2_SUITE,{init_per_group,g11,[]}}}, - {?eh,tc_done,{group_timetrap_2_SUITE,{init_per_group,g11,[]},ok}}, - {?eh,tc_done,{group_timetrap_2_SUITE,t111,{failed,{timetrap_timeout,1000}}}}, - {?eh,test_stats,{0,14,{0,0}}}, - {?eh,tc_start,{group_timetrap_2_SUITE,{end_per_group,g11,[]}}}, - {?eh,tc_done,{group_timetrap_2_SUITE,{end_per_group,g11,[]},ok}}], - - {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,tc_done,{group_timetrap_2_SUITE, + {init_per_group,g11,[]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,tc_auto_skip,{group_timetrap_2_SUITE,t111, + {group0_failed,bad_return_value}}}, + {?eh,test_stats,{0,13,{0,1}}}, + {?eh,tc_auto_skip,{group_timetrap_2_SUITE, + {end_per_group,g11}, + {group0_failed,bad_return_value}}}], {?eh,stop_logging,[]} ]; @@ -501,11 +509,13 @@ test_events(timetrap_all_no_ipg) -> {?eh,tc_done,{ct_framework,{end_per_group,g10,[{suite,group_timetrap_3_SUITE}]},ok}}], [{?eh,tc_start,{ct_framework,{init_per_group,g11,[{suite,group_timetrap_3_SUITE}]}}}, - {?eh,tc_done,{ct_framework,{init_per_group,g11,[{suite,group_timetrap_3_SUITE}]},ok}}, - {?eh,tc_done,{group_timetrap_3_SUITE,t111,{failed,{timetrap_timeout,1000}}}}, - {?eh,test_stats,{0,14,{0,0}}}, - {?eh,tc_start,{ct_framework,{end_per_group,g11,[{suite,group_timetrap_3_SUITE}]}}}, - {?eh,tc_done,{ct_framework,{end_per_group,g11,[{suite,group_timetrap_3_SUITE}]},ok}}], + {?eh,tc_done,{ct_framework, + {init_per_group,g11,[{suite,group_timetrap_3_SUITE}]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,tc_auto_skip,{group_timetrap_3_SUITE,t111,{group0_failed,bad_return_value}}}, + {?eh,test_stats,{0,13,{0,1}}}, + {?eh,tc_auto_skip,{ct_framework,{end_per_group,g11}, + {group0_failed,bad_return_value}}}], {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} @@ -539,11 +549,13 @@ test_events(require) -> [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g4,[]}}}, {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g4,[]}, - {skipped,{require_failed,{name_in_use,common2_alias,common2}}}}}, + {auto_skipped,{require_failed, + {name_in_use,common2_alias,common2}}}}}, {?eh,tc_auto_skip,{group_require_1_SUITE,t41, - {require_failed,{name_in_use,common2_alias,common2}}}}, + {require_failed, + {name_in_use,common2_alias,common2}}}}, {?eh,test_stats,{4,0,{0,1}}}, - {?eh,tc_auto_skip,{group_require_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{group_require_1_SUITE,{end_per_group,g4}, {require_failed,{name_in_use,common2_alias,common2}}}}], [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g5,[]}}}, @@ -566,16 +578,18 @@ test_events(require) -> [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g8,[]}}}, {?eh,tc_done,{group_require_1_SUITE, {init_per_group,g8,[]}, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,tc_auto_skip,{group_require_1_SUITE,t81, {require_failed,{not_available,non_existing}}}}, {?eh,test_stats,{8,0,{0,2}}}, - {?eh,tc_auto_skip,{group_require_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{group_require_1_SUITE,{end_per_group,g8}, {require_failed,{not_available,non_existing}}}}], [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g9,[]}}}, {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g9,[]},ok}}, {?eh,tc_done,{group_require_1_SUITE,t91, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,test_stats,{8,0,{0,3}}}, {?eh,tc_start,{group_require_1_SUITE,{end_per_group,g9,[]}}}, {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g9,[]},ok}}], @@ -587,12 +601,16 @@ test_events(require) -> {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g10,[]},ok}}], [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g11,[]}}}, - {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g11,[]},ok}}, - {?eh,tc_done,{group_require_1_SUITE,t111,ok}}, - {?eh,test_stats,{10,0,{0,3}}}, - {?eh,tc_start,{group_require_1_SUITE,{end_per_group,g11,[]}}}, - {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g11,[]},ok}}], - + {?eh,tc_done,{group_require_1_SUITE, + {init_per_group,g11,[]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,tc_auto_skip,{group_require_1_SUITE,t111, + {group0_failed,bad_return_value}}}, + {?eh,test_stats,{9,0,{0,4}}}, + {?eh,tc_auto_skip,{group_require_1_SUITE, + {end_per_group,g11}, + {group0_failed,bad_return_value}}}], + {?eh,tc_done,{group_require_1_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} @@ -627,11 +645,11 @@ test_events(require_default) -> [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g4,[]}}}, {?eh,tc_done,{group_require_1_SUITE, {init_per_group,g4,[]}, - {skipped,{require_failed,{not_available,common3}}}}}, + {auto_skipped,{require_failed,{not_available,common3}}}}}, {?eh,tc_auto_skip,{group_require_1_SUITE,t41, {require_failed,{not_available,common3}}}}, {?eh,test_stats,{4,0,{0,1}}}, - {?eh,tc_auto_skip,{group_require_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{group_require_1_SUITE,{end_per_group,g4}, {require_failed,{not_available,common3}}}}], [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g5,[]}}}, @@ -654,17 +672,19 @@ test_events(require_default) -> [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g8,[]}}}, {?eh,tc_done,{group_require_1_SUITE, {init_per_group,g8,[]}, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,tc_auto_skip,{group_require_1_SUITE,t81, {require_failed,{not_available,non_existing}}}}, {?eh,test_stats,{8,0,{0,2}}}, - {?eh,tc_auto_skip,{group_require_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{group_require_1_SUITE,{end_per_group,g8}, {require_failed,{not_available,non_existing}}}}], [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g9,[]}}}, {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g9,[]},ok}}, {?eh,tc_done,{group_require_1_SUITE,t91, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,test_stats,{8,0,{0,3}}}, {?eh,tc_start,{group_require_1_SUITE,{end_per_group,g9,[]}}}, {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g9,[]},ok}}], @@ -676,11 +696,15 @@ test_events(require_default) -> {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g10,[]},ok}}], [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g11,[]}}}, - {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g11,[]},ok}}, - {?eh,tc_done,{group_require_1_SUITE,t111,ok}}, - {?eh,test_stats,{10,0,{0,3}}}, - {?eh,tc_start,{group_require_1_SUITE,{end_per_group,g11,[]}}}, - {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g11,[]},ok}}], + {?eh,tc_done,{group_require_1_SUITE, + {init_per_group,g11,[]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,tc_auto_skip,{group_require_1_SUITE,t111, + {group0_failed,bad_return_value}}}, + {?eh,test_stats,{9,0,{0,4}}}, + {?eh,tc_auto_skip,{group_require_1_SUITE, + {end_per_group,g11}, + {group0_failed,bad_return_value}}}], {?eh,tc_done,{group_require_1_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, @@ -714,11 +738,12 @@ test_events(require_no_ips) -> [{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g4,[]}}}, {?eh,tc_done,{group_require_2_SUITE,{init_per_group,g4,[]}, - {skipped,{require_failed,{name_in_use,common2_alias,common2}}}}}, + {auto_skipped,{require_failed, + {name_in_use,common2_alias,common2}}}}}, {?eh,tc_auto_skip,{group_require_2_SUITE,t41, {require_failed,{name_in_use,common2_alias,common2}}}}, {?eh,test_stats,{4,0,{0,1}}}, - {?eh,tc_auto_skip,{group_require_2_SUITE,end_per_group, + {?eh,tc_auto_skip,{group_require_2_SUITE,{end_per_group,g4}, {require_failed,{name_in_use,common2_alias,common2}}}}], [{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g5,[]}}}, @@ -741,16 +766,18 @@ test_events(require_no_ips) -> [{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g8,[]}}}, {?eh,tc_done,{group_require_2_SUITE, {init_per_group,g8,[]}, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,tc_auto_skip,{group_require_2_SUITE,t81, {require_failed,{not_available,non_existing}}}}, {?eh,test_stats,{8,0,{0,2}}}, - {?eh,tc_auto_skip,{group_require_2_SUITE,end_per_group, + {?eh,tc_auto_skip,{group_require_2_SUITE,{end_per_group,g8}, {require_failed,{not_available,non_existing}}}}], [{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g9,[]}}}, {?eh,tc_done,{group_require_2_SUITE,{init_per_group,g9,[]},ok}}, {?eh,tc_done,{group_require_2_SUITE,t91, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,test_stats,{8,0,{0,3}}}, {?eh,tc_start,{group_require_2_SUITE,{end_per_group,g9,[]}}}, {?eh,tc_done,{group_require_2_SUITE,{end_per_group,g9,[]},ok}}], @@ -760,13 +787,17 @@ test_events(require_no_ips) -> {?eh,tc_done,{group_require_2_SUITE,t101,ok}}, {?eh,tc_start,{group_require_2_SUITE,{end_per_group,g10,[]}}}, {?eh,tc_done,{group_require_2_SUITE,{end_per_group,g10,[]},ok}}], - + [{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g11,[]}}}, - {?eh,tc_done,{group_require_2_SUITE,{init_per_group,g11,[]},ok}}, - {?eh,tc_done,{group_require_2_SUITE,t111,ok}}, - {?eh,test_stats,{10,0,{0,3}}}, - {?eh,tc_start,{group_require_2_SUITE,{end_per_group,g11,[]}}}, - {?eh,tc_done,{group_require_2_SUITE,{end_per_group,g11,[]},ok}}], + {?eh,tc_done,{group_require_2_SUITE, + {init_per_group,g11,[]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,tc_auto_skip,{group_require_2_SUITE,t111, + {group0_failed,bad_return_value}}}, + {?eh,test_stats,{9,0,{0,4}}}, + {?eh,tc_auto_skip,{group_require_2_SUITE, + {end_per_group,g11}, + {group0_failed,bad_return_value}}}], {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} @@ -799,11 +830,11 @@ test_events(require_no_ipg) -> [{?eh,tc_start,{ct_framework,{init_per_group,g4,[{suite,group_require_3_SUITE}]}}}, {?eh,tc_done,{ct_framework,{init_per_group,g4,[{suite,group_require_3_SUITE}]}, - {skipped,{require_failed,{name_in_use,common2_alias,common2}}}}}, + {auto_skipped,{require_failed,{name_in_use,common2_alias,common2}}}}}, {?eh,tc_auto_skip,{group_require_3_SUITE,t41, {require_failed,{name_in_use,common2_alias,common2}}}}, {?eh,test_stats,{4,0,{0,1}}}, - {?eh,tc_auto_skip,{ct_framework,end_per_group, + {?eh,tc_auto_skip,{ct_framework,{end_per_group,g4}, {require_failed,{name_in_use,common2_alias,common2}}}}], [{?eh,tc_start,{ct_framework,{init_per_group,g5,[{suite,group_require_3_SUITE}]}}}, @@ -825,16 +856,16 @@ test_events(require_no_ipg) -> [{?eh,tc_start,{ct_framework,{init_per_group,g8,[{suite,group_require_3_SUITE}]}}}, {?eh,tc_done,{ct_framework,{init_per_group,g8,[{suite,group_require_3_SUITE}]}, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed,{not_available,non_existing}}}}}, {?eh,tc_auto_skip,{group_require_3_SUITE,t81, {require_failed,{not_available,non_existing}}}}, {?eh,test_stats,{8,0,{0,2}}}, - {?eh,tc_auto_skip,{ct_framework,end_per_group, + {?eh,tc_auto_skip,{ct_framework,{end_per_group,g8}, {require_failed,{not_available,non_existing}}}}], [{?eh,tc_start,{ct_framework,{init_per_group,g9,[{suite,group_require_3_SUITE}]}}}, {?eh,tc_done,{ct_framework,{init_per_group,g9,[{suite,group_require_3_SUITE}]},ok}}, {?eh,tc_done,{group_require_3_SUITE,t91, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed,{not_available,non_existing}}}}}, {?eh,test_stats,{8,0,{0,3}}}, {?eh,tc_start,{ct_framework,{end_per_group,g9,[{suite,group_require_3_SUITE}]}}}, {?eh,tc_done,{ct_framework,{end_per_group,g9,[{suite,group_require_3_SUITE}]},ok}}], @@ -844,13 +875,14 @@ test_events(require_no_ipg) -> {?eh,tc_done,{group_require_3_SUITE,t101,ok}}, {?eh,tc_start,{ct_framework,{end_per_group,g10,[{suite,group_require_3_SUITE}]}}}, {?eh,tc_done,{ct_framework,{end_per_group,g10,[{suite,group_require_3_SUITE}]},ok}}], - + [{?eh,tc_start,{ct_framework,{init_per_group,g11,[{suite,group_require_3_SUITE}]}}}, - {?eh,tc_done,{ct_framework,{init_per_group,g11,[{suite,group_require_3_SUITE}]},ok}}, - {?eh,tc_done,{group_require_3_SUITE,t111,ok}}, - {?eh,test_stats,{10,0,{0,3}}}, - {?eh,tc_start,{ct_framework,{end_per_group,g11,[{suite,group_require_3_SUITE}]}}}, - {?eh,tc_done,{ct_framework,{end_per_group,g11,[{suite,group_require_3_SUITE}]},ok}}], + {?eh,tc_done,{ct_framework,{init_per_group,g11,[{suite,group_require_3_SUITE}]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,tc_auto_skip,{group_require_3_SUITE,t111,{group0_failed,bad_return_value}}}, + {?eh,test_stats,{9,0,{0,4}}}, + {?eh,tc_auto_skip,{ct_framework,{end_per_group,g11}, + {group0_failed,bad_return_value}}}], {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} |