From 759a1f6240117cf64eecac26c21f82de0e877332 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Thu, 31 Oct 2013 16:49:59 +0100 Subject: Fix problem with handling Config and FW reports correctly --- lib/common_test/src/ct_framework.erl | 14 +++-- lib/common_test/test/ct_skip_SUITE.erl | 92 ++++++++++++++++---------------- lib/test_server/src/test_server_ctrl.erl | 4 +- 3 files changed, 58 insertions(+), 52 deletions(-) (limited to 'lib') diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 276f902b05..76516e6a72 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -1256,14 +1256,20 @@ report(What,Data) -> {_,{SkipOrFail,_Reason}} -> add_to_stats(SkipOrFail) end; - tc_user_skip -> - %% test case specified as skipped in testspec + tc_user_skip -> + %% test case specified as skipped in testspec, or init + %% config func for suite/group has returned {skip,Reason} %% Data = {Suite,Case,Comment} ct_event:sync_notify(#event{name=tc_user_skip, node=node(), data=Data}), - ct_hooks:on_tc_skip(What, Data), - add_to_stats(user_skipped); + case Data of + {_,Func,_} when Func /= end_per_suite, Func /= end_per_group -> + ct_hooks:on_tc_skip(What, Data), + add_to_stats(user_skipped); + _ -> + ok + end; tc_auto_skip -> %% test case skipped because of error in init_per_suite %% Data = {Suite,Case,Comment} diff --git a/lib/common_test/test/ct_skip_SUITE.erl b/lib/common_test/test/ct_skip_SUITE.erl index b8be55f43a..35ac483309 100644 --- a/lib/common_test/test/ct_skip_SUITE.erl +++ b/lib/common_test/test/ct_skip_SUITE.erl @@ -447,41 +447,41 @@ test_events(user_skip) -> {?eh,tc_start,{user_skip_1_SUITE,init_per_suite}}, {?eh,tc_done, {user_skip_1_SUITE,init_per_suite,{skipped,"Whole suite skipped"}}}, - {?eh,tc_auto_skip, + {?eh,tc_user_skip, {user_skip_1_SUITE,tc1,"Whole suite skipped"}}, - {?eh,test_stats,{0,0,{0,1}}}, - {?eh,tc_auto_skip, + {?eh,test_stats,{0,0,{1,0}}}, + {?eh,tc_user_skip, {user_skip_1_SUITE,tc2,"Whole suite skipped"}}, - {?eh,test_stats,{0,0,{0,2}}}, - {?eh,tc_auto_skip, + {?eh,test_stats,{0,0,{2,0}}}, + {?eh,tc_user_skip, {user_skip_1_SUITE,tc3,"Whole suite skipped"}}, - {?eh,test_stats,{0,0,{0,3}}}, - {?eh,tc_auto_skip, + {?eh,test_stats,{0,0,{3,0}}}, + {?eh,tc_user_skip, {user_skip_1_SUITE,tc4,"Whole suite skipped"}}, - {?eh,test_stats,{0,0,{0,4}}}, - {?eh,tc_auto_skip, + {?eh,test_stats,{0,0,{4,0}}}, + {?eh,tc_user_skip, {user_skip_1_SUITE,end_per_suite,"Whole suite skipped"}}, {?eh,tc_start,{user_skip_2_SUITE,init_per_suite}}, {?eh,tc_done,{user_skip_2_SUITE,init_per_suite,ok}}, {?eh,tc_start,{user_skip_2_SUITE,tc1}}, {?eh,tc_done,{user_skip_2_SUITE,tc1,{skipped,{tc1,skipped}}}}, - {?eh,test_stats,{0,0,{1,4}}}, + {?eh,test_stats,{0,0,{5,0}}}, [{?eh,tc_start,{user_skip_2_SUITE,{init_per_group,g1,[]}}}, {?eh,tc_done,{user_skip_2_SUITE,{init_per_group,g1,[]},ok}}, {?eh,tc_start,{user_skip_2_SUITE,tc2}}, {?eh,tc_done,{user_skip_2_SUITE,tc2,ok}}, - {?eh,test_stats,{1,0,{1,4}}}, + {?eh,test_stats,{1,0,{5,0}}}, {?eh,tc_start,{user_skip_2_SUITE,tc3}}, {?eh,tc_done,{user_skip_2_SUITE,tc3,{skipped,{tc3,skipped}}}}, - {?eh,test_stats,{1,0,{2,4}}}, + {?eh,test_stats,{1,0,{6,0}}}, {?eh,tc_start,{user_skip_2_SUITE,{end_per_group,g1,[]}}}, {?eh,tc_done,{user_skip_2_SUITE,{end_per_group,g1,[]},ok}}], {?eh,tc_start,{user_skip_2_SUITE,tc4}}, {?eh,tc_done,{user_skip_2_SUITE,tc4,ok}}, - {?eh,test_stats,{2,0,{2,4}}}, + {?eh,test_stats,{2,0,{6,0}}}, {?eh,tc_start,{user_skip_2_SUITE,end_per_suite}}, {?eh,tc_done,{user_skip_2_SUITE,end_per_suite,ok}}, @@ -489,16 +489,16 @@ test_events(user_skip) -> {?eh,tc_done,{user_skip_3_SUITE,init_per_suite,ok}}, {?eh,tc_start,{user_skip_3_SUITE,tc1}}, {?eh,tc_done,{user_skip_3_SUITE,tc1,{skipped,"Test case skipped"}}}, - {?eh,test_stats,{2,0,{3,4}}}, + {?eh,test_stats,{2,0,{7,0}}}, [{?eh,tc_start,{user_skip_3_SUITE,{init_per_group,g1,[]}}}, {?eh,tc_done,{user_skip_3_SUITE,{init_per_group,g1,[]},ok}}, {?eh,tc_start,{user_skip_3_SUITE,tc2}}, {?eh,tc_done,{user_skip_3_SUITE,tc2,ok}}, - {?eh,test_stats,{3,0,{3,4}}}, + {?eh,test_stats,{3,0,{7,0}}}, {?eh,tc_start,{user_skip_3_SUITE,tc3}}, {?eh,tc_done,{user_skip_3_SUITE,tc3,{skipped,"Test case skipped"}}}, - {?eh,test_stats,{3,0,{4,4}}}, + {?eh,test_stats,{3,0,{8,0}}}, {?eh,tc_start,{user_skip_3_SUITE,{end_per_group,g1,[]}}}, {?eh,tc_done,{user_skip_3_SUITE,{end_per_group,g1,[]},ok}}], @@ -506,7 +506,7 @@ test_events(user_skip) -> {?eh,tc_done,{user_skip_3_SUITE,tc4, {skipped,{proc_info,{{current_function,{user_skip_3_SUITE,tc4,1}}, {initial_call,{erlang,apply,2}}}}}}}, - {?eh,test_stats,{3,0,{5,4}}}, + {?eh,test_stats,{3,0,{9,0}}}, {?eh,tc_start,{user_skip_3_SUITE,end_per_suite}}, {?eh,tc_done,{user_skip_3_SUITE,end_per_suite,ok}}, @@ -515,62 +515,62 @@ test_events(user_skip) -> [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g1,[]}}}, {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g1,[]},{skipped,"Group skipped"}}}, - {?eh,tc_auto_skip,{user_skip_4_SUITE,tc1,"Group skipped"}}, - {?eh,test_stats,{3,0,{5,5}}}, - {?eh,tc_auto_skip,{user_skip_4_SUITE,tc2,"Group skipped"}}, - {?eh,test_stats,{3,0,{5,6}}}, - {?eh,tc_auto_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}], + {?eh,tc_user_skip,{user_skip_4_SUITE,tc1,"Group skipped"}}, + {?eh,test_stats,{3,0,{10,0}}}, + {?eh,tc_user_skip,{user_skip_4_SUITE,tc2,"Group skipped"}}, + {?eh,test_stats,{3,0,{11,0}}}, + {?eh,tc_user_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}], [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g2,[]}}}, {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g2,[]},ok}}, {?eh,tc_start,{user_skip_4_SUITE,tc3}}, {?eh,tc_done,{user_skip_4_SUITE,tc3,ok}}, - {?eh,test_stats,{4,0,{5,6}}}, + {?eh,test_stats,{4,0,{11,0}}}, {?eh,tc_start,{user_skip_4_SUITE,tc4}}, {?eh,tc_done,{user_skip_4_SUITE,tc4,ok}}, - {?eh,test_stats,{5,0,{5,6}}}, + {?eh,test_stats,{5,0,{11,0}}}, {?eh,tc_start,{user_skip_4_SUITE,{end_per_group,g2,[]}}}, {?eh,tc_done,{user_skip_4_SUITE,{end_per_group,g2,[]},ok}}], [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g3,[]}}}, {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g3,[]},{skipped,"Group skipped"}}}, - {?eh,tc_auto_skip,{user_skip_4_SUITE,tc5,"Group skipped"}}, - {?eh,tc_auto_skip,{user_skip_4_SUITE,tc6,"Group skipped"}}, - {?eh,tc_auto_skip,{user_skip_4_SUITE,tc7,"Group skipped"}}, - {?eh,tc_auto_skip,{user_skip_4_SUITE,tc8,"Group skipped"}}, - {?eh,test_stats,{5,0,{5,10}}}, - {?eh,tc_auto_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}], + {?eh,tc_user_skip,{user_skip_4_SUITE,tc5,"Group skipped"}}, + {?eh,tc_user_skip,{user_skip_4_SUITE,tc6,"Group skipped"}}, + {?eh,tc_user_skip,{user_skip_4_SUITE,tc7,"Group skipped"}}, + {?eh,tc_user_skip,{user_skip_4_SUITE,tc8,"Group skipped"}}, + {?eh,test_stats,{5,0,{15,0}}}, + {?eh,tc_user_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}], [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g5,[]}}}, {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g5,[]},ok}}, {?eh,tc_start,{user_skip_4_SUITE,tc9}}, {?eh,tc_done,{user_skip_4_SUITE,tc9,ok}}, - {?eh,test_stats,{6,0,{5,10}}}, + {?eh,test_stats,{6,0,{15,0}}}, [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g6,[]}}}, {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g6,[]},{skipped,"Group skipped"}}}, - {?eh,tc_auto_skip,{user_skip_4_SUITE,tc10,"Group skipped"}}, - {?eh,test_stats,{6,0,{5,11}}}, - {?eh,tc_auto_skip,{user_skip_4_SUITE,tc11,"Group skipped"}}, - {?eh,test_stats,{6,0,{5,12}}}, - {?eh,tc_auto_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}], + {?eh,tc_user_skip,{user_skip_4_SUITE,tc10,"Group skipped"}}, + {?eh,test_stats,{6,0,{16,0}}}, + {?eh,tc_user_skip,{user_skip_4_SUITE,tc11,"Group skipped"}}, + {?eh,test_stats,{6,0,{17,0}}}, + {?eh,tc_user_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}], {?eh,tc_start,{user_skip_4_SUITE,{end_per_group,g5,[]}}}, {?eh,tc_done,{user_skip_4_SUITE,{end_per_group,g5,[]},ok}}], {?eh,tc_start,{user_skip_4_SUITE,end_per_suite}}, {?eh,tc_done,{user_skip_4_SUITE,end_per_suite,ok}}, - {ct_test_support_eh,tc_start,{user_skip_5_SUITE,init_per_suite}}, + {?eh,tc_start,{user_skip_5_SUITE,init_per_suite}}, {?eh,tc_done,{user_skip_5_SUITE,init_per_suite, {skipped,{bad,'Whole suite skipped'}}}}, - {?eh,tc_auto_skip,{user_skip_5_SUITE,tc1,{bad,'Whole suite skipped'}}}, - {?eh,test_stats,{6,0,{5,13}}}, - {?eh,tc_auto_skip,{user_skip_5_SUITE,tc2,{bad,'Whole suite skipped'}}}, - {?eh,test_stats,{6,0,{5,14}}}, - {?eh,tc_auto_skip,{user_skip_5_SUITE,tc3,{bad,'Whole suite skipped'}}}, - {?eh,test_stats,{6,0,{5,15}}}, - {?eh,tc_auto_skip,{user_skip_5_SUITE,tc4,{bad,'Whole suite skipped'}}}, - {?eh,test_stats,{6,0,{5,16}}}, - {?eh,tc_auto_skip,{user_skip_5_SUITE,end_per_suite,{bad,'Whole suite skipped'}}}, + {?eh,tc_user_skip,{user_skip_5_SUITE,tc1,{bad,'Whole suite skipped'}}}, + {?eh,test_stats,{6,0,{18,0}}}, + {?eh,tc_user_skip,{user_skip_5_SUITE,tc2,{bad,'Whole suite skipped'}}}, + {?eh,test_stats,{6,0,{19,0}}}, + {?eh,tc_user_skip,{user_skip_5_SUITE,tc3,{bad,'Whole suite skipped'}}}, + {?eh,test_stats,{6,0,{20,0}}}, + {?eh,tc_user_skip,{user_skip_5_SUITE,tc4,{bad,'Whole suite skipped'}}}, + {?eh,test_stats,{6,0,{21,0}}}, + {?eh,tc_user_skip,{user_skip_5_SUITE,end_per_suite,{bad,'Whole suite skipped'}}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index 9ab51df028..7b7cd24bff 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -2662,7 +2662,7 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, stop_minor_log_file(), run_test_cases_loop(skip_cases_upto(Ref, Cases, Reason, conf, CurrMode, skip_case), - Config, TimetrapData, Mode, + [hd(Config)|Config], TimetrapData, Mode, delete_status(Ref, Status2)); {_,{skip_and_save,Reason,_SavedConfig},_} when StartConf -> ReportAbortRepeat(skipped), @@ -2672,7 +2672,7 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, stop_minor_log_file(), run_test_cases_loop(skip_cases_upto(Ref, Cases, Reason, conf, CurrMode, skip_case), - Config, TimetrapData, Mode, + [hd(Config)|Config], TimetrapData, Mode, delete_status(Ref, Status2)); {_,_Other,_} when Func == init_per_suite -> print(minor, "~n*** init_per_suite failed to return a Config list.~n", []), -- cgit v1.2.3