diff options
| -rw-r--r-- | lib/common_test/src/test_server_ctrl.erl | 53 | ||||
| -rw-r--r-- | lib/common_test/test/ct_error_SUITE.erl | 8 | ||||
| -rw-r--r-- | lib/common_test/test/ct_hooks_SUITE.erl | 88 | ||||
| -rw-r--r-- | lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl | 19 | ||||
| -rw-r--r-- | lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl | 15 |
5 files changed, 153 insertions, 30 deletions
diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl index d3a0f74f7a..39c523f8b3 100644 --- a/lib/common_test/src/test_server_ctrl.erl +++ b/lib/common_test/src/test_server_ctrl.erl @@ -2939,22 +2939,21 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status) exit(framework_error); %% sequential execution of test case finished {Time,RetVal,_} -> + RetTag = + if is_tuple(RetVal) -> element(1,RetVal); + true -> undefined + end, {Failed,Status1} = - case Time of - died -> - {true,update_status(failed, Mod, Func, Status)}; - _ when is_tuple(RetVal) -> - case element(1, RetVal) of - R when R=='EXIT'; R==failed -> - {true,update_status(failed, Mod, Func, Status)}; - R when R==skip; R==skipped -> - {false,update_status(skipped, Mod, Func, Status)}; - _ -> - {false,update_status(ok, Mod, Func, Status)} - end; - _ -> - {false,update_status(ok, Mod, Func, Status)} - end, + case RetTag of + Skip when Skip==skip; Skip==skipped -> + {false,update_status(skipped, Mod, Func, Status)}; + Fail when Fail=='EXIT'; Fail==failed -> + {true,update_status(failed, Mod, Func, Status)}; + _ when Time==died, RetVal=/=ok -> + {true,update_status(failed, Mod, Func, Status)}; + _ -> + {false,update_status(ok, Mod, Func, Status)} + end, case check_prop(sequence, Mode) of false -> stop_minor_log_file(), @@ -3809,7 +3808,15 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, {died,{timetrap_timeout,TimetrapTimeout}} -> progress(failed, Num, Mod, Func, GrName, Loc, timetrap_timeout, TimetrapTimeout, Comment, Style); - {died,Reason} -> + {died,{Skip,Reason}} when Skip==skip; Skip==skipped -> + %% died in init_per_testcase + progress(skip, Num, Mod, Func, GrName, Loc, Reason, + Time, Comment, Style); + {died,Reason} when Reason=/=ok -> + %% (If Reason==ok it means that process died in + %% end_per_testcase after successfully completing the + %% test case itself - then we shall not fail, but a + %% warning will be issued in the comment field.) progress(failed, Num, Mod, Func, GrName, Loc, Reason, Time, Comment, Style); {_,{'EXIT',{Skip,Reason}}} when Skip==skip; Skip==skipped; @@ -3958,6 +3965,9 @@ progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time, [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName}, {ReportTag,Reason1}}]), + TimeStr = io_lib:format(if is_float(Time) -> "~.3fs"; + true -> "~w" + end, [Time]), ReasonStr = escape_chars(reason_to_string(Reason1)), ReasonStr1 = lists:flatten([string:strip(S,left) || S <- string:tokens(ReasonStr,[$\n])]), @@ -3972,10 +3982,10 @@ progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time, _ -> xhtml("<br>(","<br />(") ++ to_string(Comment) ++ ")" end, print(html, - "<td>" ++ St0 ++ "~.3fs" ++ St1 ++ "</td>" + "<td>" ++ St0 ++ "~ts" ++ St1 ++ "</td>" "<td><font color=\"~ts\">SKIPPED</font></td>" "<td>~ts~ts</td></tr>\n", - [Time,Color,ReasonStr2,Comment1]), + [TimeStr,Color,ReasonStr2,Comment1]), FormatLoc = test_server_sup:format_loc(Loc), print(minor, "=== Location: ~ts", [FormatLoc]), print(minor, "=== Reason: ~ts", [ReasonStr1]), @@ -4113,6 +4123,9 @@ progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time, Comment0, {St0,St1}) -> print(minor, "successfully completed test case", []), test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName},ok}]), + TimeStr = io_lib:format(if is_float(Time) -> "~.3fs"; + true -> "~w" + end, [Time]), Comment = case RetVal of {comment,RetComment} -> @@ -4131,10 +4144,10 @@ progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time, end, print(major, "=elapsed ~p", [Time]), print(html, - "<td>" ++ St0 ++ "~.3fs" ++ St1 ++ "</td>" + "<td>" ++ St0 ++ "~ts" ++ St1 ++ "</td>" "<td><font color=\"green\">Ok</font></td>" "~ts</tr>\n", - [Time,Comment]), + [TimeStr,Comment]), print(minor, escape_chars(io_lib:format("=== Returned value: ~tp", [RetVal])), []), diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index fae23484e6..621f3b6d2d 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -1531,17 +1531,17 @@ test_events(config_func_errors) -> {?eh,tc_start,{config_func_error_1_SUITE,exit_in_iptc}}, {?eh,tc_done,{config_func_error_1_SUITE,exit_in_iptc,'_'}}, - {?eh,test_stats,{0,1,{0,0}}}, + {?eh,test_stats,{0,0,{0,1}}}, {?eh,tc_start,{config_func_error_1_SUITE,exit_in_eptc}}, {?eh,tc_done,{config_func_error_1_SUITE,exit_in_eptc,'_'}}, - {?eh,test_stats,{0,2,{0,0}}}, + {?eh,test_stats,{1,0,{0,1}}}, [{?eh,tc_start,{config_func_error_1_SUITE,{init_per_group,g1,[]}}}, {?eh,tc_done,{config_func_error_1_SUITE,{init_per_group,g1,[]},ok}}, {?eh,tc_start,{config_func_error_1_SUITE,exit_in_iptc}}, {?eh,tc_done,{config_func_error_1_SUITE,exit_in_iptc,'_'}}, - {?eh,test_stats,{0,3,{0,0}}}, + {?eh,test_stats,{1,0,{0,2}}}, {?eh,tc_start,{config_func_error_1_SUITE,{end_per_group,g1,[]}}}, {?eh,tc_done,{config_func_error_1_SUITE,{end_per_group,g1,[]},ok}}], @@ -1549,7 +1549,7 @@ test_events(config_func_errors) -> {?eh,tc_done,{config_func_error_1_SUITE,{init_per_group,g2,[]},ok}}, {?eh,tc_start,{config_func_error_1_SUITE,exit_in_eptc}}, {?eh,tc_done,{config_func_error_1_SUITE,exit_in_eptc,'_'}}, - {?eh,test_stats,{0,4,{0,0}}}, + {?eh,test_stats,{2,0,{0,2}}}, {?eh,tc_start,{config_func_error_1_SUITE,{end_per_group,g2,[]}}}, {?eh,tc_done,{config_func_error_1_SUITE,{end_per_group,g2,[]},ok}}], diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl index 4624324295..e5d8f15453 100644 --- a/lib/common_test/test/ct_hooks_SUITE.erl +++ b/lib/common_test/test/ct_hooks_SUITE.erl @@ -1644,7 +1644,7 @@ test_events(callbacks_on_skip) -> {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, {?eh,cth,{empty_cth,id,[[]]}}, {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, - {?eh,start_info,{6,6,12}}, + {?eh,start_info,{6,6,15}}, %% all_hook_callbacks_SUITE is skipped in spec %% Only the on_tc_skip callback shall be called @@ -1923,6 +1923,86 @@ test_events(callbacks_on_skip) -> []]}}, {?eh,test_stats,{0,0,{5,5}}}, + %% Exit in init_per_testcase -> pre/post_end_per_testcase + %% shall not be called + {?eh,tc_start,{skip_case_SUITE,exit_in_init}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [skip_case_SUITE,exit_in_init, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [skip_case_SUITE,exit_in_init, + '$proplist', + {skip,{failed,'_'}}, + []]}}, + {?eh,tc_done,{skip_case_SUITE,exit_in_init, + {auto_skipped,{failed,'_'}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_case_SUITE,exit_in_init, + {tc_auto_skip,{auto_skipped,{failed,'_'}}}, + []]}}, + {?eh,test_stats,{0,0,{5,6}}}, + + %% Fail in end_per_testcase -> all hooks shall be called and + %% test shall succeed. + {?eh,tc_start,{skip_case_SUITE,fail_in_end}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [skip_case_SUITE,fail_in_end, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [skip_case_SUITE,fail_in_end, + '$proplist', + ok, + []]}}, + {?eh,cth,{empty_cth,pre_end_per_testcase, + [skip_case_SUITE,fail_in_end, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_end_per_testcase, + [skip_case_SUITE,fail_in_end, + '$proplist', + {failed, + {skip_case_SUITE,end_per_testcase, + {'EXIT', + {test_case_failed,"Failed in end_per_testcase/2"}}}}, + []]}}, + {?eh,tc_done,{skip_case_SUITE,fail_in_end, + {failed, + {skip_case_SUITE,end_per_testcase, + {'EXIT', + {test_case_failed,"Failed in end_per_testcase/2"}}}}}}, + {?eh,test_stats,{1,0,{5,6}}}, + + %% Exit in end_per_testcase -> all hooks shall be called and + %% test shall succeed. + {?eh,tc_start,{skip_case_SUITE,exit_in_end}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [skip_case_SUITE,exit_in_end, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [skip_case_SUITE,exit_in_end, + '$proplist', + ok, + []]}}, + {?eh,cth,{empty_cth,pre_end_per_testcase, + [skip_case_SUITE,exit_in_end, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_end_per_testcase, + [skip_case_SUITE,exit_in_end, + '$proplist', + {failed, + {skip_case_SUITE,end_per_testcase, + {'EXIT',"Exit in end_per_testcase/2"}}}, + []]}}, + {?eh,tc_done,{skip_case_SUITE,exit_in_end, + {failed, + {skip_case_SUITE,end_per_testcase, + {'EXIT',"Exit in end_per_testcase/2"}}}}}, + {?eh,test_stats,{2,0,{5,6}}}, + %% Skip in testcase function -> all callbacks shall be called {?eh,tc_start,{skip_case_SUITE,skip_in_case}}, {?eh,cth,{empty_cth,pre_init_per_testcase, @@ -1948,7 +2028,7 @@ test_events(callbacks_on_skip) -> [skip_case_SUITE,skip_in_case, {tc_user_skip,{skipped,"Skipped in test case function"}}, []]}}, - {?eh,test_stats,{0,0,{6,5}}}, + {?eh,test_stats,{2,0,{6,6}}}, %% Auto skip due to failed 'require' -> only the on_tc_skip %% callback shall be called @@ -1960,7 +2040,7 @@ test_events(callbacks_on_skip) -> {tc_auto_skip, {auto_skipped,{require_failed,{not_available,whatever}}}}, []]}}, - {?eh,test_stats,{0,0,{6,6}}}, + {?eh,test_stats,{2,0,{6,7}}}, %% Auto skip due to failed testcase/0 function -> only the %% on_tc_skip callback shall be called @@ -1972,7 +2052,7 @@ test_events(callbacks_on_skip) -> {tc_auto_skip, {auto_skipped,{testcase0_failed,bad_return_value}}}, []]}}, - {?eh,test_stats,{0,0,{6,7}}}, + {?eh,test_stats,{2,0,{6,8}}}, {?eh,tc_start,{skip_case_SUITE,end_per_suite}}, {?eh,cth,{empty_cth,pre_end_per_suite, diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl index 436d496c32..dad80ae914 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl @@ -43,9 +43,15 @@ init_per_testcase(skip_in_init,Config) -> {skip,"Skipped in init_per_testcase/2"}; init_per_testcase(fail_in_init,Config) -> ct:fail("Failed in init_per_testcase/2"); +init_per_testcase(exit_in_init,Config) -> + exit(self(),"Exit in init_per_testcase/2"); init_per_testcase(_,Config) -> Config. +end_per_testcase(fail_in_end,_) -> + ct:fail("Failed in end_per_testcase/2"); +end_per_testcase(exit_in_end,_) -> + exit(self(),"Exit in end_per_testcase/2"); end_per_testcase(_,_) -> ok. @@ -53,6 +59,9 @@ all() -> [skip_in_spec, skip_in_init, fail_in_init, + exit_in_init, + fail_in_end, + exit_in_end, skip_in_case, req_auto_skip, fail_auto_skip @@ -71,6 +80,16 @@ fail_in_init(Config) -> ct:fail("This test shall never be run. " "It shall fail in init_per_testcase/2."). +exit_in_init(Config) -> + ct:fail("This test shall never be run. " + "It shall exit in init_per_testcase/2."). + +fail_in_end(Config) -> + ok. + +exit_in_end(Config) -> + ok. + skip_in_case(Config) -> {skip,"Skipped in test case function"}. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl index 7d8f406763..16f015fe7a 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl @@ -101,6 +101,9 @@ post_end_per_group(Suite,Group,Config,Return,State) -> pre_init_per_testcase(Suite,TC,Config,State) -> (Suite==skip_case_SUITE andalso (TC==skip_in_init orelse TC==fail_in_init + orelse TC==exit_in_init + orelse TC==fail_in_end + orelse TC==exit_in_end orelse TC==skip_in_case)) orelse (Suite==seq_SUITE andalso TC==test_case_1) orelse (Suite==repeat_SUITE andalso TC==test_case_1) @@ -110,6 +113,9 @@ pre_init_per_testcase(Suite,TC,Config,State) -> post_init_per_testcase(Suite,TC,Config,Return,State) -> (Suite==skip_case_SUITE andalso (TC==skip_in_init orelse TC==fail_in_init + orelse TC==exit_in_init + orelse TC==fail_in_end + orelse TC==exit_in_end orelse TC==skip_in_case)) orelse (Suite==seq_SUITE andalso TC==test_case_1) orelse (Suite==repeat_SUITE andalso TC==test_case_1) @@ -117,14 +123,18 @@ post_init_per_testcase(Suite,TC,Config,Return,State) -> empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). pre_end_per_testcase(Suite,TC,Config,State) -> - (Suite==skip_case_SUITE andalso TC==skip_in_case) + (Suite==skip_case_SUITE andalso (TC==skip_in_case + orelse TC==fail_in_end + orelse TC==exit_in_end)) orelse (Suite==seq_SUITE andalso TC==test_case_1) orelse (Suite==repeat_SUITE andalso TC==test_case_1) orelse ?fail({Suite,TC}), empty_cth:pre_end_per_testcase(Suite,TC,Config,State). post_end_per_testcase(Suite,TC,Config,Return,State) -> - (Suite==skip_case_SUITE andalso TC==skip_in_case) + (Suite==skip_case_SUITE andalso (TC==skip_in_case + orelse TC==fail_in_end + orelse TC==exit_in_end)) orelse (Suite==seq_SUITE andalso TC==test_case_1) orelse (Suite==repeat_SUITE andalso TC==test_case_1) orelse ?fail({Suite,TC}), @@ -155,6 +165,7 @@ on_tc_skip(skip_case_SUITE=Suite,TC,Reason,State) when TC==skip_in_spec; TC==skip_in_init; TC==fail_in_init; + TC==exit_in_init; TC==skip_in_case; TC==req_auto_skip; TC==fail_auto_skip -> |