aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSiri Hansen <[email protected]>2017-02-16 12:16:21 +0100
committerSiri Hansen <[email protected]>2017-02-20 12:40:08 +0100
commit827732949ba9cd844347bca1a46b2e2055810e69 (patch)
treeb025a5c4ce577fc1658d2a097785f04814f0ecd9
parent6b7463094db2ec3c2c0e4f8784dcd64ef186c96b (diff)
downloadotp-827732949ba9cd844347bca1a46b2e2055810e69.tar.gz
otp-827732949ba9cd844347bca1a46b2e2055810e69.tar.bz2
otp-827732949ba9cd844347bca1a46b2e2055810e69.zip
[ct] Correctly handle process died in init and end_per_testcase
Documentation says that a test case is skipped if init_per_testcase fails. This is correct if init_per_testcase fails in a normal way, but if it gets an exit signal causing the process to die, then the case was earlier marked as failed with reason {skip,...}. This is now corrected, so the case is auto skipped. If end_per_testcase crashes, it does not change the result of a test case, it only adds a warning in the comment field. But if it gets an exit signal causing the process to die while running end_per_testcase, then the case was earlier marked as failed in addition to this warning. This is now corrected, so the result of the test case is not changed by an exit signal during end_per_testcase.
-rw-r--r--lib/common_test/src/test_server_ctrl.erl53
-rw-r--r--lib/common_test/test/ct_error_SUITE.erl8
-rw-r--r--lib/common_test/test/ct_hooks_SUITE.erl88
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl19
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl15
5 files changed, 153 insertions, 30 deletions
diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl
index d3a0f74f7a..39c523f8b3 100644
--- a/lib/common_test/src/test_server_ctrl.erl
+++ b/lib/common_test/src/test_server_ctrl.erl
@@ -2939,22 +2939,21 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status)
exit(framework_error);
%% sequential execution of test case finished
{Time,RetVal,_} ->
+ RetTag =
+ if is_tuple(RetVal) -> element(1,RetVal);
+ true -> undefined
+ end,
{Failed,Status1} =
- case Time of
- died ->
- {true,update_status(failed, Mod, Func, Status)};
- _ when is_tuple(RetVal) ->
- case element(1, RetVal) of
- R when R=='EXIT'; R==failed ->
- {true,update_status(failed, Mod, Func, Status)};
- R when R==skip; R==skipped ->
- {false,update_status(skipped, Mod, Func, Status)};
- _ ->
- {false,update_status(ok, Mod, Func, Status)}
- end;
- _ ->
- {false,update_status(ok, Mod, Func, Status)}
- end,
+ case RetTag of
+ Skip when Skip==skip; Skip==skipped ->
+ {false,update_status(skipped, Mod, Func, Status)};
+ Fail when Fail=='EXIT'; Fail==failed ->
+ {true,update_status(failed, Mod, Func, Status)};
+ _ when Time==died, RetVal=/=ok ->
+ {true,update_status(failed, Mod, Func, Status)};
+ _ ->
+ {false,update_status(ok, Mod, Func, Status)}
+ end,
case check_prop(sequence, Mode) of
false ->
stop_minor_log_file(),
@@ -3809,7 +3808,15 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit,
{died,{timetrap_timeout,TimetrapTimeout}} ->
progress(failed, Num, Mod, Func, GrName, Loc,
timetrap_timeout, TimetrapTimeout, Comment, Style);
- {died,Reason} ->
+ {died,{Skip,Reason}} when Skip==skip; Skip==skipped ->
+ %% died in init_per_testcase
+ progress(skip, Num, Mod, Func, GrName, Loc, Reason,
+ Time, Comment, Style);
+ {died,Reason} when Reason=/=ok ->
+ %% (If Reason==ok it means that process died in
+ %% end_per_testcase after successfully completing the
+ %% test case itself - then we shall not fail, but a
+ %% warning will be issued in the comment field.)
progress(failed, Num, Mod, Func, GrName, Loc, Reason,
Time, Comment, Style);
{_,{'EXIT',{Skip,Reason}}} when Skip==skip; Skip==skipped;
@@ -3958,6 +3965,9 @@ progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time,
[get_info_str(Mod,Func, CaseNum, get(test_server_cases))]),
test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName},
{ReportTag,Reason1}}]),
+ TimeStr = io_lib:format(if is_float(Time) -> "~.3fs";
+ true -> "~w"
+ end, [Time]),
ReasonStr = escape_chars(reason_to_string(Reason1)),
ReasonStr1 = lists:flatten([string:strip(S,left) ||
S <- string:tokens(ReasonStr,[$\n])]),
@@ -3972,10 +3982,10 @@ progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time,
_ -> xhtml("<br>(","<br />(") ++ to_string(Comment) ++ ")"
end,
print(html,
- "<td>" ++ St0 ++ "~.3fs" ++ St1 ++ "</td>"
+ "<td>" ++ St0 ++ "~ts" ++ St1 ++ "</td>"
"<td><font color=\"~ts\">SKIPPED</font></td>"
"<td>~ts~ts</td></tr>\n",
- [Time,Color,ReasonStr2,Comment1]),
+ [TimeStr,Color,ReasonStr2,Comment1]),
FormatLoc = test_server_sup:format_loc(Loc),
print(minor, "=== Location: ~ts", [FormatLoc]),
print(minor, "=== Reason: ~ts", [ReasonStr1]),
@@ -4113,6 +4123,9 @@ progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time,
Comment0, {St0,St1}) ->
print(minor, "successfully completed test case", []),
test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName},ok}]),
+ TimeStr = io_lib:format(if is_float(Time) -> "~.3fs";
+ true -> "~w"
+ end, [Time]),
Comment =
case RetVal of
{comment,RetComment} ->
@@ -4131,10 +4144,10 @@ progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time,
end,
print(major, "=elapsed ~p", [Time]),
print(html,
- "<td>" ++ St0 ++ "~.3fs" ++ St1 ++ "</td>"
+ "<td>" ++ St0 ++ "~ts" ++ St1 ++ "</td>"
"<td><font color=\"green\">Ok</font></td>"
"~ts</tr>\n",
- [Time,Comment]),
+ [TimeStr,Comment]),
print(minor,
escape_chars(io_lib:format("=== Returned value: ~tp", [RetVal])),
[]),
diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl
index fae23484e6..621f3b6d2d 100644
--- a/lib/common_test/test/ct_error_SUITE.erl
+++ b/lib/common_test/test/ct_error_SUITE.erl
@@ -1531,17 +1531,17 @@ test_events(config_func_errors) ->
{?eh,tc_start,{config_func_error_1_SUITE,exit_in_iptc}},
{?eh,tc_done,{config_func_error_1_SUITE,exit_in_iptc,'_'}},
- {?eh,test_stats,{0,1,{0,0}}},
+ {?eh,test_stats,{0,0,{0,1}}},
{?eh,tc_start,{config_func_error_1_SUITE,exit_in_eptc}},
{?eh,tc_done,{config_func_error_1_SUITE,exit_in_eptc,'_'}},
- {?eh,test_stats,{0,2,{0,0}}},
+ {?eh,test_stats,{1,0,{0,1}}},
[{?eh,tc_start,{config_func_error_1_SUITE,{init_per_group,g1,[]}}},
{?eh,tc_done,{config_func_error_1_SUITE,{init_per_group,g1,[]},ok}},
{?eh,tc_start,{config_func_error_1_SUITE,exit_in_iptc}},
{?eh,tc_done,{config_func_error_1_SUITE,exit_in_iptc,'_'}},
- {?eh,test_stats,{0,3,{0,0}}},
+ {?eh,test_stats,{1,0,{0,2}}},
{?eh,tc_start,{config_func_error_1_SUITE,{end_per_group,g1,[]}}},
{?eh,tc_done,{config_func_error_1_SUITE,{end_per_group,g1,[]},ok}}],
@@ -1549,7 +1549,7 @@ test_events(config_func_errors) ->
{?eh,tc_done,{config_func_error_1_SUITE,{init_per_group,g2,[]},ok}},
{?eh,tc_start,{config_func_error_1_SUITE,exit_in_eptc}},
{?eh,tc_done,{config_func_error_1_SUITE,exit_in_eptc,'_'}},
- {?eh,test_stats,{0,4,{0,0}}},
+ {?eh,test_stats,{2,0,{0,2}}},
{?eh,tc_start,{config_func_error_1_SUITE,{end_per_group,g2,[]}}},
{?eh,tc_done,{config_func_error_1_SUITE,{end_per_group,g2,[]},ok}}],
diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl
index 4624324295..e5d8f15453 100644
--- a/lib/common_test/test/ct_hooks_SUITE.erl
+++ b/lib/common_test/test/ct_hooks_SUITE.erl
@@ -1644,7 +1644,7 @@ test_events(callbacks_on_skip) ->
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
{?eh,cth,{empty_cth,id,[[]]}},
{?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
- {?eh,start_info,{6,6,12}},
+ {?eh,start_info,{6,6,15}},
%% all_hook_callbacks_SUITE is skipped in spec
%% Only the on_tc_skip callback shall be called
@@ -1923,6 +1923,86 @@ test_events(callbacks_on_skip) ->
[]]}},
{?eh,test_stats,{0,0,{5,5}}},
+ %% Exit in init_per_testcase -> pre/post_end_per_testcase
+ %% shall not be called
+ {?eh,tc_start,{skip_case_SUITE,exit_in_init}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [skip_case_SUITE,exit_in_init,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [skip_case_SUITE,exit_in_init,
+ '$proplist',
+ {skip,{failed,'_'}},
+ []]}},
+ {?eh,tc_done,{skip_case_SUITE,exit_in_init,
+ {auto_skipped,{failed,'_'}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_case_SUITE,exit_in_init,
+ {tc_auto_skip,{auto_skipped,{failed,'_'}}},
+ []]}},
+ {?eh,test_stats,{0,0,{5,6}}},
+
+ %% Fail in end_per_testcase -> all hooks shall be called and
+ %% test shall succeed.
+ {?eh,tc_start,{skip_case_SUITE,fail_in_end}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [skip_case_SUITE,fail_in_end,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [skip_case_SUITE,fail_in_end,
+ '$proplist',
+ ok,
+ []]}},
+ {?eh,cth,{empty_cth,pre_end_per_testcase,
+ [skip_case_SUITE,fail_in_end,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_end_per_testcase,
+ [skip_case_SUITE,fail_in_end,
+ '$proplist',
+ {failed,
+ {skip_case_SUITE,end_per_testcase,
+ {'EXIT',
+ {test_case_failed,"Failed in end_per_testcase/2"}}}},
+ []]}},
+ {?eh,tc_done,{skip_case_SUITE,fail_in_end,
+ {failed,
+ {skip_case_SUITE,end_per_testcase,
+ {'EXIT',
+ {test_case_failed,"Failed in end_per_testcase/2"}}}}}},
+ {?eh,test_stats,{1,0,{5,6}}},
+
+ %% Exit in end_per_testcase -> all hooks shall be called and
+ %% test shall succeed.
+ {?eh,tc_start,{skip_case_SUITE,exit_in_end}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [skip_case_SUITE,exit_in_end,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [skip_case_SUITE,exit_in_end,
+ '$proplist',
+ ok,
+ []]}},
+ {?eh,cth,{empty_cth,pre_end_per_testcase,
+ [skip_case_SUITE,exit_in_end,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_end_per_testcase,
+ [skip_case_SUITE,exit_in_end,
+ '$proplist',
+ {failed,
+ {skip_case_SUITE,end_per_testcase,
+ {'EXIT',"Exit in end_per_testcase/2"}}},
+ []]}},
+ {?eh,tc_done,{skip_case_SUITE,exit_in_end,
+ {failed,
+ {skip_case_SUITE,end_per_testcase,
+ {'EXIT',"Exit in end_per_testcase/2"}}}}},
+ {?eh,test_stats,{2,0,{5,6}}},
+
%% Skip in testcase function -> all callbacks shall be called
{?eh,tc_start,{skip_case_SUITE,skip_in_case}},
{?eh,cth,{empty_cth,pre_init_per_testcase,
@@ -1948,7 +2028,7 @@ test_events(callbacks_on_skip) ->
[skip_case_SUITE,skip_in_case,
{tc_user_skip,{skipped,"Skipped in test case function"}},
[]]}},
- {?eh,test_stats,{0,0,{6,5}}},
+ {?eh,test_stats,{2,0,{6,6}}},
%% Auto skip due to failed 'require' -> only the on_tc_skip
%% callback shall be called
@@ -1960,7 +2040,7 @@ test_events(callbacks_on_skip) ->
{tc_auto_skip,
{auto_skipped,{require_failed,{not_available,whatever}}}},
[]]}},
- {?eh,test_stats,{0,0,{6,6}}},
+ {?eh,test_stats,{2,0,{6,7}}},
%% Auto skip due to failed testcase/0 function -> only the
%% on_tc_skip callback shall be called
@@ -1972,7 +2052,7 @@ test_events(callbacks_on_skip) ->
{tc_auto_skip,
{auto_skipped,{testcase0_failed,bad_return_value}}},
[]]}},
- {?eh,test_stats,{0,0,{6,7}}},
+ {?eh,test_stats,{2,0,{6,8}}},
{?eh,tc_start,{skip_case_SUITE,end_per_suite}},
{?eh,cth,{empty_cth,pre_end_per_suite,
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl
index 436d496c32..dad80ae914 100644
--- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl
@@ -43,9 +43,15 @@ init_per_testcase(skip_in_init,Config) ->
{skip,"Skipped in init_per_testcase/2"};
init_per_testcase(fail_in_init,Config) ->
ct:fail("Failed in init_per_testcase/2");
+init_per_testcase(exit_in_init,Config) ->
+ exit(self(),"Exit in init_per_testcase/2");
init_per_testcase(_,Config) ->
Config.
+end_per_testcase(fail_in_end,_) ->
+ ct:fail("Failed in end_per_testcase/2");
+end_per_testcase(exit_in_end,_) ->
+ exit(self(),"Exit in end_per_testcase/2");
end_per_testcase(_,_) ->
ok.
@@ -53,6 +59,9 @@ all() ->
[skip_in_spec,
skip_in_init,
fail_in_init,
+ exit_in_init,
+ fail_in_end,
+ exit_in_end,
skip_in_case,
req_auto_skip,
fail_auto_skip
@@ -71,6 +80,16 @@ fail_in_init(Config) ->
ct:fail("This test shall never be run. "
"It shall fail in init_per_testcase/2.").
+exit_in_init(Config) ->
+ ct:fail("This test shall never be run. "
+ "It shall exit in init_per_testcase/2.").
+
+fail_in_end(Config) ->
+ ok.
+
+exit_in_end(Config) ->
+ ok.
+
skip_in_case(Config) ->
{skip,"Skipped in test case function"}.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl
index 7d8f406763..16f015fe7a 100644
--- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl
@@ -101,6 +101,9 @@ post_end_per_group(Suite,Group,Config,Return,State) ->
pre_init_per_testcase(Suite,TC,Config,State) ->
(Suite==skip_case_SUITE andalso (TC==skip_in_init
orelse TC==fail_in_init
+ orelse TC==exit_in_init
+ orelse TC==fail_in_end
+ orelse TC==exit_in_end
orelse TC==skip_in_case))
orelse (Suite==seq_SUITE andalso TC==test_case_1)
orelse (Suite==repeat_SUITE andalso TC==test_case_1)
@@ -110,6 +113,9 @@ pre_init_per_testcase(Suite,TC,Config,State) ->
post_init_per_testcase(Suite,TC,Config,Return,State) ->
(Suite==skip_case_SUITE andalso (TC==skip_in_init
orelse TC==fail_in_init
+ orelse TC==exit_in_init
+ orelse TC==fail_in_end
+ orelse TC==exit_in_end
orelse TC==skip_in_case))
orelse (Suite==seq_SUITE andalso TC==test_case_1)
orelse (Suite==repeat_SUITE andalso TC==test_case_1)
@@ -117,14 +123,18 @@ post_init_per_testcase(Suite,TC,Config,Return,State) ->
empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State).
pre_end_per_testcase(Suite,TC,Config,State) ->
- (Suite==skip_case_SUITE andalso TC==skip_in_case)
+ (Suite==skip_case_SUITE andalso (TC==skip_in_case
+ orelse TC==fail_in_end
+ orelse TC==exit_in_end))
orelse (Suite==seq_SUITE andalso TC==test_case_1)
orelse (Suite==repeat_SUITE andalso TC==test_case_1)
orelse ?fail({Suite,TC}),
empty_cth:pre_end_per_testcase(Suite,TC,Config,State).
post_end_per_testcase(Suite,TC,Config,Return,State) ->
- (Suite==skip_case_SUITE andalso TC==skip_in_case)
+ (Suite==skip_case_SUITE andalso (TC==skip_in_case
+ orelse TC==fail_in_end
+ orelse TC==exit_in_end))
orelse (Suite==seq_SUITE andalso TC==test_case_1)
orelse (Suite==repeat_SUITE andalso TC==test_case_1)
orelse ?fail({Suite,TC}),
@@ -155,6 +165,7 @@ on_tc_skip(skip_case_SUITE=Suite,TC,Reason,State)
when TC==skip_in_spec;
TC==skip_in_init;
TC==fail_in_init;
+ TC==exit_in_init;
TC==skip_in_case;
TC==req_auto_skip;
TC==fail_auto_skip ->