aboutsummaryrefslogtreecommitdiffstats
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/common_test/src/test_server_ctrl.erl53
-rw-r--r--lib/common_test/test/ct_error_SUITE.erl8
-rw-r--r--lib/common_test/test/ct_hooks_SUITE.erl88
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl19
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl15
5 files changed, 153 insertions, 30 deletions
diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl
index d3a0f74f7a..39c523f8b3 100644
--- a/lib/common_test/src/test_server_ctrl.erl
+++ b/lib/common_test/src/test_server_ctrl.erl
@@ -2939,22 +2939,21 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status)
exit(framework_error);
%% sequential execution of test case finished
{Time,RetVal,_} ->
+ RetTag =
+ if is_tuple(RetVal) -> element(1,RetVal);
+ true -> undefined
+ end,
{Failed,Status1} =
- case Time of
- died ->
- {true,update_status(failed, Mod, Func, Status)};
- _ when is_tuple(RetVal) ->
- case element(1, RetVal) of
- R when R=='EXIT'; R==failed ->
- {true,update_status(failed, Mod, Func, Status)};
- R when R==skip; R==skipped ->
- {false,update_status(skipped, Mod, Func, Status)};
- _ ->
- {false,update_status(ok, Mod, Func, Status)}
- end;
- _ ->
- {false,update_status(ok, Mod, Func, Status)}
- end,
+ case RetTag of
+ Skip when Skip==skip; Skip==skipped ->
+ {false,update_status(skipped, Mod, Func, Status)};
+ Fail when Fail=='EXIT'; Fail==failed ->
+ {true,update_status(failed, Mod, Func, Status)};
+ _ when Time==died, RetVal=/=ok ->
+ {true,update_status(failed, Mod, Func, Status)};
+ _ ->
+ {false,update_status(ok, Mod, Func, Status)}
+ end,
case check_prop(sequence, Mode) of
false ->
stop_minor_log_file(),
@@ -3809,7 +3808,15 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit,
{died,{timetrap_timeout,TimetrapTimeout}} ->
progress(failed, Num, Mod, Func, GrName, Loc,
timetrap_timeout, TimetrapTimeout, Comment, Style);
- {died,Reason} ->
+ {died,{Skip,Reason}} when Skip==skip; Skip==skipped ->
+ %% died in init_per_testcase
+ progress(skip, Num, Mod, Func, GrName, Loc, Reason,
+ Time, Comment, Style);
+ {died,Reason} when Reason=/=ok ->
+ %% (If Reason==ok it means that process died in
+ %% end_per_testcase after successfully completing the
+ %% test case itself - then we shall not fail, but a
+ %% warning will be issued in the comment field.)
progress(failed, Num, Mod, Func, GrName, Loc, Reason,
Time, Comment, Style);
{_,{'EXIT',{Skip,Reason}}} when Skip==skip; Skip==skipped;
@@ -3958,6 +3965,9 @@ progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time,
[get_info_str(Mod,Func, CaseNum, get(test_server_cases))]),
test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName},
{ReportTag,Reason1}}]),
+ TimeStr = io_lib:format(if is_float(Time) -> "~.3fs";
+ true -> "~w"
+ end, [Time]),
ReasonStr = escape_chars(reason_to_string(Reason1)),
ReasonStr1 = lists:flatten([string:strip(S,left) ||
S <- string:tokens(ReasonStr,[$\n])]),
@@ -3972,10 +3982,10 @@ progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time,
_ -> xhtml("<br>(","<br />(") ++ to_string(Comment) ++ ")"
end,
print(html,
- "<td>" ++ St0 ++ "~.3fs" ++ St1 ++ "</td>"
+ "<td>" ++ St0 ++ "~ts" ++ St1 ++ "</td>"
"<td><font color=\"~ts\">SKIPPED</font></td>"
"<td>~ts~ts</td></tr>\n",
- [Time,Color,ReasonStr2,Comment1]),
+ [TimeStr,Color,ReasonStr2,Comment1]),
FormatLoc = test_server_sup:format_loc(Loc),
print(minor, "=== Location: ~ts", [FormatLoc]),
print(minor, "=== Reason: ~ts", [ReasonStr1]),
@@ -4113,6 +4123,9 @@ progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time,
Comment0, {St0,St1}) ->
print(minor, "successfully completed test case", []),
test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName},ok}]),
+ TimeStr = io_lib:format(if is_float(Time) -> "~.3fs";
+ true -> "~w"
+ end, [Time]),
Comment =
case RetVal of
{comment,RetComment} ->
@@ -4131,10 +4144,10 @@ progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time,
end,
print(major, "=elapsed ~p", [Time]),
print(html,
- "<td>" ++ St0 ++ "~.3fs" ++ St1 ++ "</td>"
+ "<td>" ++ St0 ++ "~ts" ++ St1 ++ "</td>"
"<td><font color=\"green\">Ok</font></td>"
"~ts</tr>\n",
- [Time,Comment]),
+ [TimeStr,Comment]),
print(minor,
escape_chars(io_lib:format("=== Returned value: ~tp", [RetVal])),
[]),
diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl
index fae23484e6..621f3b6d2d 100644
--- a/lib/common_test/test/ct_error_SUITE.erl
+++ b/lib/common_test/test/ct_error_SUITE.erl
@@ -1531,17 +1531,17 @@ test_events(config_func_errors) ->
{?eh,tc_start,{config_func_error_1_SUITE,exit_in_iptc}},
{?eh,tc_done,{config_func_error_1_SUITE,exit_in_iptc,'_'}},
- {?eh,test_stats,{0,1,{0,0}}},
+ {?eh,test_stats,{0,0,{0,1}}},
{?eh,tc_start,{config_func_error_1_SUITE,exit_in_eptc}},
{?eh,tc_done,{config_func_error_1_SUITE,exit_in_eptc,'_'}},
- {?eh,test_stats,{0,2,{0,0}}},
+ {?eh,test_stats,{1,0,{0,1}}},
[{?eh,tc_start,{config_func_error_1_SUITE,{init_per_group,g1,[]}}},
{?eh,tc_done,{config_func_error_1_SUITE,{init_per_group,g1,[]},ok}},
{?eh,tc_start,{config_func_error_1_SUITE,exit_in_iptc}},
{?eh,tc_done,{config_func_error_1_SUITE,exit_in_iptc,'_'}},
- {?eh,test_stats,{0,3,{0,0}}},
+ {?eh,test_stats,{1,0,{0,2}}},
{?eh,tc_start,{config_func_error_1_SUITE,{end_per_group,g1,[]}}},
{?eh,tc_done,{config_func_error_1_SUITE,{end_per_group,g1,[]},ok}}],
@@ -1549,7 +1549,7 @@ test_events(config_func_errors) ->
{?eh,tc_done,{config_func_error_1_SUITE,{init_per_group,g2,[]},ok}},
{?eh,tc_start,{config_func_error_1_SUITE,exit_in_eptc}},
{?eh,tc_done,{config_func_error_1_SUITE,exit_in_eptc,'_'}},
- {?eh,test_stats,{0,4,{0,0}}},
+ {?eh,test_stats,{2,0,{0,2}}},
{?eh,tc_start,{config_func_error_1_SUITE,{end_per_group,g2,[]}}},
{?eh,tc_done,{config_func_error_1_SUITE,{end_per_group,g2,[]},ok}}],
diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl
index 4624324295..e5d8f15453 100644
--- a/lib/common_test/test/ct_hooks_SUITE.erl
+++ b/lib/common_test/test/ct_hooks_SUITE.erl
@@ -1644,7 +1644,7 @@ test_events(callbacks_on_skip) ->
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
{?eh,cth,{empty_cth,id,[[]]}},
{?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
- {?eh,start_info,{6,6,12}},
+ {?eh,start_info,{6,6,15}},
%% all_hook_callbacks_SUITE is skipped in spec
%% Only the on_tc_skip callback shall be called
@@ -1923,6 +1923,86 @@ test_events(callbacks_on_skip) ->
[]]}},
{?eh,test_stats,{0,0,{5,5}}},
+ %% Exit in init_per_testcase -> pre/post_end_per_testcase
+ %% shall not be called
+ {?eh,tc_start,{skip_case_SUITE,exit_in_init}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [skip_case_SUITE,exit_in_init,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [skip_case_SUITE,exit_in_init,
+ '$proplist',
+ {skip,{failed,'_'}},
+ []]}},
+ {?eh,tc_done,{skip_case_SUITE,exit_in_init,
+ {auto_skipped,{failed,'_'}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_case_SUITE,exit_in_init,
+ {tc_auto_skip,{auto_skipped,{failed,'_'}}},
+ []]}},
+ {?eh,test_stats,{0,0,{5,6}}},
+
+ %% Fail in end_per_testcase -> all hooks shall be called and
+ %% test shall succeed.
+ {?eh,tc_start,{skip_case_SUITE,fail_in_end}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [skip_case_SUITE,fail_in_end,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [skip_case_SUITE,fail_in_end,
+ '$proplist',
+ ok,
+ []]}},
+ {?eh,cth,{empty_cth,pre_end_per_testcase,
+ [skip_case_SUITE,fail_in_end,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_end_per_testcase,
+ [skip_case_SUITE,fail_in_end,
+ '$proplist',
+ {failed,
+ {skip_case_SUITE,end_per_testcase,
+ {'EXIT',
+ {test_case_failed,"Failed in end_per_testcase/2"}}}},
+ []]}},
+ {?eh,tc_done,{skip_case_SUITE,fail_in_end,
+ {failed,
+ {skip_case_SUITE,end_per_testcase,
+ {'EXIT',
+ {test_case_failed,"Failed in end_per_testcase/2"}}}}}},
+ {?eh,test_stats,{1,0,{5,6}}},
+
+ %% Exit in end_per_testcase -> all hooks shall be called and
+ %% test shall succeed.
+ {?eh,tc_start,{skip_case_SUITE,exit_in_end}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [skip_case_SUITE,exit_in_end,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [skip_case_SUITE,exit_in_end,
+ '$proplist',
+ ok,
+ []]}},
+ {?eh,cth,{empty_cth,pre_end_per_testcase,
+ [skip_case_SUITE,exit_in_end,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_end_per_testcase,
+ [skip_case_SUITE,exit_in_end,
+ '$proplist',
+ {failed,
+ {skip_case_SUITE,end_per_testcase,
+ {'EXIT',"Exit in end_per_testcase/2"}}},
+ []]}},
+ {?eh,tc_done,{skip_case_SUITE,exit_in_end,
+ {failed,
+ {skip_case_SUITE,end_per_testcase,
+ {'EXIT',"Exit in end_per_testcase/2"}}}}},
+ {?eh,test_stats,{2,0,{5,6}}},
+
%% Skip in testcase function -> all callbacks shall be called
{?eh,tc_start,{skip_case_SUITE,skip_in_case}},
{?eh,cth,{empty_cth,pre_init_per_testcase,
@@ -1948,7 +2028,7 @@ test_events(callbacks_on_skip) ->
[skip_case_SUITE,skip_in_case,
{tc_user_skip,{skipped,"Skipped in test case function"}},
[]]}},
- {?eh,test_stats,{0,0,{6,5}}},
+ {?eh,test_stats,{2,0,{6,6}}},
%% Auto skip due to failed 'require' -> only the on_tc_skip
%% callback shall be called
@@ -1960,7 +2040,7 @@ test_events(callbacks_on_skip) ->
{tc_auto_skip,
{auto_skipped,{require_failed,{not_available,whatever}}}},
[]]}},
- {?eh,test_stats,{0,0,{6,6}}},
+ {?eh,test_stats,{2,0,{6,7}}},
%% Auto skip due to failed testcase/0 function -> only the
%% on_tc_skip callback shall be called
@@ -1972,7 +2052,7 @@ test_events(callbacks_on_skip) ->
{tc_auto_skip,
{auto_skipped,{testcase0_failed,bad_return_value}}},
[]]}},
- {?eh,test_stats,{0,0,{6,7}}},
+ {?eh,test_stats,{2,0,{6,8}}},
{?eh,tc_start,{skip_case_SUITE,end_per_suite}},
{?eh,cth,{empty_cth,pre_end_per_suite,
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl
index 436d496c32..dad80ae914 100644
--- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl
@@ -43,9 +43,15 @@ init_per_testcase(skip_in_init,Config) ->
{skip,"Skipped in init_per_testcase/2"};
init_per_testcase(fail_in_init,Config) ->
ct:fail("Failed in init_per_testcase/2");
+init_per_testcase(exit_in_init,Config) ->
+ exit(self(),"Exit in init_per_testcase/2");
init_per_testcase(_,Config) ->
Config.
+end_per_testcase(fail_in_end,_) ->
+ ct:fail("Failed in end_per_testcase/2");
+end_per_testcase(exit_in_end,_) ->
+ exit(self(),"Exit in end_per_testcase/2");
end_per_testcase(_,_) ->
ok.
@@ -53,6 +59,9 @@ all() ->
[skip_in_spec,
skip_in_init,
fail_in_init,
+ exit_in_init,
+ fail_in_end,
+ exit_in_end,
skip_in_case,
req_auto_skip,
fail_auto_skip
@@ -71,6 +80,16 @@ fail_in_init(Config) ->
ct:fail("This test shall never be run. "
"It shall fail in init_per_testcase/2.").
+exit_in_init(Config) ->
+ ct:fail("This test shall never be run. "
+ "It shall exit in init_per_testcase/2.").
+
+fail_in_end(Config) ->
+ ok.
+
+exit_in_end(Config) ->
+ ok.
+
skip_in_case(Config) ->
{skip,"Skipped in test case function"}.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl
index 7d8f406763..16f015fe7a 100644
--- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl
@@ -101,6 +101,9 @@ post_end_per_group(Suite,Group,Config,Return,State) ->
pre_init_per_testcase(Suite,TC,Config,State) ->
(Suite==skip_case_SUITE andalso (TC==skip_in_init
orelse TC==fail_in_init
+ orelse TC==exit_in_init
+ orelse TC==fail_in_end
+ orelse TC==exit_in_end
orelse TC==skip_in_case))
orelse (Suite==seq_SUITE andalso TC==test_case_1)
orelse (Suite==repeat_SUITE andalso TC==test_case_1)
@@ -110,6 +113,9 @@ pre_init_per_testcase(Suite,TC,Config,State) ->
post_init_per_testcase(Suite,TC,Config,Return,State) ->
(Suite==skip_case_SUITE andalso (TC==skip_in_init
orelse TC==fail_in_init
+ orelse TC==exit_in_init
+ orelse TC==fail_in_end
+ orelse TC==exit_in_end
orelse TC==skip_in_case))
orelse (Suite==seq_SUITE andalso TC==test_case_1)
orelse (Suite==repeat_SUITE andalso TC==test_case_1)
@@ -117,14 +123,18 @@ post_init_per_testcase(Suite,TC,Config,Return,State) ->
empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State).
pre_end_per_testcase(Suite,TC,Config,State) ->
- (Suite==skip_case_SUITE andalso TC==skip_in_case)
+ (Suite==skip_case_SUITE andalso (TC==skip_in_case
+ orelse TC==fail_in_end
+ orelse TC==exit_in_end))
orelse (Suite==seq_SUITE andalso TC==test_case_1)
orelse (Suite==repeat_SUITE andalso TC==test_case_1)
orelse ?fail({Suite,TC}),
empty_cth:pre_end_per_testcase(Suite,TC,Config,State).
post_end_per_testcase(Suite,TC,Config,Return,State) ->
- (Suite==skip_case_SUITE andalso TC==skip_in_case)
+ (Suite==skip_case_SUITE andalso (TC==skip_in_case
+ orelse TC==fail_in_end
+ orelse TC==exit_in_end))
orelse (Suite==seq_SUITE andalso TC==test_case_1)
orelse (Suite==repeat_SUITE andalso TC==test_case_1)
orelse ?fail({Suite,TC}),
@@ -155,6 +165,7 @@ on_tc_skip(skip_case_SUITE=Suite,TC,Reason,State)
when TC==skip_in_spec;
TC==skip_in_init;
TC==fail_in_init;
+ TC==exit_in_init;
TC==skip_in_case;
TC==req_auto_skip;
TC==fail_auto_skip ->
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-24 02:13:23 +0000