aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPeter Andersson <[email protected]>2013-11-28 17:37:25 +0100
committerPeter Andersson <[email protected]>2013-11-28 17:37:33 +0100
commitac7902caa784d4f4c73eb7291ab4736a75ad13a4 (patch)
tree7c292d5ffd0907a42efb49e8ab1c60f4ab515649
parent9f998969014095333545cb118467e1ff8d450c31 (diff)
parent093890dc793e85a09b40f1eca878f410c73cf625 (diff)
downloadotp-ac7902caa784d4f4c73eb7291ab4736a75ad13a4.tar.gz
otp-ac7902caa784d4f4c73eb7291ab4736a75ad13a4.tar.bz2
otp-ac7902caa784d4f4c73eb7291ab4736a75ad13a4.zip
Merge remote branch 'origin/peppe/common_test/change_skip_behaviour' into maint
* origin/peppe/common_test/change_skip_behaviour: Modify the auto_skip report for group config funcs to include group name Fix problems with info functions and add more tests Add test cases for new and modified functionality Fix problem with suites and groups skipped from test specification Change report tag for failed init_per_testcase from skipped to auto_skipped Add tests for skipping parallel groups Correct tests cases that fail because of modified events Correct various bugs related to auto_skip and groups Change status from skip to auto_skip for config func that fails due to require Fix problem with handling Config and FW reports correctly Change skip vs auto_skip behaviour OTP-11305
-rw-r--r--lib/common_test/doc/src/ct_hooks.xml5
-rw-r--r--lib/common_test/src/ct_framework.erl274
-rw-r--r--lib/common_test/src/ct_logs.erl6
-rw-r--r--lib/common_test/src/cth_surefire.erl5
-rw-r--r--lib/common_test/test/ct_config_SUITE.erl34
-rw-r--r--lib/common_test/test/ct_config_info_SUITE.erl6
-rw-r--r--lib/common_test/test/ct_error_SUITE.erl52
-rw-r--r--lib/common_test/test/ct_group_info_SUITE.erl156
-rw-r--r--lib/common_test/test/ct_groups_test_2_SUITE.erl12
-rw-r--r--lib/common_test/test/ct_hooks_SUITE.erl26
-rw-r--r--lib/common_test/test/ct_repeat_1_SUITE.erl26
-rw-r--r--lib/common_test/test/ct_repeat_testrun_SUITE.erl21
-rw-r--r--lib/common_test/test/ct_sequence_1_SUITE.erl4
-rw-r--r--lib/common_test/test/ct_skip_SUITE.erl433
-rw-r--r--lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_12_SUITE.erl121
-rw-r--r--lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_3_SUITE.erl2
-rw-r--r--lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_6_SUITE.erl118
-rw-r--r--lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_7_SUITE.erl113
-rw-r--r--lib/common_test/test/ct_surefire_SUITE.erl7
-rw-r--r--lib/common_test/test/ct_test_server_if_1_SUITE.erl182
-rw-r--r--lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_7_SUITE.erl16
-rw-r--r--lib/common_test/test/ct_test_support.erl30
-rw-r--r--lib/common_test/test/ct_testspec_1_SUITE.erl230
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE.erl10
-rw-r--r--lib/test_server/src/test_server.erl12
-rw-r--r--lib/test_server/src/test_server_ctrl.erl422
-rw-r--r--lib/test_server/test/test_server_SUITE.erl14
-rw-r--r--lib/test_server/test/test_server_test_lib.erl5
28 files changed, 1726 insertions, 616 deletions
diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml
index b3e713c77f..859ff9df14 100644
--- a/lib/common_test/doc/src/ct_hooks.xml
+++ b/lib/common_test/doc/src/ct_hooks.xml
@@ -485,8 +485,9 @@
NewCTHState</name>
<fsummary>Called after the CTH scope ends</fsummary>
<type>
- <v>TestcaseName = end_per_suite | init_per_group |
- end_per_group | atom()</v>
+ <v>TestcaseName = end_per_suite | {init_per_group,GroupName} |
+ {end_per_group,GroupName} | atom()</v>
+ <v>GroupName = atom()</v>
<v>Reason = {tc_auto_skip | tc_user_skip, term()}</v>
<v>CTHState = NewCTHState = term()</v>
</type>
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index 276f902b05..e81b69a1b5 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -69,13 +69,13 @@ init_tc(Mod,Func,Config) ->
andalso Func=/=end_per_group
andalso ct_util:get_testdata(skip_rest) of
true ->
- {skip,"Repeated test stopped by force_stop option"};
+ {auto_skip,"Repeated test stopped by force_stop option"};
_ ->
case ct_util:get_testdata(curr_tc) of
{Suite,{suite0_failed,{require,Reason}}} ->
- {skip,{require_failed_in_suite0,Reason}};
+ {auto_skip,{require_failed_in_suite0,Reason}};
{Suite,{suite0_failed,_}=Failure} ->
- {skip,Failure};
+ {fail,Failure};
_ ->
ct_util:update_testdata(curr_tc,
fun(undefined) ->
@@ -103,7 +103,7 @@ init_tc(Mod,Func,Config) ->
end,
init_tc1(Mod,Suite,Func,Config);
{failed,Seq,BadFunc} ->
- {skip,{sequence_failed,Seq,BadFunc}}
+ {auto_skip,{sequence_failed,Seq,BadFunc}}
end
end
end.
@@ -115,9 +115,9 @@ init_tc1(?MODULE,_,error_in_suite,[Config0]) when is_list(Config0) ->
data={?MODULE,error_in_suite}}),
case ?val(error, Config0) of
undefined ->
- {skip,"unknown_error_in_suite"};
+ {fail,"unknown_error_in_suite"};
Reason ->
- {skip,Reason}
+ {fail,Reason}
end;
init_tc1(Mod,Suite,Func,[Config0]) when is_list(Config0) ->
@@ -159,22 +159,28 @@ init_tc1(Mod,Suite,Func,[Config0]) when is_list(Config0) ->
true ->
ct_config:delete_default_config(testcase)
end,
+ Initialize = fun() ->
+ ct_logs:init_tc(false),
+ ct_event:notify(#event{name=tc_start,
+ node=node(),
+ data={Mod,FuncSpec}})
+ end,
case add_defaults(Mod,Func,AllGroups) of
Error = {suite0_failed,_} ->
- ct_logs:init_tc(false),
- ct_event:notify(#event{name=tc_start,
- node=node(),
- data={Mod,FuncSpec}}),
+ Initialize(),
ct_util:set_testdata({curr_tc,{Suite,Error}}),
{error,Error};
+ Error = {group0_failed,_} ->
+ Initialize(),
+ {auto_skip,Error};
+ Error = {testcase0_failed,_} ->
+ Initialize(),
+ {auto_skip,Error};
{SuiteInfo,MergeResult} ->
case MergeResult of
{error,Reason} ->
- ct_logs:init_tc(false),
- ct_event:notify(#event{name=tc_start,
- node=node(),
- data={Mod,FuncSpec}}),
- {skip,Reason};
+ Initialize(),
+ {fail,Reason};
_ ->
init_tc2(Mod,Suite,Func,SuiteInfo,MergeResult,Config)
end
@@ -222,11 +228,11 @@ init_tc2(Mod,Suite,Func,SuiteInfo,MergeResult,Config) ->
{suite0_failed,Reason} ->
ct_util:set_testdata({curr_tc,{Mod,{suite0_failed,
{require,Reason}}}}),
- {skip,{require_failed_in_suite0,Reason}};
+ {auto_skip,{require_failed_in_suite0,Reason}};
{error,Reason} ->
{auto_skip,{require_failed,Reason}};
{'EXIT',Reason} ->
- {auto_skip,Reason};
+ {fail,Reason};
{ok,PostInitHook,Config1} ->
case get('$test_server_framework_test') of
undefined ->
@@ -272,6 +278,8 @@ add_defaults(Mod,Func, GroupPath) ->
SuiteInfo = merge_with_suite_defaults(Suite,[]),
SuiteInfoNoCTH = [I || I <- SuiteInfo, element(1,I) =/= ct_hooks],
case add_defaults1(Mod,Func, GroupPath, SuiteInfoNoCTH) of
+ Error = {group0_failed,_} -> Error;
+ Error = {testcase0_failed,_} -> Error;
Error = {error,_} -> {SuiteInfo,Error};
MergedInfo -> {SuiteInfo,MergedInfo}
end;
@@ -292,13 +300,16 @@ add_defaults(Mod,Func, GroupPath) ->
element(1,I) =/= ct_hooks],
case add_defaults1(Mod,Func, GroupPath,
SuiteInfoNoCTH) of
+ Error = {group0_failed,_} -> Error;
+ Error = {testcase0_failed,_} -> Error;
Error = {error,_} -> {SuiteInfo1,Error};
MergedInfo -> {SuiteInfo1,MergedInfo}
end;
false ->
ErrStr = io_lib:format("~n*** ERROR *** "
"Invalid return value from "
- "~w:suite/0: ~p~n", [Suite,SuiteInfo]),
+ "~w:suite/0: ~p~n",
+ [Suite,SuiteInfo]),
io:format(ErrStr, []),
io:format(user, ErrStr, []),
{suite0_failed,bad_return_value}
@@ -318,36 +329,69 @@ add_defaults1(Mod,Func, GroupPath, SuiteInfo) ->
%% [LevelXGroupInfo, LevelX-1GroupInfo, ..., TopLevelGroupInfo]
GroupPathInfo =
lists:map(fun(GroupProps) ->
- Name = ?val(name, GroupProps),
- case catch Suite:group(Name) of
- GrInfo when is_list(GrInfo) -> GrInfo;
- _ -> []
+ case ?val(name, GroupProps) of
+ undefined ->
+ [];
+ Name ->
+ case catch Suite:group(Name) of
+ GrInfo when is_list(GrInfo) -> GrInfo;
+ {'EXIT',{undef,_}} -> [];
+ BadGr0 -> {error,BadGr0,Name}
+ end
end
end, GroupPath),
- Args = if Func == init_per_group ; Func == end_per_group ->
- [?val(name, hd(GroupPath))];
- true ->
- []
- end,
- TestCaseInfo =
- case catch apply(Mod,Func,Args) of
- TCInfo when is_list(TCInfo) -> TCInfo;
- _ -> []
- end,
- %% let test case info (also for all config funcs) override group info,
- %% and lower level group info override higher level info
- TCAndGroupInfo = [TestCaseInfo | remove_info_in_prev(TestCaseInfo,
- GroupPathInfo)],
- %% find and save require terms found in suite info
- SuiteReqs =
- [SDDef || SDDef <- SuiteInfo,
- ((require == element(1,SDDef)) or
- (default_config == element(1,SDDef)))],
- case check_for_clashes(TestCaseInfo, GroupPathInfo, SuiteReqs) of
- [] ->
- add_defaults2(Mod,Func, TCAndGroupInfo,SuiteInfo,SuiteReqs);
- Clashes ->
- {error,{config_name_already_in_use,Clashes}}
+ case lists:keysearch(error, 1, GroupPathInfo) of
+ {value,{error,BadGr0Val,GrName}} ->
+ Gr0ErrStr = io_lib:format("~n*** ERROR *** "
+ "Invalid return value from "
+ "~w:group(~w): ~p~n",
+ [Mod,GrName,BadGr0Val]),
+ io:format(Gr0ErrStr, []),
+ io:format(user, Gr0ErrStr, []),
+ {group0_failed,bad_return_value};
+ _ ->
+ Args = if Func == init_per_group ; Func == end_per_group ->
+ [?val(name, hd(GroupPath))];
+ true ->
+ []
+ end,
+ TestCaseInfo =
+ case catch apply(Mod,Func,Args) of
+ TCInfo when is_list(TCInfo) -> TCInfo;
+ {'EXIT',{undef,_}} -> [];
+ BadTC0 -> {error,BadTC0}
+ end,
+
+ case TestCaseInfo of
+ {error,BadTC0Val} ->
+ TC0ErrStr = io_lib:format("~n*** ERROR *** "
+ "Invalid return value from "
+ "~w:~w/0: ~p~n",
+ [Mod,Func,BadTC0Val]),
+ io:format(TC0ErrStr, []),
+ io:format(user, TC0ErrStr, []),
+ {testcase0_failed,bad_return_value};
+ _ ->
+ %% let test case info (also for all config funcs) override
+ %% group info, and lower level group info override higher
+ %% level info
+ TCAndGroupInfo =
+ [TestCaseInfo | remove_info_in_prev(TestCaseInfo,
+ GroupPathInfo)],
+ %% find and save require terms found in suite info
+ SuiteReqs =
+ [SDDef || SDDef <- SuiteInfo,
+ ((require == element(1,SDDef))
+ or (default_config == element(1,SDDef)))],
+ case check_for_clashes(TestCaseInfo, GroupPathInfo,
+ SuiteReqs) of
+ [] ->
+ add_defaults2(Mod,Func, TCAndGroupInfo,
+ SuiteInfo,SuiteReqs);
+ Clashes ->
+ {error,{config_name_already_in_use,Clashes}}
+ end
+ end
end.
get_suite_name(?MODULE, [Cfg|_]) when is_list(Cfg), Cfg /= [] ->
@@ -621,31 +665,34 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) ->
_ -> Func
end,
- case get('$test_server_framework_test') of
- undefined ->
- {FinalResult,FinalNotify} =
- case ct_hooks:end_tc(
- Suite, FuncSpec, Args, Result, Return) of
- '$ct_no_change' ->
- {ok,Result};
- FinalResult1 ->
- {FinalResult1,FinalResult1}
- end,
- %% send sync notification so that event handlers may print
- %% in the log file before it gets closed
- ct_event:sync_notify(#event{name=tc_done,
- node=node(),
- data={Mod,FuncSpec,
- tag_cth(FinalNotify)}});
- Fun ->
- %% send sync notification so that event handlers may print
- %% in the log file before it gets closed
- ct_event:sync_notify(#event{name=tc_done,
- node=node(),
- data={Mod,FuncSpec,tag(Result)}}),
- FinalResult = Fun(end_tc, Return)
- end,
-
+ {Result1,FinalNotify} =
+ case ct_hooks:end_tc(
+ Suite, FuncSpec, Args, Result, Return) of
+ '$ct_no_change' ->
+ {ok,Result};
+ HookResult ->
+ {HookResult,HookResult}
+ end,
+ FinalResult =
+ case get('$test_server_framework_test') of
+ undefined ->
+ %% send sync notification so that event handlers may print
+ %% in the log file before it gets closed
+ ct_event:sync_notify(#event{name=tc_done,
+ node=node(),
+ data={Mod,FuncSpec,
+ tag_cth(FinalNotify)}}),
+ Result1;
+ Fun ->
+ %% send sync notification so that event handlers may print
+ %% in the log file before it gets closed
+ ct_event:sync_notify(#event{name=tc_done,
+ node=node(),
+ data={Mod,FuncSpec,
+ tag(FinalNotify)}}),
+ Fun(end_tc, Return)
+ end,
+
case FuncSpec of
{_,GroupName,_Props} ->
if Func == end_per_group ->
@@ -685,10 +732,10 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) ->
(Unexpected) ->
exit({error,{reset_curr_tc,{Mod,Func},Unexpected}})
end,
- ct_util:update_testdata(curr_tc,ClearCurrTC),
+ ct_util:update_testdata(curr_tc, ClearCurrTC),
case FinalResult of
- {skip,{sequence_failed,_,_}} ->
+ {auto_skip,{sequence_failed,_,_}} ->
%% ct_logs:init_tc is never called for a skipped test case
%% in a failing sequence, so neither should end_tc
ok;
@@ -711,8 +758,13 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) ->
%% {error,Reason} | {skip,Reason} | {timetrap_timeout,TVal} |
%% {testcase_aborted,Reason} | testcase_aborted_or_killed |
%% {'EXIT',Reason} | Other (ignored return value, e.g. 'ok')
-tag({STag,Reason}) when STag == skip; STag == skipped ->
- {skipped,Reason};
+tag({STag,Reason}) when STag == skip; STag == skipped ->
+ case Reason of
+ {failed,{_,init_per_testcase,_}} -> {auto_skipped,Reason};
+ _ -> {skipped,Reason}
+ end;
+tag({auto_skip,Reason}) ->
+ {auto_skipped,Reason};
tag(E = {ETag,_}) when ETag == error; ETag == 'EXIT';
ETag == timetrap_timeout;
ETag == testcase_aborted ->
@@ -722,13 +774,20 @@ tag(E = testcase_aborted_or_killed) ->
tag(Other) ->
Other.
+tag_cth({skipped,Reason={failed,{_,init_per_testcase,_}}}) ->
+ {auto_skipped,Reason};
tag_cth({STag,Reason}) when STag == skip; STag == skipped ->
- {skipped,Reason};
-tag_cth({fail, Reason}) ->
- {failed, {error,Reason}};
+ case Reason of
+ {failed,{_,init_per_testcase,_}} -> {auto_skipped,Reason};
+ _ -> {skipped,Reason}
+ end;
+tag_cth({auto_skip,Reason}) ->
+ {auto_skipped,Reason};
+tag_cth({fail,Reason}) ->
+ {failed,{error,Reason}};
tag_cth(E = {ETag,_}) when ETag == error; ETag == 'EXIT';
- ETag == timetrap_timeout;
- ETag == testcase_aborted ->
+ ETag == timetrap_timeout;
+ ETag == testcase_aborted ->
{failed,E};
tag_cth(E = testcase_aborted_or_killed) ->
{failed,E};
@@ -1229,6 +1288,8 @@ report(What,Data) ->
ct_hooks:on_tc_skip(tc_auto_skip, Data);
{skipped,_} ->
ct_hooks:on_tc_skip(tc_user_skip, Data);
+ {auto_skipped,_} ->
+ ct_hooks:on_tc_skip(tc_auto_skip, Data);
_Else ->
ok
end,
@@ -1253,31 +1314,54 @@ report(What,Data) ->
add_to_stats(auto_skipped);
{_,{skipped,_}} ->
add_to_stats(user_skipped);
+ {_,{auto_skipped,_}} ->
+ add_to_stats(auto_skipped);
{_,{SkipOrFail,_Reason}} ->
add_to_stats(SkipOrFail)
end;
- tc_user_skip ->
- %% test case specified as skipped in testspec
- %% Data = {Suite,Case,Comment}
+ tc_user_skip ->
+ %% test case or config function specified as skipped in testspec,
+ %% or init config func for suite/group has returned {skip,Reason}
+ %% Data = {Suite,Case,Comment} |
+ %% {Suite,{GroupConfigFunc,GroupName},Comment}
+ {Func,Data1} = case Data of
+ {Suite,{ConfigFunc,undefined},Cmt} ->
+ {ConfigFunc,{Suite,ConfigFunc,Cmt}};
+ {_,{ConfigFunc,_},_} -> {ConfigFunc,Data};
+ {_,Case,_} -> {Case,Data}
+ end,
+
ct_event:sync_notify(#event{name=tc_user_skip,
node=node(),
- data=Data}),
- ct_hooks:on_tc_skip(What, Data),
- add_to_stats(user_skipped);
- tc_auto_skip ->
- %% test case skipped because of error in init_per_suite
- %% Data = {Suite,Case,Comment}
-
- {_Suite,Case,_Result} = Data,
+ data=Data1}),
+ ct_hooks:on_tc_skip(What, Data1),
+ if Func /= init_per_suite, Func /= init_per_group,
+ Func /= end_per_suite, Func /= end_per_group ->
+ add_to_stats(user_skipped);
+ true ->
+ ok
+ end;
+ tc_auto_skip ->
+ %% test case skipped because of error in config function, or
+ %% config function skipped because of error in info function
+ %% Data = {Suite,Case,Comment} |
+ %% {Suite,{GroupConfigFunc,GroupName},Comment}
+ {Func,Data1} = case Data of
+ {Suite,{ConfigFunc,undefined},Cmt} ->
+ {ConfigFunc,{Suite,ConfigFunc,Cmt}};
+ {_,{ConfigFunc,_},_} -> {ConfigFunc,Data};
+ {_,Case,_} -> {Case,Data}
+ end,
%% this test case does not have a log, so printouts
%% from event handlers should end up in the main log
ct_event:sync_notify(#event{name=tc_auto_skip,
node=node(),
- data=Data}),
- ct_hooks:on_tc_skip(What, Data),
- if Case /= end_per_suite,
- Case /= end_per_group ->
+ data=Data1}),
+ ct_hooks:on_tc_skip(What, Data1),
+
+ if Func /= end_per_suite,
+ Func /= end_per_group ->
add_to_stats(auto_skipped);
true ->
ok
diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl
index 1a6e4d31a8..a7fb45a4e4 100644
--- a/lib/common_test/src/ct_logs.erl
+++ b/lib/common_test/src/ct_logs.erl
@@ -1249,7 +1249,8 @@ make_one_index_entry1(SuiteName, Link, Label, Success, Fail, UserSkip, AutoSkip,
integer_to_list(NotBuilt),"</a></td>\n"]
end,
FailStr =
- if Fail > 0 ->
+ if (Fail > 0) or (NotBuilt > 0) or
+ ((Success+Fail+UserSkip+AutoSkip) == 0) ->
["<font color=\"red\">",
integer_to_list(Fail),"</font>"];
true ->
@@ -1904,7 +1905,8 @@ runentry(Dir, undefined, _) ->
runentry(Dir, Totals={Node,Label,Logs,
{TotSucc,TotFail,UserSkip,AutoSkip,NotBuilt}}, Index) ->
TotFailStr =
- if TotFail > 0 ->
+ if (TotFail > 0) or (NotBuilt > 0) or
+ ((TotSucc+TotFail+UserSkip+AutoSkip) == 0) ->
["<font color=\"red\">",
integer_to_list(TotFail),"</font>"];
true ->
diff --git a/lib/common_test/src/cth_surefire.erl b/lib/common_test/src/cth_surefire.erl
index 1a38b6584b..7ed2018bdf 100644
--- a/lib/common_test/src/cth_surefire.erl
+++ b/lib/common_test/src/cth_surefire.erl
@@ -138,6 +138,9 @@ on_tc_fail(_TC, Res, State) ->
{fail,lists:flatten(io_lib:format("~p",[Res]))} },
State#state{ test_cases = [NewTC | tl(TCs)]}.
+on_tc_skip({ConfigFunc,_GrName},{Type,_Reason} = Res, State0)
+ when Type == tc_auto_skip; Type == tc_user_skip ->
+ on_tc_skip(ConfigFunc, Res, State0);
on_tc_skip(Tc,{Type,_Reason} = Res, State0) when Type == tc_auto_skip ->
TcStr = atom_to_list(Tc),
State =
@@ -330,5 +333,7 @@ count_tcs([#testcase{result={fail,_}}|TCs],Ok,F,S) ->
count_tcs(TCs,Ok,F+1,S);
count_tcs([#testcase{result={skipped,_}}|TCs],Ok,F,S) ->
count_tcs(TCs,Ok,F,S+1);
+count_tcs([#testcase{result={auto_skipped,_}}|TCs],Ok,F,S) ->
+ count_tcs(TCs,Ok,F,S+1);
count_tcs([],Ok,F,S) ->
{Ok+F+S,F,S}.
diff --git a/lib/common_test/test/ct_config_SUITE.erl b/lib/common_test/test/ct_config_SUITE.erl
index d92be9ec6e..284f836517 100644
--- a/lib/common_test/test/ct_config_SUITE.erl
+++ b/lib/common_test/test/ct_config_SUITE.erl
@@ -234,25 +234,25 @@ expected_events(config_static_SUITE)->
?sok(test_get_config_deep_nested,{3,0,{0,0}}),
?sok(test_default_suitewide,{4,0,{0,0}}),
?snok(test_config_name_already_in_use1,
- {skipped,{config_name_already_in_use,[x1]}},{4,0,{1,0}}),
- ?sok(test_default_tclocal,{5,0,{1,0}}),
+ {failed,{error,{config_name_already_in_use,[x1]}}},{4,1,{0,0}}),
+ ?sok(test_default_tclocal,{5,1,{0,0}}),
?snok(test_config_name_already_in_use2,
- {skipped,{config_name_already_in_use,[alias,x1]}},{5,0,{2,0}}),
- ?sok(test_alias_tclocal,{6,0,{2,0}}),
- ?sok(test_get_config_undefined,{7,0,{2,0}}),
- ?sok(test_require_subvals,{8,0,{2,0}}),
+ {failed,{error,{config_name_already_in_use,[alias,x1]}}},{5,2,{0,0}}),
+ ?sok(test_alias_tclocal,{6,2,{0,0}}),
+ ?sok(test_get_config_undefined,{7,2,{0,0}}),
+ ?sok(test_require_subvals,{8,2,{0,0}}),
?snok(test_require_subvals2,
- {skipped,{require_failed,
- {not_available,{gen_cfg,[a,b,c,d]}}}},{8,0,{2,1}}),
- ?sok(test_require_deep_config,{9,0,{2,1}}),
- ?sok(test_shadow_all,{10,0,{2,1}}),
- ?sok(test_element,{11,0,{2,1}}),
- ?sok(test_shadow_all_element,{12,0,{2,1}}),
- ?sok(test_internal_deep,{13,0,{2,1}}),
- ?sok(test_alias_tclocal_nested,{14,0,{2,1}}),
- ?sok(test_alias_tclocal_nested_backward_compat,{15,0,{2,1}}),
- ?sok(test_alias_tclocal_nested_backward_compat_subvals,{16,0,{2,1}}),
- ?sok(test_config_same_name_already_in_use,{17,0,{2,1}}),
+ {auto_skipped,{require_failed,
+ {not_available,{gen_cfg,[a,b,c,d]}}}},{8,2,{0,1}}),
+ ?sok(test_require_deep_config,{9,2,{0,1}}),
+ ?sok(test_shadow_all,{10,2,{0,1}}),
+ ?sok(test_element,{11,2,{0,1}}),
+ ?sok(test_shadow_all_element,{12,2,{0,1}}),
+ ?sok(test_internal_deep,{13,2,{0,1}}),
+ ?sok(test_alias_tclocal_nested,{14,2,{0,1}}),
+ ?sok(test_alias_tclocal_nested_backward_compat,{15,2,{0,1}}),
+ ?sok(test_alias_tclocal_nested_backward_compat_subvals,{16,2,{0,1}}),
+ ?sok(test_config_same_name_already_in_use,{17,2,{0,1}}),
{?eh,tc_start,{config_static_SUITE,end_per_suite}},
{?eh,tc_done,{config_static_SUITE,end_per_suite,ok}},
{?eh,test_done,{'DEF','STOP_TIME'}},
diff --git a/lib/common_test/test/ct_config_info_SUITE.erl b/lib/common_test/test/ct_config_info_SUITE.erl
index 10fe8286dd..8f2f0eb75f 100644
--- a/lib/common_test/test/ct_config_info_SUITE.erl
+++ b/lib/common_test/test/ct_config_info_SUITE.erl
@@ -127,7 +127,7 @@ test_events(config_info) ->
{failed,{timetrap_timeout,350}}}},
{?eh,tc_auto_skip,{config_info_1_SUITE,t11,
{failed,{config_info_1_SUITE,init_per_group,{timetrap_timeout,350}}}}},
- {?eh,tc_auto_skip,{config_info_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{config_info_1_SUITE,{end_per_group,g1},
{failed,{config_info_1_SUITE,init_per_group,
{timetrap_timeout,350}}}}}],
@@ -145,12 +145,12 @@ test_events(config_info) ->
{?eh,tc_auto_skip,{config_info_1_SUITE,t41,
{failed,{config_info_1_SUITE,init_per_group,
{timetrap_timeout,400}}}}},
- {?eh,tc_auto_skip,{config_info_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{config_info_1_SUITE,{end_per_group,g4},
{failed,{config_info_1_SUITE,init_per_group,
{timetrap_timeout,400}}}}}],
{?eh,tc_start,{config_info_1_SUITE,t31}},
{?eh,tc_done,{config_info_1_SUITE,t31,
- {skipped,{failed,{config_info_1_SUITE,init_per_testcase,
+ {auto_skipped,{failed,{config_info_1_SUITE,init_per_testcase,
{timetrap_timeout,250}}}}}},
{?eh,tc_start,{config_info_1_SUITE,t32}},
{?eh,tc_done,{config_info_1_SUITE,t32,
diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl
index 3881ced17d..28f0494d20 100644
--- a/lib/common_test/test/ct_error_SUITE.erl
+++ b/lib/common_test/test/ct_error_SUITE.erl
@@ -465,7 +465,7 @@ test_events(cfg_error) ->
{'EXIT',{init_per_group_fails,g1}}}}}},
{?eh,test_stats,{4,0,{0,11}}},
{?eh,tc_auto_skip,
- {cfg_error_8_SUITE,end_per_group,
+ {cfg_error_8_SUITE,{end_per_group,g1},
{failed,{cfg_error_8_SUITE,init_per_group,
{'EXIT',{init_per_group_fails,g1}}}}}}],
@@ -476,7 +476,7 @@ test_events(cfg_error) ->
{failed,{cfg_error_8_SUITE,init_per_group,
{timetrap_timeout,2000}}}}},
{?eh,test_stats,{4,0,{0,12}}},
- {?eh,tc_auto_skip,{cfg_error_8_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{cfg_error_8_SUITE,{end_per_group,g2},
{failed,{cfg_error_8_SUITE,init_per_group,
{timetrap_timeout,2000}}}}}],
@@ -490,7 +490,7 @@ test_events(cfg_error) ->
{'EXIT',{{badmatch,42},'_'}}}}}},
{?eh,test_stats,{4,0,{0,13}}},
{?eh,tc_auto_skip,
- {cfg_error_8_SUITE,end_per_group,
+ {cfg_error_8_SUITE,{end_per_group,g3},
{failed,{cfg_error_8_SUITE,init_per_group,
{'EXIT',{{badmatch,42},'_'}}}}}}],
@@ -516,7 +516,7 @@ test_events(cfg_error) ->
{'EXIT',{sub_group_failed,g6}}}}}},
{?eh,test_stats,{6,0,{0,14}}},
{?eh,tc_auto_skip,
- {cfg_error_8_SUITE,end_per_group,
+ {cfg_error_8_SUITE,{end_per_group,g6},
{failed,{cfg_error_8_SUITE,init_per_group,
{'EXIT',{sub_group_failed,g6}}}}}}],
{?eh,tc_start,{cfg_error_8_SUITE,tc3}},
@@ -550,23 +550,23 @@ test_events(cfg_error) ->
{?eh,tc_done,{cfg_error_9_SUITE,init_per_suite,ok}},
{?eh,tc_start,{cfg_error_9_SUITE,tc1}},
{?eh,tc_done,{cfg_error_9_SUITE,tc1,
- {skipped,{failed,{cfg_error_9_SUITE,init_per_testcase,
- {tc1_should_be_skipped,'_'}}}}}},
+ {auto_skipped,{failed,{cfg_error_9_SUITE,init_per_testcase,
+ {tc1_should_be_skipped,'_'}}}}}},
{?eh,test_stats,{9,0,{0,15}}},
{?eh,tc_start,{cfg_error_9_SUITE,tc2}},
{?eh,tc_done,{cfg_error_9_SUITE,tc2,
- {skipped,{failed,{cfg_error_9_SUITE,init_per_testcase,
- {timetrap_timeout,2000}}}}}},
+ {auto_skipped,{failed,{cfg_error_9_SUITE,init_per_testcase,
+ {timetrap_timeout,2000}}}}}},
{?eh,test_stats,{9,0,{0,16}}},
{?eh,tc_start,{cfg_error_9_SUITE,tc3}},
{?eh,tc_done,{cfg_error_9_SUITE,tc3,
- {skipped,{failed,{cfg_error_9_SUITE,init_per_testcase,
- {{badmatch,undefined},'_'}}}}}},
+ {auto_skipped,{failed,{cfg_error_9_SUITE,init_per_testcase,
+ {{badmatch,undefined},'_'}}}}}},
{?eh,test_stats,{9,0,{0,17}}},
{?eh,tc_start,{cfg_error_9_SUITE,tc4}},
{?eh,tc_done,
{cfg_error_9_SUITE,tc4,
- {skipped,{failed,{cfg_error_9_SUITE,init_per_testcase,bad_return}}}}},
+ {auto_skipped,{failed,{cfg_error_9_SUITE,init_per_testcase,bad_return}}}}},
{?eh,test_stats,{9,0,{0,18}}},
{?eh,tc_start,{cfg_error_9_SUITE,tc5}},
{?eh,tc_done,
@@ -622,41 +622,41 @@ test_events(cfg_error) ->
{?eh,tc_done,{cfg_error_11_SUITE,init_per_suite,ok}},
{?eh,tc_start,{cfg_error_11_SUITE,tc1}},
{?eh,tc_done, {cfg_error_11_SUITE,tc1,
- {skipped,{config_name_already_in_use,[dummy_alias]}}}},
- {?eh,test_stats,{12,6,{1,19}}},
+ {failed,{error,{config_name_already_in_use,[dummy_alias]}}}}},
+ {?eh,test_stats,{12,7,{0,19}}},
{?eh,tc_start,{cfg_error_11_SUITE,tc2}},
{?eh,tc_done,{cfg_error_11_SUITE,tc2,ok}},
- {?eh,test_stats,{13,6,{1,19}}},
+ {?eh,test_stats,{13,7,{0,19}}},
{?eh,tc_start,{cfg_error_11_SUITE,end_per_suite}},
{?eh,tc_done,{cfg_error_11_SUITE,end_per_suite,ok}},
{?eh,tc_start,{cfg_error_12_SUITE,tc1}},
{?eh,tc_done,{ct_framework,init_tc,{framework_error,{timetrap,500}}}},
- {?eh,test_stats,{13,7,{1,19}}},
+ {?eh,test_stats,{13,8,{0,19}}},
{?eh,tc_start,{cfg_error_12_SUITE,tc2}},
{?eh,tc_done,{cfg_error_12_SUITE,tc2,{failed,
{cfg_error_12_SUITE,end_per_testcase,
{timetrap_timeout,500}}}}},
- {?eh,test_stats,{14,7,{1,19}}},
+ {?eh,test_stats,{14,8,{0,19}}},
{?eh,tc_start,{cfg_error_12_SUITE,tc3}},
{?eh,tc_done,{cfg_error_12_SUITE,tc3,ok}},
- {?eh,test_stats,{15,7,{1,19}}},
+ {?eh,test_stats,{15,8,{0,19}}},
{?eh,tc_start,{cfg_error_12_SUITE,tc4}},
{?eh,tc_done,{cfg_error_12_SUITE,tc4,{failed,
{cfg_error_12_SUITE,end_per_testcase,
{timetrap_timeout,500}}}}},
- {?eh,test_stats,{16,7,{1,19}}},
+ {?eh,test_stats,{16,8,{0,19}}},
{?eh,tc_start,{cfg_error_13_SUITE,init_per_suite}},
{?eh,tc_done,{cfg_error_13_SUITE,init_per_suite,ok}},
{?eh,tc_start,{cfg_error_13_SUITE,tc1}},
{?eh,tc_done,{cfg_error_13_SUITE,tc1,ok}},
- {?eh,test_stats,{17,7,{1,19}}},
+ {?eh,test_stats,{17,8,{0,19}}},
{?eh,tc_start,{cfg_error_13_SUITE,end_per_suite}},
{?eh,tc_done,{cfg_error_13_SUITE,end_per_suite,ok}},
{?eh,tc_start,{cfg_error_14_SUITE,init_per_suite}},
{?eh,tc_done,{cfg_error_14_SUITE,init_per_suite,ok}},
{?eh,tc_start,{cfg_error_14_SUITE,tc1}},
{?eh,tc_done,{cfg_error_14_SUITE,tc1,ok}},
- {?eh,test_stats,{18,7,{1,19}}},
+ {?eh,test_stats,{18,8,{0,19}}},
{?eh,tc_start,{cfg_error_14_SUITE,end_per_suite}},
{?eh,tc_done,{cfg_error_14_SUITE,end_per_suite,
{comment,
@@ -1102,7 +1102,7 @@ test_events(timetrap_fun_group) ->
{failed,{timetrap_8_SUITE,init_per_group,
{user_timetrap_error,{kaboom,'_'}}}}}},
{?eh,test_stats,{0,11,{0,2}}},
- {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,g4},
{failed,{timetrap_8_SUITE,init_per_group,
{user_timetrap_error,{kaboom,'_'}}}}}}],
@@ -1117,7 +1117,7 @@ test_events(timetrap_fun_group) ->
{failed,{timetrap_8_SUITE,init_per_group,
{user_timetrap_error,{kaboom,'_'}}}}}},
{?eh,test_stats,{0,11,{0,4}}},
- {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,g5},
{failed,{timetrap_8_SUITE,init_per_group,
{user_timetrap_error,{kaboom,'_'}}}}}}],
@@ -1132,7 +1132,7 @@ test_events(timetrap_fun_group) ->
{failed,{timetrap_8_SUITE,init_per_group,
{timetrap_timeout,'_'}}}}},
{?eh,test_stats,{0,11,{0,6}}},
- {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,g6},
{failed,{timetrap_8_SUITE,init_per_group,
{timetrap_timeout,'_'}}}}}],
@@ -1285,7 +1285,7 @@ test_events(timetrap_fun_group) ->
{failed,{timetrap_8_SUITE,init_per_group,
{user_timetrap_error,{kaboom,'_'}}}}}},
{?eh,test_stats,{4,26,{0,8}}},
- {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,pg4},
{failed,{timetrap_8_SUITE,init_per_group,
{user_timetrap_error,{kaboom,'_'}}}}}}]},
@@ -1301,7 +1301,7 @@ test_events(timetrap_fun_group) ->
{failed,{timetrap_8_SUITE,init_per_group,
{user_timetrap_error,{kaboom,'_'}}}}}},
{?eh,test_stats,{4,26,{0,10}}},
- {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,pg5},
{failed,{timetrap_8_SUITE,init_per_group,
{user_timetrap_error,{kaboom,'_'}}}}}}]},
@@ -1317,7 +1317,7 @@ test_events(timetrap_fun_group) ->
{failed,{timetrap_8_SUITE,init_per_group,
{timetrap_timeout,'_'}}}}},
{?eh,test_stats,{4,26,{0,12}}},
- {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,pg6},
{failed,{timetrap_8_SUITE,init_per_group,
{timetrap_timeout,'_'}}}}}]},
diff --git a/lib/common_test/test/ct_group_info_SUITE.erl b/lib/common_test/test/ct_group_info_SUITE.erl
index c56fa952e8..e7bc5baaa1 100644
--- a/lib/common_test/test/ct_group_info_SUITE.erl
+++ b/lib/common_test/test/ct_group_info_SUITE.erl
@@ -269,13 +269,19 @@ test_events(timetrap_all) ->
{?eh,tc_done,{group_timetrap_1_SUITE,{end_per_group,g10,[]},ok}}],
[{?eh,tc_start,{group_timetrap_1_SUITE,{init_per_group,g11,[]}}},
- {?eh,tc_done,{group_timetrap_1_SUITE,{init_per_group,g11,[]},ok}},
- {?eh,tc_done,{group_timetrap_1_SUITE,t111,{failed,{timetrap_timeout,1000}}}},
- {?eh,test_stats,{0,14,{0,0}}},
- {?eh,tc_start,{group_timetrap_1_SUITE,{end_per_group,g11,[]}}},
- {?eh,tc_done,{group_timetrap_1_SUITE,{end_per_group,g11,[]},ok}}],
-
+ {?eh,tc_done,{group_timetrap_1_SUITE,
+ {init_per_group,g11,[]},
+ {auto_skipped,{group0_failed,bad_return_value}}}},
+ {?eh,tc_auto_skip,
+ {group_timetrap_1_SUITE,t111,{group0_failed,bad_return_value}}},
+ {?eh,test_stats,{0,13,{0,1}}},
+ {?eh,tc_auto_skip,{group_timetrap_1_SUITE,
+ {end_per_group,g11},
+ {group0_failed,bad_return_value}}}],
+
+ {?eh,tc_start,{group_timetrap_1_SUITE,end_per_suite}},
{?eh,tc_done,{group_timetrap_1_SUITE,end_per_suite,ok}},
+
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}
];
@@ -422,13 +428,15 @@ test_events(timetrap_all_no_ips) ->
{?eh,tc_done,{group_timetrap_2_SUITE,{end_per_group,g10,[]},ok}}],
[{?eh,tc_start,{group_timetrap_2_SUITE,{init_per_group,g11,[]}}},
- {?eh,tc_done,{group_timetrap_2_SUITE,{init_per_group,g11,[]},ok}},
- {?eh,tc_done,{group_timetrap_2_SUITE,t111,{failed,{timetrap_timeout,1000}}}},
- {?eh,test_stats,{0,14,{0,0}}},
- {?eh,tc_start,{group_timetrap_2_SUITE,{end_per_group,g11,[]}}},
- {?eh,tc_done,{group_timetrap_2_SUITE,{end_per_group,g11,[]},ok}}],
-
- {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,tc_done,{group_timetrap_2_SUITE,
+ {init_per_group,g11,[]},
+ {auto_skipped,{group0_failed,bad_return_value}}}},
+ {?eh,tc_auto_skip,{group_timetrap_2_SUITE,t111,
+ {group0_failed,bad_return_value}}},
+ {?eh,test_stats,{0,13,{0,1}}},
+ {?eh,tc_auto_skip,{group_timetrap_2_SUITE,
+ {end_per_group,g11},
+ {group0_failed,bad_return_value}}}],
{?eh,stop_logging,[]}
];
@@ -501,11 +509,13 @@ test_events(timetrap_all_no_ipg) ->
{?eh,tc_done,{ct_framework,{end_per_group,g10,[{suite,group_timetrap_3_SUITE}]},ok}}],
[{?eh,tc_start,{ct_framework,{init_per_group,g11,[{suite,group_timetrap_3_SUITE}]}}},
- {?eh,tc_done,{ct_framework,{init_per_group,g11,[{suite,group_timetrap_3_SUITE}]},ok}},
- {?eh,tc_done,{group_timetrap_3_SUITE,t111,{failed,{timetrap_timeout,1000}}}},
- {?eh,test_stats,{0,14,{0,0}}},
- {?eh,tc_start,{ct_framework,{end_per_group,g11,[{suite,group_timetrap_3_SUITE}]}}},
- {?eh,tc_done,{ct_framework,{end_per_group,g11,[{suite,group_timetrap_3_SUITE}]},ok}}],
+ {?eh,tc_done,{ct_framework,
+ {init_per_group,g11,[{suite,group_timetrap_3_SUITE}]},
+ {auto_skipped,{group0_failed,bad_return_value}}}},
+ {?eh,tc_auto_skip,{group_timetrap_3_SUITE,t111,{group0_failed,bad_return_value}}},
+ {?eh,test_stats,{0,13,{0,1}}},
+ {?eh,tc_auto_skip,{ct_framework,{end_per_group,g11},
+ {group0_failed,bad_return_value}}}],
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}
@@ -539,11 +549,13 @@ test_events(require) ->
[{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g4,[]}}},
{?eh,tc_done,{group_require_1_SUITE,{init_per_group,g4,[]},
- {skipped,{require_failed,{name_in_use,common2_alias,common2}}}}},
+ {auto_skipped,{require_failed,
+ {name_in_use,common2_alias,common2}}}}},
{?eh,tc_auto_skip,{group_require_1_SUITE,t41,
- {require_failed,{name_in_use,common2_alias,common2}}}},
+ {require_failed,
+ {name_in_use,common2_alias,common2}}}},
{?eh,test_stats,{4,0,{0,1}}},
- {?eh,tc_auto_skip,{group_require_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{group_require_1_SUITE,{end_per_group,g4},
{require_failed,{name_in_use,common2_alias,common2}}}}],
[{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g5,[]}}},
@@ -566,16 +578,18 @@ test_events(require) ->
[{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g8,[]}}},
{?eh,tc_done,{group_require_1_SUITE,
{init_per_group,g8,[]},
- {skipped,{require_failed,{not_available,non_existing}}}}},
+ {auto_skipped,{require_failed,
+ {not_available,non_existing}}}}},
{?eh,tc_auto_skip,{group_require_1_SUITE,t81,
{require_failed,{not_available,non_existing}}}},
{?eh,test_stats,{8,0,{0,2}}},
- {?eh,tc_auto_skip,{group_require_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{group_require_1_SUITE,{end_per_group,g8},
{require_failed,{not_available,non_existing}}}}],
[{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g9,[]}}},
{?eh,tc_done,{group_require_1_SUITE,{init_per_group,g9,[]},ok}},
{?eh,tc_done,{group_require_1_SUITE,t91,
- {skipped,{require_failed,{not_available,non_existing}}}}},
+ {auto_skipped,{require_failed,
+ {not_available,non_existing}}}}},
{?eh,test_stats,{8,0,{0,3}}},
{?eh,tc_start,{group_require_1_SUITE,{end_per_group,g9,[]}}},
{?eh,tc_done,{group_require_1_SUITE,{end_per_group,g9,[]},ok}}],
@@ -587,12 +601,16 @@ test_events(require) ->
{?eh,tc_done,{group_require_1_SUITE,{end_per_group,g10,[]},ok}}],
[{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g11,[]}}},
- {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g11,[]},ok}},
- {?eh,tc_done,{group_require_1_SUITE,t111,ok}},
- {?eh,test_stats,{10,0,{0,3}}},
- {?eh,tc_start,{group_require_1_SUITE,{end_per_group,g11,[]}}},
- {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g11,[]},ok}}],
-
+ {?eh,tc_done,{group_require_1_SUITE,
+ {init_per_group,g11,[]},
+ {auto_skipped,{group0_failed,bad_return_value}}}},
+ {?eh,tc_auto_skip,{group_require_1_SUITE,t111,
+ {group0_failed,bad_return_value}}},
+ {?eh,test_stats,{9,0,{0,4}}},
+ {?eh,tc_auto_skip,{group_require_1_SUITE,
+ {end_per_group,g11},
+ {group0_failed,bad_return_value}}}],
+
{?eh,tc_done,{group_require_1_SUITE,end_per_suite,ok}},
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}
@@ -627,11 +645,11 @@ test_events(require_default) ->
[{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g4,[]}}},
{?eh,tc_done,{group_require_1_SUITE,
{init_per_group,g4,[]},
- {skipped,{require_failed,{not_available,common3}}}}},
+ {auto_skipped,{require_failed,{not_available,common3}}}}},
{?eh,tc_auto_skip,{group_require_1_SUITE,t41,
{require_failed,{not_available,common3}}}},
{?eh,test_stats,{4,0,{0,1}}},
- {?eh,tc_auto_skip,{group_require_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{group_require_1_SUITE,{end_per_group,g4},
{require_failed,{not_available,common3}}}}],
[{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g5,[]}}},
@@ -654,17 +672,19 @@ test_events(require_default) ->
[{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g8,[]}}},
{?eh,tc_done,{group_require_1_SUITE,
{init_per_group,g8,[]},
- {skipped,{require_failed,{not_available,non_existing}}}}},
+ {auto_skipped,{require_failed,
+ {not_available,non_existing}}}}},
{?eh,tc_auto_skip,{group_require_1_SUITE,t81,
{require_failed,{not_available,non_existing}}}},
{?eh,test_stats,{8,0,{0,2}}},
- {?eh,tc_auto_skip,{group_require_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{group_require_1_SUITE,{end_per_group,g8},
{require_failed,{not_available,non_existing}}}}],
[{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g9,[]}}},
{?eh,tc_done,{group_require_1_SUITE,{init_per_group,g9,[]},ok}},
{?eh,tc_done,{group_require_1_SUITE,t91,
- {skipped,{require_failed,{not_available,non_existing}}}}},
+ {auto_skipped,{require_failed,
+ {not_available,non_existing}}}}},
{?eh,test_stats,{8,0,{0,3}}},
{?eh,tc_start,{group_require_1_SUITE,{end_per_group,g9,[]}}},
{?eh,tc_done,{group_require_1_SUITE,{end_per_group,g9,[]},ok}}],
@@ -676,11 +696,15 @@ test_events(require_default) ->
{?eh,tc_done,{group_require_1_SUITE,{end_per_group,g10,[]},ok}}],
[{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g11,[]}}},
- {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g11,[]},ok}},
- {?eh,tc_done,{group_require_1_SUITE,t111,ok}},
- {?eh,test_stats,{10,0,{0,3}}},
- {?eh,tc_start,{group_require_1_SUITE,{end_per_group,g11,[]}}},
- {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g11,[]},ok}}],
+ {?eh,tc_done,{group_require_1_SUITE,
+ {init_per_group,g11,[]},
+ {auto_skipped,{group0_failed,bad_return_value}}}},
+ {?eh,tc_auto_skip,{group_require_1_SUITE,t111,
+ {group0_failed,bad_return_value}}},
+ {?eh,test_stats,{9,0,{0,4}}},
+ {?eh,tc_auto_skip,{group_require_1_SUITE,
+ {end_per_group,g11},
+ {group0_failed,bad_return_value}}}],
{?eh,tc_done,{group_require_1_SUITE,end_per_suite,ok}},
{?eh,test_done,{'DEF','STOP_TIME'}},
@@ -714,11 +738,12 @@ test_events(require_no_ips) ->
[{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g4,[]}}},
{?eh,tc_done,{group_require_2_SUITE,{init_per_group,g4,[]},
- {skipped,{require_failed,{name_in_use,common2_alias,common2}}}}},
+ {auto_skipped,{require_failed,
+ {name_in_use,common2_alias,common2}}}}},
{?eh,tc_auto_skip,{group_require_2_SUITE,t41,
{require_failed,{name_in_use,common2_alias,common2}}}},
{?eh,test_stats,{4,0,{0,1}}},
- {?eh,tc_auto_skip,{group_require_2_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{group_require_2_SUITE,{end_per_group,g4},
{require_failed,{name_in_use,common2_alias,common2}}}}],
[{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g5,[]}}},
@@ -741,16 +766,18 @@ test_events(require_no_ips) ->
[{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g8,[]}}},
{?eh,tc_done,{group_require_2_SUITE,
{init_per_group,g8,[]},
- {skipped,{require_failed,{not_available,non_existing}}}}},
+ {auto_skipped,{require_failed,
+ {not_available,non_existing}}}}},
{?eh,tc_auto_skip,{group_require_2_SUITE,t81,
{require_failed,{not_available,non_existing}}}},
{?eh,test_stats,{8,0,{0,2}}},
- {?eh,tc_auto_skip,{group_require_2_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{group_require_2_SUITE,{end_per_group,g8},
{require_failed,{not_available,non_existing}}}}],
[{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g9,[]}}},
{?eh,tc_done,{group_require_2_SUITE,{init_per_group,g9,[]},ok}},
{?eh,tc_done,{group_require_2_SUITE,t91,
- {skipped,{require_failed,{not_available,non_existing}}}}},
+ {auto_skipped,{require_failed,
+ {not_available,non_existing}}}}},
{?eh,test_stats,{8,0,{0,3}}},
{?eh,tc_start,{group_require_2_SUITE,{end_per_group,g9,[]}}},
{?eh,tc_done,{group_require_2_SUITE,{end_per_group,g9,[]},ok}}],
@@ -760,13 +787,17 @@ test_events(require_no_ips) ->
{?eh,tc_done,{group_require_2_SUITE,t101,ok}},
{?eh,tc_start,{group_require_2_SUITE,{end_per_group,g10,[]}}},
{?eh,tc_done,{group_require_2_SUITE,{end_per_group,g10,[]},ok}}],
-
+
[{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g11,[]}}},
- {?eh,tc_done,{group_require_2_SUITE,{init_per_group,g11,[]},ok}},
- {?eh,tc_done,{group_require_2_SUITE,t111,ok}},
- {?eh,test_stats,{10,0,{0,3}}},
- {?eh,tc_start,{group_require_2_SUITE,{end_per_group,g11,[]}}},
- {?eh,tc_done,{group_require_2_SUITE,{end_per_group,g11,[]},ok}}],
+ {?eh,tc_done,{group_require_2_SUITE,
+ {init_per_group,g11,[]},
+ {auto_skipped,{group0_failed,bad_return_value}}}},
+ {?eh,tc_auto_skip,{group_require_2_SUITE,t111,
+ {group0_failed,bad_return_value}}},
+ {?eh,test_stats,{9,0,{0,4}}},
+ {?eh,tc_auto_skip,{group_require_2_SUITE,
+ {end_per_group,g11},
+ {group0_failed,bad_return_value}}}],
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}
@@ -799,11 +830,11 @@ test_events(require_no_ipg) ->
[{?eh,tc_start,{ct_framework,{init_per_group,g4,[{suite,group_require_3_SUITE}]}}},
{?eh,tc_done,{ct_framework,{init_per_group,g4,[{suite,group_require_3_SUITE}]},
- {skipped,{require_failed,{name_in_use,common2_alias,common2}}}}},
+ {auto_skipped,{require_failed,{name_in_use,common2_alias,common2}}}}},
{?eh,tc_auto_skip,{group_require_3_SUITE,t41,
{require_failed,{name_in_use,common2_alias,common2}}}},
{?eh,test_stats,{4,0,{0,1}}},
- {?eh,tc_auto_skip,{ct_framework,end_per_group,
+ {?eh,tc_auto_skip,{ct_framework,{end_per_group,g4},
{require_failed,{name_in_use,common2_alias,common2}}}}],
[{?eh,tc_start,{ct_framework,{init_per_group,g5,[{suite,group_require_3_SUITE}]}}},
@@ -825,16 +856,16 @@ test_events(require_no_ipg) ->
[{?eh,tc_start,{ct_framework,{init_per_group,g8,[{suite,group_require_3_SUITE}]}}},
{?eh,tc_done,{ct_framework,{init_per_group,g8,[{suite,group_require_3_SUITE}]},
- {skipped,{require_failed,{not_available,non_existing}}}}},
+ {auto_skipped,{require_failed,{not_available,non_existing}}}}},
{?eh,tc_auto_skip,{group_require_3_SUITE,t81,
{require_failed,{not_available,non_existing}}}},
{?eh,test_stats,{8,0,{0,2}}},
- {?eh,tc_auto_skip,{ct_framework,end_per_group,
+ {?eh,tc_auto_skip,{ct_framework,{end_per_group,g8},
{require_failed,{not_available,non_existing}}}}],
[{?eh,tc_start,{ct_framework,{init_per_group,g9,[{suite,group_require_3_SUITE}]}}},
{?eh,tc_done,{ct_framework,{init_per_group,g9,[{suite,group_require_3_SUITE}]},ok}},
{?eh,tc_done,{group_require_3_SUITE,t91,
- {skipped,{require_failed,{not_available,non_existing}}}}},
+ {auto_skipped,{require_failed,{not_available,non_existing}}}}},
{?eh,test_stats,{8,0,{0,3}}},
{?eh,tc_start,{ct_framework,{end_per_group,g9,[{suite,group_require_3_SUITE}]}}},
{?eh,tc_done,{ct_framework,{end_per_group,g9,[{suite,group_require_3_SUITE}]},ok}}],
@@ -844,13 +875,14 @@ test_events(require_no_ipg) ->
{?eh,tc_done,{group_require_3_SUITE,t101,ok}},
{?eh,tc_start,{ct_framework,{end_per_group,g10,[{suite,group_require_3_SUITE}]}}},
{?eh,tc_done,{ct_framework,{end_per_group,g10,[{suite,group_require_3_SUITE}]},ok}}],
-
+
[{?eh,tc_start,{ct_framework,{init_per_group,g11,[{suite,group_require_3_SUITE}]}}},
- {?eh,tc_done,{ct_framework,{init_per_group,g11,[{suite,group_require_3_SUITE}]},ok}},
- {?eh,tc_done,{group_require_3_SUITE,t111,ok}},
- {?eh,test_stats,{10,0,{0,3}}},
- {?eh,tc_start,{ct_framework,{end_per_group,g11,[{suite,group_require_3_SUITE}]}}},
- {?eh,tc_done,{ct_framework,{end_per_group,g11,[{suite,group_require_3_SUITE}]},ok}}],
+ {?eh,tc_done,{ct_framework,{init_per_group,g11,[{suite,group_require_3_SUITE}]},
+ {auto_skipped,{group0_failed,bad_return_value}}}},
+ {?eh,tc_auto_skip,{group_require_3_SUITE,t111,{group0_failed,bad_return_value}}},
+ {?eh,test_stats,{9,0,{0,4}}},
+ {?eh,tc_auto_skip,{ct_framework,{end_per_group,g11},
+ {group0_failed,bad_return_value}}}],
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}
diff --git a/lib/common_test/test/ct_groups_test_2_SUITE.erl b/lib/common_test/test/ct_groups_test_2_SUITE.erl
index 1b2ad12e2f..8b0de98709 100644
--- a/lib/common_test/test/ct_groups_test_2_SUITE.erl
+++ b/lib/common_test/test/ct_groups_test_2_SUITE.erl
@@ -171,16 +171,20 @@ test_events(missing_conf) ->
{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
{?eh,start_info,{1,1,2}},
- {?eh,tc_start,{ct_framework,{init_per_group,group1,[]}}},
- {?eh,tc_done,{ct_framework,{init_per_group,group1,[]},ok}},
+ {?eh,tc_start,{ct_framework,{init_per_group,group1,
+ [{suite,missing_conf_SUITE}]}}},
+ {?eh,tc_done,{ct_framework,{init_per_group,group1,
+ [{suite,missing_conf_SUITE}]},ok}},
{?eh,tc_start,{missing_conf_SUITE,tc1}},
{?eh,tc_done,{missing_conf_SUITE,tc1,ok}},
{?eh,test_stats,{1,0,{0,0}}},
{?eh,tc_start,{missing_conf_SUITE,tc2}},
{?eh,tc_done,{missing_conf_SUITE,tc2,ok}},
{?eh,test_stats,{2,0,{0,0}}},
- {?eh,tc_start,{ct_framework,{end_per_group,group1,[]}}},
- {?eh,tc_done,{ct_framework,{end_per_group,group1,[]},ok}},
+ {?eh,tc_start,{ct_framework,{end_per_group,group1,
+ [{suite,missing_conf_SUITE}]}}},
+ {?eh,tc_done,{ct_framework,{end_per_group,group1,
+ [{suite,missing_conf_SUITE}]},ok}},
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}
];
diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl
index 596bfe3ff0..083c87b49e 100644
--- a/lib/common_test/test/ct_hooks_SUITE.erl
+++ b/lib/common_test/test/ct_hooks_SUITE.erl
@@ -749,11 +749,10 @@ test_events(skip_pre_suite_cth) ->
{?eh,cth,{'_',on_tc_skip,
[init_per_suite,{tc_user_skip,{skipped,"Test skip"}},[]]}},
- {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}},
- {?eh,cth,{'_',on_tc_skip,[test_case,{tc_auto_skip,"Test skip"},[]]}},
+ {?eh,tc_user_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}},
+ {?eh,cth,{'_',on_tc_skip,[test_case,{tc_user_skip,"Test skip"},[]]}},
- {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}},
- {?eh,cth,{'_',on_tc_skip,[end_per_suite,{tc_auto_skip,"Test skip"},[]]}},
+ {?eh,tc_user_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}},
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,cth, {'_',terminate,[[]]}},
@@ -773,11 +772,10 @@ test_events(skip_post_suite_cth) ->
{?eh,cth,{'_',on_tc_skip,
[init_per_suite,{tc_user_skip,{skipped,"Test skip"}},[]]}},
- {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}},
- {?eh,cth,{'_',on_tc_skip,[test_case,{tc_auto_skip,"Test skip"},[]]}},
+ {?eh,tc_user_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}},
+ {?eh,cth,{'_',on_tc_skip,[test_case,{tc_user_skip,"Test skip"},[]]}},
- {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}},
- {?eh,cth,{'_',on_tc_skip,[end_per_suite,{tc_auto_skip,"Test skip"},[]]}},
+ {?eh,tc_user_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}},
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,cth,{'_',terminate,[[]]}},
@@ -1219,11 +1217,15 @@ test_events(cth_log) ->
{?eh,tc_start,{cth_log_SUITE,init_per_suite}},
{parallel,
- [{?eh,tc_start,{ct_framework,{init_per_group,g1,[parallel]}}},
- {?eh,tc_done,{ct_framework,{init_per_group,g1,[parallel]},ok}},
+ [{?eh,tc_start,{ct_framework,{init_per_group,g1,
+ [{suite,cth_log_SUITE},parallel]}}},
+ {?eh,tc_done,{ct_framework,{init_per_group,g1,
+ [{suite,cth_log_SUITE},parallel]},ok}},
{?eh,test_stats,{30,0,{0,0}}},
- {?eh,tc_start,{ct_framework,{end_per_group,g1,[parallel]}}},
- {?eh,tc_done,{ct_framework,{end_per_group,g1,[parallel]},ok}}]},
+ {?eh,tc_start,{ct_framework,{end_per_group,g1,
+ [{suite,cth_log_SUITE},parallel]}}},
+ {?eh,tc_done,{ct_framework,{end_per_group,g1,
+ [{suite,cth_log_SUITE},parallel]},ok}}]},
{?eh,tc_done,{cth_log_SUITE,end_per_suite,ok}},
{?eh,test_done,{'DEF','STOP_TIME'}},
diff --git a/lib/common_test/test/ct_repeat_1_SUITE.erl b/lib/common_test/test/ct_repeat_1_SUITE.erl
index 090002d0c2..98eaa28763 100644
--- a/lib/common_test/test/ct_repeat_1_SUITE.erl
+++ b/lib/common_test/test/ct_repeat_1_SUITE.erl
@@ -229,7 +229,7 @@ test_events(repeat_cs_and_grs) ->
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',fails_on_purpose}}}}},
{?eh,test_stats,{3,1,{0,1}}},
- {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init},
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',fails_on_purpose}}}}}],
{?eh,test_stats,{4,1,{0,1}}},
@@ -251,7 +251,7 @@ test_events(repeat_cs_and_grs) ->
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',fails_on_purpose}}}}},
{?eh,test_stats,{7,2,{0,2}}},
- {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init},
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',fails_on_purpose}}}}}],
{?eh,test_stats,{8,2,{0,2}}},
@@ -311,7 +311,7 @@ test_events(repeat_seq) ->
{init_per_group,gr_fail_init,[]},
{failed,{error,fails_on_purpose}}}},
{?eh,test_stats,{7,2,{0,5}}},
- {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init},
{failed,
{repeat_1_SUITE,init_per_group,
{'EXIT',fails_on_purpose}}}}}],
@@ -410,7 +410,7 @@ test_events(repeat_gr_until_any_ok) ->
{init_per_group,gr_fail_init,[]},
{failed,{error,fails_on_purpose}}}},
{?eh,test_stats,{1,1,{0,1}}},
- {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init},
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',fails_on_purpose}}}}}],
{?eh,test_stats,{1,2,{0,1}}},
@@ -452,7 +452,7 @@ test_events(repeat_gr_until_any_ok) ->
{init_per_group,gr_fail_init,[]},
{failed,{error,fails_on_purpose}}}},
{?eh,test_stats,{5,6,{0,3}}},
- {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init},
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',fails_on_purpose}}}}}],
{?eh,tc_done,{repeat_1_SUITE,
@@ -469,7 +469,7 @@ test_events(repeat_gr_until_any_ok) ->
{init_per_group,gr_fail_init,[]},
{failed,{error,fails_on_purpose}}}},
{?eh,test_stats,{7,7,{0,4}}},
- {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init},
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',fails_on_purpose}}}}}],
{?eh,tc_done,{repeat_1_SUITE,
@@ -770,7 +770,7 @@ test_events(repeat_gr_until_any_fail) ->
{'EXIT',failing_this_time}}}}},
{?eh,test_stats,{14,0,{0,1}}},
{?eh,tc_auto_skip,
- {repeat_1_SUITE,end_per_group,
+ {repeat_1_SUITE,{end_per_group,gr_ok_then_fail_init},
{failed,
{repeat_1_SUITE,init_per_group,
{'EXIT',failing_this_time}}}}}],
@@ -967,7 +967,7 @@ test_events(repeat_gr_until_all_ok) ->
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',failing_this_time}}}}},
{?eh,test_stats,{7,1,{0,1}}},
- {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init_then_ok},
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',failing_this_time}}}}}],
{?eh,tc_done,{repeat_1_SUITE,tc_ok_1,ok}},
@@ -1083,7 +1083,7 @@ test_events(repeat_gr_until_all_fail) ->
{init_per_group,gr_fail_init,[]},
{failed,{error,fails_on_purpose}}}},
{?eh,test_stats,{0,1,{0,1}}},
- {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init},
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',fails_on_purpose}}}}}],
{?eh,tc_done,{repeat_1_SUITE,tc_ok_then_fail_1,ok}},
@@ -1102,7 +1102,7 @@ test_events(repeat_gr_until_all_fail) ->
{init_per_group,gr_fail_init,[]},
{failed,{error,fails_on_purpose}}}},
{?eh,test_stats,{2,2,{0,2}}},
- {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init},
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',fails_on_purpose}}}}}],
{?eh,tc_done,{repeat_1_SUITE,tc_ok_then_fail_1,
@@ -1134,7 +1134,7 @@ test_events(repeat_gr_until_all_fail) ->
{init_per_group,gr_ok_then_fail_init,[]},
{failed,{error,failing_this_time}}}},
{?eh,test_stats,{4,4,{0,3}}},
- {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_ok_then_fail_init},
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',failing_this_time}}}}}],
{?eh,tc_start,{repeat_1_SUITE,tc_fail_1}},
@@ -1293,7 +1293,7 @@ test_events(repeat_seq_until_any_fail) ->
{init_per_group,gr_ok_then_fail_init,[]},
{failed,{error,failing_this_time}}}},
{?eh,test_stats,{24,1,{0,5}}},
- {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_ok_then_fail_init},
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',failing_this_time}}}}}],
{?eh,tc_auto_skip,{repeat_1_SUITE,tc_ok_1,
@@ -1516,7 +1516,7 @@ test_events(repeat_shuffled_seq_until_any_fail) ->
{init_per_group,repeat_shuffled_seq_until_any_fail_5,
[{shuffle,'_'},{repeat_until_any_fail,2},
sequence]},ok}},
- [{?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ [{?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_ok_then_fail_init},
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',failing_this_time}}}}}],
{?eh,tc_start,{repeat_1_SUITE,
diff --git a/lib/common_test/test/ct_repeat_testrun_SUITE.erl b/lib/common_test/test/ct_repeat_testrun_SUITE.erl
index 35d67a10f2..bb2aba2c5a 100644
--- a/lib/common_test/test/ct_repeat_testrun_SUITE.erl
+++ b/lib/common_test/test/ct_repeat_testrun_SUITE.erl
@@ -36,7 +36,7 @@
-define(eh, ct_test_support_eh).
-define(skip_reason, "Repeated test stopped by force_stop option").
--define(skipped, {skipped, ?skip_reason}).
+-define(skipped, {auto_skipped, ?skip_reason}).
%% Timers used in this test.
@@ -78,7 +78,7 @@ init_per_suite(Config0) ->
{1,0,{0,0}} = ct_test_support:run(ct,run_test,[Opts2],Config),
%% Time the shortest testcase to use for offset
- {T0,{1,0,{0,0}}} = timer:tc(ct_test_support,run,[ct,run_test,[Opts1],Config]),
+ {_T0,{1,0,{0,0}}} = timer:tc(ct_test_support,run,[ct,run_test,[Opts1],Config]),
%% -2 is to ensure we hit inside the target test case and not after
% T = round(T0/1000000)-2,
@@ -222,7 +222,7 @@ until_force_stop_skip_rest_group(Config) when is_list(Config) ->
fun() ->
[_] = ct_test_support:run_ct_run_test(
Opts++[{until,until_str(?t3,1,Config)}],Config),
- 0 = ct_test_support:run_ct_script_start(
+ 1 = ct_test_support:run_ct_script_start(
Opts++[{until,until_str(?t3,1,Config)}],Config)
end,
ok = execute(ExecuteFun,
@@ -341,19 +341,18 @@ skip_first_tc1(Suite) ->
{?eh,tc_done,{Suite,tc1,ok}},
{?eh,test_stats,{'_',0,{0,0}}},
{?eh,tc_done,{Suite,tc2,?skipped}},
- {?eh,test_stats,{'_',0,{1,0}}},
+ {?eh,test_stats,{'_',0,{0,1}}},
{?eh,tc_done,{Suite,{init_per_group,g,[]},?skipped}},
{?eh,tc_auto_skip,{Suite,tc1,?skip_reason}},
- {?eh,test_stats,{'_',0,{1,1}}},
+ {?eh,test_stats,{'_',0,{0,2}}},
{?eh,tc_auto_skip,{Suite,tc2,?skip_reason}},
- {?eh,test_stats,{'_',0,{1,2}}},
- {?eh,tc_auto_skip,{Suite,end_per_group,?skip_reason}},
+ {?eh,test_stats,{'_',0,{0,3}}},
+ {?eh,tc_auto_skip,{Suite,{end_per_group,g},?skip_reason}},
{?eh,tc_done,{Suite,tc2,?skipped}},
- {?eh,test_stats,{'_',0,{2,2}}},
+ {?eh,test_stats,{'_',0,{0,4}}},
{?eh,tc_start,{Suite,end_per_suite}},
{?eh,tc_done,{Suite,end_per_suite,ok}}].
-
skip_tc1_in_group(Suite) ->
[{?eh,tc_start,{Suite,init_per_suite}},
{?eh,tc_done,{Suite,init_per_suite,ok}},
@@ -369,10 +368,10 @@ skip_tc1_in_group(Suite) ->
{?eh,tc_done,{Suite,tc1,ok}},
{?eh,test_stats,{'_',0,{0,0}}},
{?eh,tc_done,{Suite,tc2,?skipped}},
- {?eh,test_stats,{'_',0,{1,0}}},
+ {?eh,test_stats,{'_',0,{0,1}}},
{?eh,tc_start,{Suite,{end_per_group,g,[]}}},
{?eh,tc_done,{Suite,{end_per_group,g,[]},ok}}],
{?eh,tc_done,{Suite,tc2,?skipped}},
- {?eh,test_stats,{'_',0,{2,0}}},
+ {?eh,test_stats,{'_',0,{0,2}}},
{?eh,tc_start,{Suite,end_per_suite}},
{?eh,tc_done,{Suite,end_per_suite,ok}}].
diff --git a/lib/common_test/test/ct_sequence_1_SUITE.erl b/lib/common_test/test/ct_sequence_1_SUITE.erl
index 5facf90656..8c87236838 100644
--- a/lib/common_test/test/ct_sequence_1_SUITE.erl
+++ b/lib/common_test/test/ct_sequence_1_SUITE.erl
@@ -212,7 +212,7 @@ test_events(subgroup_init_fail) ->
{failed,{subgroups_1_SUITE,init_per_group,
{'EXIT',init_per_group_fails_on_purpose}}}}},
{?eh,test_stats,{0,0,{0,1}}},
- {?eh,tc_auto_skip,{subgroups_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{subgroups_1_SUITE,{end_per_group,fail_init},
{failed,{subgroups_1_SUITE,init_per_group,
{'EXIT',init_per_group_fails_on_purpose}}}}}],
{?eh,tc_auto_skip,{subgroups_1_SUITE,ok_tc,{group_result,fail_init,failed}}},
@@ -294,7 +294,7 @@ test_events(case_after_subgroup_fail_init) ->
{subgroups_1_SUITE,init_per_group,
{'EXIT',init_per_group_fails_on_purpose}}}}},
{?eh,test_stats,{0,0,{0,1}}},
- {?eh,tc_auto_skip,{subgroups_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{subgroups_1_SUITE,{end_per_group,fail_init},
{failed,
{subgroups_1_SUITE,init_per_group,
{'EXIT',init_per_group_fails_on_purpose}}}}}],
diff --git a/lib/common_test/test/ct_skip_SUITE.erl b/lib/common_test/test/ct_skip_SUITE.erl
index b8be55f43a..b0a6c839a2 100644
--- a/lib/common_test/test/ct_skip_SUITE.erl
+++ b/lib/common_test/test/ct_skip_SUITE.erl
@@ -59,7 +59,7 @@ end_per_testcase(TestCase, Config) ->
suite() -> [{ct_hooks,[ts_install_cth]}].
all() ->
- [auto_skip, user_skip].
+ [auto_skip, user_skip, testspec_skip].
groups() ->
[].
@@ -91,7 +91,8 @@ auto_skip(Config) when is_list(Config) ->
Join(DataDir, "auto_skip_8_SUITE"),
Join(DataDir, "auto_skip_9_SUITE"),
Join(DataDir, "auto_skip_10_SUITE"),
- Join(DataDir, "auto_skip_11_SUITE")
+ Join(DataDir, "auto_skip_11_SUITE"),
+ Join(DataDir, "auto_skip_12_SUITE")
],
{Opts,ERPid} = setup({suite,Suites}, Config),
@@ -116,7 +117,8 @@ user_skip(Config) when is_list(Config) ->
Join(DataDir, "user_skip_2_SUITE"),
Join(DataDir, "user_skip_3_SUITE"),
Join(DataDir, "user_skip_4_SUITE"),
- Join(DataDir, "user_skip_5_SUITE")],
+ Join(DataDir, "user_skip_5_SUITE"),
+ Join(DataDir, "user_skip_6_SUITE")],
{Opts,ERPid} = setup({suite,Suites}, Config),
ok = ct_test_support:run(Opts, Config),
@@ -131,8 +133,56 @@ user_skip(Config) when is_list(Config) ->
ok = ct_test_support:verify_events(TestEvents, Events, Config).
%%%-----------------------------------------------------------------
+%%%
+testspec_skip(Config) when is_list(Config) ->
+ TestDir = filename:join(?config(data_dir, Config),
+ filename:join("skip", "test")),
+ TestSpec1 = [{suites, TestDir, user_skip_7_SUITE},
+ {skip_cases, TestDir, user_skip_7_SUITE, [tc1,tc3], "SKIPPED"}],
+
+ TestSpec2 = [{suites, TestDir, user_skip_7_SUITE},
+ {skip_groups, TestDir, user_skip_7_SUITE, ptop1, "SKIPPED"}],
+
+ TestSpec3 = [{suites, TestDir, user_skip_7_SUITE},
+ {skip_groups, TestDir, user_skip_7_SUITE, psub1, "SKIPPED"}],
+
+ TestSpec4 = [{suites, TestDir, user_skip_7_SUITE},
+ {skip_suites, TestDir, user_skip_7_SUITE, "SKIPPED"}],
+
+ TestSpec5 = [{groups, TestDir, user_skip_6_SUITE, ptop1},
+ {skip_groups, TestDir, user_skip_6_SUITE, psub1, "SKIPPED"}],
+
+ {Opts,ERPid} = setup_testspec([{ts1,TestSpec1},
+ {ts2,TestSpec2},
+ {ts3,TestSpec3},
+ {ts4,TestSpec4},
+ {ts5,TestSpec5}], Config),
+
+ ok = ct_test_support:run(Opts, Config),
+
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(testspec_skip,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config),
+ Opts),
+
+ TestEvents = events_to_check(testspec_skip),
+ ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
+%%%-----------------------------------------------------------------
%%% HELP FUNCTIONS
%%%-----------------------------------------------------------------
+setup_testspec(TestSpecs, Config) ->
+ SpecFiles =
+ [begin SpecFile = filename:join(?config(priv_dir, Config),
+ atom_to_list(SpecName)++".spec"),
+ {ok,Dev} = file:open(SpecFile, [write]),
+ [io:format(Dev, "~p.~n", [Term]) || Term <- TestSpec],
+ file:close(Dev),
+ SpecFile
+ end || {SpecName,TestSpec} <- TestSpecs],
+ setup({spec,SpecFiles}, Config).
setup(Test, Config) ->
Opts0 = ct_test_support:get_opts(Config),
@@ -163,7 +213,7 @@ test_events(auto_skip) ->
[
{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
- {?eh,start_info,{11,11,34}},
+ {?eh,start_info,{12,12,43}},
{?eh,tc_start,{auto_skip_1_SUITE,init_per_suite}},
{?eh,tc_done,
@@ -196,8 +246,8 @@ test_events(auto_skip) ->
{?eh,tc_start,{auto_skip_3_SUITE,tc1}},
{?eh,tc_done,
{auto_skip_3_SUITE,tc1,
- {skipped,{failed,{auto_skip_3_SUITE,init_per_testcase,
- {{init_per_testcase,tc1,failed},'_'}}}}}},
+ {auto_skipped,{failed,{auto_skip_3_SUITE,init_per_testcase,
+ {'init_per_testcase for tc1 failed','_'}}}}}},
{?eh,test_stats,{0,0,{0,4}}},
{?eh,tc_start,{auto_skip_3_SUITE,tc2}},
{?eh,tc_done,{auto_skip_3_SUITE,tc2,ok}},
@@ -209,7 +259,7 @@ test_events(auto_skip) ->
{?eh,tc_done,{auto_skip_4_SUITE,init_per_suite,ok}},
{?eh,tc_start,{auto_skip_4_SUITE,tc1}},
{?eh,tc_done,{auto_skip_4_SUITE,tc1,
- {skipped,{failed,{auto_skip_4_SUITE,init_per_testcase,
+ {auto_skipped,{failed,{auto_skip_4_SUITE,init_per_testcase,
{timetrap_timeout,1000}}}}}},
{?eh,test_stats,{1,0,{0,5}}},
{?eh,tc_start,{auto_skip_4_SUITE,tc2}},
@@ -232,8 +282,9 @@ test_events(auto_skip) ->
{'EXIT',{group,g1,failed}}}}}},
{?eh,test_stats,{2,0,{0,7}}},
{?eh,tc_auto_skip,
- {auto_skip_5_SUITE,end_per_group,{failed,{auto_skip_5_SUITE,init_per_group,
- {'EXIT',{group,g1,failed}}}}}}],
+ {auto_skip_5_SUITE,{end_per_group,g1},
+ {failed,{auto_skip_5_SUITE,init_per_group,
+ {'EXIT',{group,g1,failed}}}}}}],
{?eh,tc_start,{auto_skip_5_SUITE,end_per_suite}},
{?eh,tc_done,{auto_skip_5_SUITE,end_per_suite,ok}},
@@ -260,8 +311,9 @@ test_events(auto_skip) ->
{'EXIT',{group,g1,failed}}}}}},
{?eh,test_stats,{2,0,{0,11}}},
{?eh,tc_auto_skip,
- {auto_skip_6_SUITE,end_per_group,{failed,{auto_skip_6_SUITE,init_per_group,
- {'EXIT',{group,g1,failed}}}}}}],
+ {auto_skip_6_SUITE,{end_per_group,g1},
+ {failed,{auto_skip_6_SUITE,init_per_group,
+ {'EXIT',{group,g1,failed}}}}}}],
[{?eh,tc_start,{auto_skip_6_SUITE,{init_per_group,g3,[]}}},
{?eh,tc_done,{auto_skip_6_SUITE,{init_per_group,g3,[]},ok}},
@@ -280,8 +332,9 @@ test_events(auto_skip) ->
{'EXIT',{group,g4,failed}}}}}},
{?eh,test_stats,{3,0,{0,13}}},
{?eh,tc_auto_skip,
- {auto_skip_6_SUITE,end_per_group,{failed,{auto_skip_6_SUITE,init_per_group,
- {'EXIT',{group,g4,failed}}}}}}],
+ {auto_skip_6_SUITE,{end_per_group,g4},
+ {failed,{auto_skip_6_SUITE,init_per_group,
+ {'EXIT',{group,g4,failed}}}}}}],
{?eh,tc_start,{auto_skip_6_SUITE,tc2}},
{?eh,tc_done,{auto_skip_6_SUITE,tc2,ok}},
{?eh,test_stats,{4,0,{0,13}}},
@@ -320,7 +373,7 @@ test_events(auto_skip) ->
{?eh,test_stats,{5,0,{0,17}}},
{?eh,tc_start,{auto_skip_9_SUITE,tc2}},
{?eh,tc_done,{auto_skip_9_SUITE,tc2,
- {skipped,{failed,{auto_skip_9_SUITE,init_per_testcase,bad_return}}}}},
+ {auto_skipped,{failed,{auto_skip_9_SUITE,init_per_testcase,bad_return}}}}},
{?eh,test_stats,{5,0,{0,18}}},
[{?eh,tc_start,{auto_skip_9_SUITE,{init_per_group,g1,[]}}},
@@ -343,7 +396,7 @@ test_events(auto_skip) ->
{?eh,tc_done,{auto_skip_9_SUITE,{init_per_group,g3,[]},ok}},
{?eh,tc_start,{auto_skip_9_SUITE,tc5}},
{?eh,tc_done,{auto_skip_9_SUITE,tc5,
- {skipped,{failed,{auto_skip_9_SUITE,init_per_testcase,bad_return}}}}},
+ {auto_skipped,{failed,{auto_skip_9_SUITE,init_per_testcase,bad_return}}}}},
{?eh,test_stats,{7,0,{0,19}}},
{?eh,tc_start,{auto_skip_9_SUITE,{end_per_group,g3,[]}}},
{?eh,tc_done,{auto_skip_9_SUITE,{end_per_group,g3,[]},ok}}],
@@ -363,7 +416,7 @@ test_events(auto_skip) ->
{?eh,tc_start,{auto_skip_9_SUITE,tc8}},
{?eh,tc_done,
{auto_skip_9_SUITE,tc8,
- {skipped,{failed,{auto_skip_9_SUITE,init_per_testcase,
+ {auto_skipped,{failed,{auto_skip_9_SUITE,init_per_testcase,
{{badmatch,undefined},'_'}}}}}},
{?eh,tc_start,
{auto_skip_9_SUITE,{end_per_group,g5,[parallel]}}},
@@ -383,26 +436,26 @@ test_events(auto_skip) ->
{?eh,tc_start,{auto_skip_10_SUITE,init_per_suite}},
{?eh,tc_done,{auto_skip_10_SUITE,init_per_suite,
- {skipped,
- {require_failed_in_suite0,
- {not_available,undefined_config_variable}}}}},
- {?eh,tc_auto_skip,
- {auto_skip_10_SUITE,tc1,
- {require_failed_in_suite0,{not_available,undefined_config_variable}}}},
+ {auto_skipped,{require_failed_in_suite0,
+ {not_available,undefined_config_variable}}}}},
+ {?eh,tc_auto_skip,{auto_skip_10_SUITE,tc1,
+ {require_failed_in_suite0,
+ {not_available,undefined_config_variable}}}},
{?eh,test_stats,{9,0,{0,21}}},
- {?eh,tc_auto_skip,
- {auto_skip_10_SUITE,tc2,
- {require_failed_in_suite0,{not_available,undefined_config_variable}}}},
+ {?eh,tc_auto_skip,{auto_skip_10_SUITE,tc2,
+ {require_failed_in_suite0,
+ {not_available,undefined_config_variable}}}},
{?eh,test_stats,{9,0,{0,22}}},
- {?eh,tc_auto_skip,
- {auto_skip_10_SUITE,end_per_suite,
- {require_failed_in_suite0,{not_available,undefined_config_variable}}}},
+ {?eh,tc_auto_skip,{auto_skip_10_SUITE,end_per_suite,
+ {require_failed_in_suite0,
+ {not_available,undefined_config_variable}}}},
{?eh,tc_start,{auto_skip_11_SUITE,init_per_suite}},
{?eh,tc_done,{auto_skip_11_SUITE,init_per_suite,ok}},
{?eh,tc_start,{auto_skip_11_SUITE,tc1}},
{?eh,tc_done,{auto_skip_11_SUITE,tc1,
- {skipped,{require_failed,{not_available,undefined_config_variable}}}}},
+ {auto_skipped,{require_failed,
+ {not_available,undefined_config_variable}}}}},
{?eh,test_stats,{9,0,{0,23}}},
{?eh,tc_start,{auto_skip_11_SUITE,tc2}},
{?eh,tc_done,{auto_skip_11_SUITE,tc2,ok}},
@@ -421,7 +474,8 @@ test_events(auto_skip) ->
{?eh,tc_start,{auto_skip_11_SUITE,tc3}},
{?eh,tc_done,
{auto_skip_11_SUITE,tc3,
- {skipped,{require_failed,{not_available,undefined_config_variable}}}}},
+ {auto_skipped,{require_failed,
+ {not_available,undefined_config_variable}}}}},
{?eh,test_stats,{10,0,{0,24}}},
{?eh,tc_start,
{auto_skip_11_SUITE,{end_per_group,g2,[parallel]}}},
@@ -434,6 +488,75 @@ test_events(auto_skip) ->
{?eh,tc_start,{auto_skip_11_SUITE,end_per_suite}},
{?eh,tc_done,{auto_skip_11_SUITE,end_per_suite,ok}},
+
+ {?eh,tc_start,{auto_skip_12_SUITE,init_per_suite}},
+ {?eh,tc_done,{auto_skip_12_SUITE,init_per_suite,ok}},
+
+ [{?eh,tc_start,{ct_framework,{init_per_group,g1,
+ [{suite,auto_skip_12_SUITE}]}}},
+ {?eh,tc_done,{ct_framework,{init_per_group,g1,
+ [{suite,auto_skip_12_SUITE}]},
+ {auto_skipped,
+ {require_failed,{not_available,unknown_variable_g1}}}}},
+ {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc1,
+ {require_failed,{not_available,unknown_variable_g1}}}},
+ {?eh,test_stats,{10,0,{0,25}}},
+ {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc2,
+ {require_failed,{not_available,unknown_variable_g1}}}},
+ {?eh,test_stats,{10,0,{0,26}}},
+ {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3,
+ {require_failed,{not_available,unknown_variable_g1}}}},
+ {?eh,test_stats,{10,0,{0,27}}},
+ {?eh,tc_auto_skip,{ct_framework,{end_per_group,g1},
+ {require_failed,{not_available,unknown_variable_g1}}}}],
+
+ [{?eh,tc_start,{ct_framework,{init_per_group,g1,
+ [{suite,auto_skip_12_SUITE}]}}},
+ {?eh,tc_done,{ct_framework,{init_per_group,g1,
+ [{suite,auto_skip_12_SUITE}]},
+ {auto_skipped,
+ {require_failed,{not_available,unknown_variable_g1}}}}},
+ {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc1,
+ {require_failed,{not_available,unknown_variable_g1}}}},
+ {?eh,test_stats,{10,0,{0,28}}},
+ {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc2,
+ {require_failed,{not_available,unknown_variable_g1}}}},
+ {?eh,test_stats,{10,0,{0,29}}},
+ {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3,
+ {require_failed,{not_available,unknown_variable_g1}}}},
+ {?eh,test_stats,{10,0,{0,30}}},
+ {?eh,tc_auto_skip,{ct_framework,{end_per_group,g1},
+ {require_failed,{not_available,unknown_variable_g1}}}}],
+
+ [{?eh,tc_start,{ct_framework,{init_per_group,g3,
+ [{suite,auto_skip_12_SUITE}]}}},
+ {?eh,tc_done,{ct_framework,{init_per_group,g3,
+ [{suite,auto_skip_12_SUITE}]},ok}},
+ {?eh,tc_start,{auto_skip_12_SUITE,tc1}},
+ {?eh,tc_done,{auto_skip_12_SUITE,tc1,ok}},
+ {?eh,test_stats,{11,0,{0,30}}},
+ {?eh,tc_start,{auto_skip_12_SUITE,tc2}},
+ {?eh,tc_done,{auto_skip_12_SUITE,tc2,ok}},
+ {?eh,test_stats,{12,0,{0,30}}},
+ [{?eh,tc_start,{ct_framework,{init_per_group,g4,
+ [{suite,auto_skip_12_SUITE}]}}},
+ {?eh,tc_done,{ct_framework,{init_per_group,g4,
+ [{suite,auto_skip_12_SUITE}]},
+ {auto_skipped,
+ {require_failed,{not_available,unknown_variable_g4}}}}},
+ {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3,
+ {require_failed,{not_available,unknown_variable_g4}}}},
+ {?eh,test_stats,{12,0,{0,31}}},
+ {?eh,tc_auto_skip,{ct_framework,{end_per_group,g4},
+ {require_failed,{not_available,unknown_variable_g4}}}}],
+
+ {?eh,tc_start,{ct_framework,{end_per_group,g3,
+ [{suite,auto_skip_12_SUITE}]}}},
+ {?eh,tc_done,{ct_framework,{end_per_group,g3,
+ [{suite,auto_skip_12_SUITE}]},ok}}],
+
+ {?eh,tc_start,{auto_skip_12_SUITE,end_per_suite}},
+ {?eh,tc_done,{auto_skip_12_SUITE,end_per_suite,ok}},
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}
@@ -442,46 +565,46 @@ test_events(auto_skip) ->
test_events(user_skip) ->
[{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
- {?eh,start_info,{5,5,27}},
+ {?eh,start_info,{6,6,35}},
{?eh,tc_start,{user_skip_1_SUITE,init_per_suite}},
{?eh,tc_done,
{user_skip_1_SUITE,init_per_suite,{skipped,"Whole suite skipped"}}},
- {?eh,tc_auto_skip,
+ {?eh,tc_user_skip,
{user_skip_1_SUITE,tc1,"Whole suite skipped"}},
- {?eh,test_stats,{0,0,{0,1}}},
- {?eh,tc_auto_skip,
+ {?eh,test_stats,{0,0,{1,0}}},
+ {?eh,tc_user_skip,
{user_skip_1_SUITE,tc2,"Whole suite skipped"}},
- {?eh,test_stats,{0,0,{0,2}}},
- {?eh,tc_auto_skip,
+ {?eh,test_stats,{0,0,{2,0}}},
+ {?eh,tc_user_skip,
{user_skip_1_SUITE,tc3,"Whole suite skipped"}},
- {?eh,test_stats,{0,0,{0,3}}},
- {?eh,tc_auto_skip,
+ {?eh,test_stats,{0,0,{3,0}}},
+ {?eh,tc_user_skip,
{user_skip_1_SUITE,tc4,"Whole suite skipped"}},
- {?eh,test_stats,{0,0,{0,4}}},
- {?eh,tc_auto_skip,
+ {?eh,test_stats,{0,0,{4,0}}},
+ {?eh,tc_user_skip,
{user_skip_1_SUITE,end_per_suite,"Whole suite skipped"}},
{?eh,tc_start,{user_skip_2_SUITE,init_per_suite}},
{?eh,tc_done,{user_skip_2_SUITE,init_per_suite,ok}},
{?eh,tc_start,{user_skip_2_SUITE,tc1}},
{?eh,tc_done,{user_skip_2_SUITE,tc1,{skipped,{tc1,skipped}}}},
- {?eh,test_stats,{0,0,{1,4}}},
+ {?eh,test_stats,{0,0,{5,0}}},
[{?eh,tc_start,{user_skip_2_SUITE,{init_per_group,g1,[]}}},
{?eh,tc_done,{user_skip_2_SUITE,{init_per_group,g1,[]},ok}},
{?eh,tc_start,{user_skip_2_SUITE,tc2}},
{?eh,tc_done,{user_skip_2_SUITE,tc2,ok}},
- {?eh,test_stats,{1,0,{1,4}}},
+ {?eh,test_stats,{1,0,{5,0}}},
{?eh,tc_start,{user_skip_2_SUITE,tc3}},
{?eh,tc_done,{user_skip_2_SUITE,tc3,{skipped,{tc3,skipped}}}},
- {?eh,test_stats,{1,0,{2,4}}},
+ {?eh,test_stats,{1,0,{6,0}}},
{?eh,tc_start,{user_skip_2_SUITE,{end_per_group,g1,[]}}},
{?eh,tc_done,{user_skip_2_SUITE,{end_per_group,g1,[]},ok}}],
{?eh,tc_start,{user_skip_2_SUITE,tc4}},
{?eh,tc_done,{user_skip_2_SUITE,tc4,ok}},
- {?eh,test_stats,{2,0,{2,4}}},
+ {?eh,test_stats,{2,0,{6,0}}},
{?eh,tc_start,{user_skip_2_SUITE,end_per_suite}},
{?eh,tc_done,{user_skip_2_SUITE,end_per_suite,ok}},
@@ -489,16 +612,16 @@ test_events(user_skip) ->
{?eh,tc_done,{user_skip_3_SUITE,init_per_suite,ok}},
{?eh,tc_start,{user_skip_3_SUITE,tc1}},
{?eh,tc_done,{user_skip_3_SUITE,tc1,{skipped,"Test case skipped"}}},
- {?eh,test_stats,{2,0,{3,4}}},
+ {?eh,test_stats,{2,0,{7,0}}},
[{?eh,tc_start,{user_skip_3_SUITE,{init_per_group,g1,[]}}},
{?eh,tc_done,{user_skip_3_SUITE,{init_per_group,g1,[]},ok}},
{?eh,tc_start,{user_skip_3_SUITE,tc2}},
{?eh,tc_done,{user_skip_3_SUITE,tc2,ok}},
- {?eh,test_stats,{3,0,{3,4}}},
+ {?eh,test_stats,{3,0,{7,0}}},
{?eh,tc_start,{user_skip_3_SUITE,tc3}},
{?eh,tc_done,{user_skip_3_SUITE,tc3,{skipped,"Test case skipped"}}},
- {?eh,test_stats,{3,0,{4,4}}},
+ {?eh,test_stats,{3,0,{8,0}}},
{?eh,tc_start,{user_skip_3_SUITE,{end_per_group,g1,[]}}},
{?eh,tc_done,{user_skip_3_SUITE,{end_per_group,g1,[]},ok}}],
@@ -506,7 +629,7 @@ test_events(user_skip) ->
{?eh,tc_done,{user_skip_3_SUITE,tc4,
{skipped,{proc_info,{{current_function,{user_skip_3_SUITE,tc4,1}},
{initial_call,{erlang,apply,2}}}}}}},
- {?eh,test_stats,{3,0,{5,4}}},
+ {?eh,test_stats,{3,0,{9,0}}},
{?eh,tc_start,{user_skip_3_SUITE,end_per_suite}},
{?eh,tc_done,{user_skip_3_SUITE,end_per_suite,ok}},
@@ -515,63 +638,219 @@ test_events(user_skip) ->
[{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g1,[]}}},
{?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g1,[]},{skipped,"Group skipped"}}},
- {?eh,tc_auto_skip,{user_skip_4_SUITE,tc1,"Group skipped"}},
- {?eh,test_stats,{3,0,{5,5}}},
- {?eh,tc_auto_skip,{user_skip_4_SUITE,tc2,"Group skipped"}},
- {?eh,test_stats,{3,0,{5,6}}},
- {?eh,tc_auto_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}],
+ {?eh,tc_user_skip,{user_skip_4_SUITE,tc1,"Group skipped"}},
+ {?eh,test_stats,{3,0,{10,0}}},
+ {?eh,tc_user_skip,{user_skip_4_SUITE,tc2,"Group skipped"}},
+ {?eh,test_stats,{3,0,{11,0}}},
+ {?eh,tc_user_skip,{user_skip_4_SUITE,{end_per_group,g1},"Group skipped"}}],
[{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g2,[]}}},
{?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g2,[]},ok}},
{?eh,tc_start,{user_skip_4_SUITE,tc3}},
{?eh,tc_done,{user_skip_4_SUITE,tc3,ok}},
- {?eh,test_stats,{4,0,{5,6}}},
+ {?eh,test_stats,{4,0,{11,0}}},
{?eh,tc_start,{user_skip_4_SUITE,tc4}},
{?eh,tc_done,{user_skip_4_SUITE,tc4,ok}},
- {?eh,test_stats,{5,0,{5,6}}},
+ {?eh,test_stats,{5,0,{11,0}}},
{?eh,tc_start,{user_skip_4_SUITE,{end_per_group,g2,[]}}},
{?eh,tc_done,{user_skip_4_SUITE,{end_per_group,g2,[]},ok}}],
[{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g3,[]}}},
{?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g3,[]},{skipped,"Group skipped"}}},
- {?eh,tc_auto_skip,{user_skip_4_SUITE,tc5,"Group skipped"}},
- {?eh,tc_auto_skip,{user_skip_4_SUITE,tc6,"Group skipped"}},
- {?eh,tc_auto_skip,{user_skip_4_SUITE,tc7,"Group skipped"}},
- {?eh,tc_auto_skip,{user_skip_4_SUITE,tc8,"Group skipped"}},
- {?eh,test_stats,{5,0,{5,10}}},
- {?eh,tc_auto_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}],
+ {?eh,tc_user_skip,{user_skip_4_SUITE,tc5,"Group skipped"}},
+ {?eh,tc_user_skip,{user_skip_4_SUITE,tc6,"Group skipped"}},
+ {?eh,tc_user_skip,{user_skip_4_SUITE,tc7,"Group skipped"}},
+ {?eh,tc_user_skip,{user_skip_4_SUITE,tc8,"Group skipped"}},
+ {?eh,test_stats,{5,0,{15,0}}},
+ {?eh,tc_user_skip,{user_skip_4_SUITE,{end_per_group,g3},"Group skipped"}}],
[{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g5,[]}}},
{?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g5,[]},ok}},
{?eh,tc_start,{user_skip_4_SUITE,tc9}},
{?eh,tc_done,{user_skip_4_SUITE,tc9,ok}},
- {?eh,test_stats,{6,0,{5,10}}},
+ {?eh,test_stats,{6,0,{15,0}}},
[{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g6,[]}}},
{?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g6,[]},{skipped,"Group skipped"}}},
- {?eh,tc_auto_skip,{user_skip_4_SUITE,tc10,"Group skipped"}},
- {?eh,test_stats,{6,0,{5,11}}},
- {?eh,tc_auto_skip,{user_skip_4_SUITE,tc11,"Group skipped"}},
- {?eh,test_stats,{6,0,{5,12}}},
- {?eh,tc_auto_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}],
+ {?eh,tc_user_skip,{user_skip_4_SUITE,tc10,"Group skipped"}},
+ {?eh,test_stats,{6,0,{16,0}}},
+ {?eh,tc_user_skip,{user_skip_4_SUITE,tc11,"Group skipped"}},
+ {?eh,test_stats,{6,0,{17,0}}},
+ {?eh,tc_user_skip,{user_skip_4_SUITE,{end_per_group,g6},"Group skipped"}}],
{?eh,tc_start,{user_skip_4_SUITE,{end_per_group,g5,[]}}},
{?eh,tc_done,{user_skip_4_SUITE,{end_per_group,g5,[]},ok}}],
{?eh,tc_start,{user_skip_4_SUITE,end_per_suite}},
{?eh,tc_done,{user_skip_4_SUITE,end_per_suite,ok}},
- {ct_test_support_eh,tc_start,{user_skip_5_SUITE,init_per_suite}},
+ {?eh,tc_start,{user_skip_5_SUITE,init_per_suite}},
{?eh,tc_done,{user_skip_5_SUITE,init_per_suite,
{skipped,{bad,'Whole suite skipped'}}}},
- {?eh,tc_auto_skip,{user_skip_5_SUITE,tc1,{bad,'Whole suite skipped'}}},
- {?eh,test_stats,{6,0,{5,13}}},
- {?eh,tc_auto_skip,{user_skip_5_SUITE,tc2,{bad,'Whole suite skipped'}}},
- {?eh,test_stats,{6,0,{5,14}}},
- {?eh,tc_auto_skip,{user_skip_5_SUITE,tc3,{bad,'Whole suite skipped'}}},
- {?eh,test_stats,{6,0,{5,15}}},
- {?eh,tc_auto_skip,{user_skip_5_SUITE,tc4,{bad,'Whole suite skipped'}}},
- {?eh,test_stats,{6,0,{5,16}}},
- {?eh,tc_auto_skip,{user_skip_5_SUITE,end_per_suite,{bad,'Whole suite skipped'}}},
+ {?eh,tc_user_skip,{user_skip_5_SUITE,tc1,{bad,'Whole suite skipped'}}},
+ {?eh,test_stats,{6,0,{18,0}}},
+ {?eh,tc_user_skip,{user_skip_5_SUITE,tc2,{bad,'Whole suite skipped'}}},
+ {?eh,test_stats,{6,0,{19,0}}},
+ {?eh,tc_user_skip,{user_skip_5_SUITE,tc3,{bad,'Whole suite skipped'}}},
+ {?eh,test_stats,{6,0,{20,0}}},
+ {?eh,tc_user_skip,{user_skip_5_SUITE,tc4,{bad,'Whole suite skipped'}}},
+ {?eh,test_stats,{6,0,{21,0}}},
+ {?eh,tc_user_skip,{user_skip_5_SUITE,end_per_suite,{bad,'Whole suite skipped'}}},
+
+ {parallel,
+ [{?eh,tc_start,{user_skip_6_SUITE,{init_per_group,ptop1,[parallel]}}},
+ {?eh,tc_done,{user_skip_6_SUITE,
+ {init_per_group,ptop1,[parallel]},
+ {skipped,"Top group skipped"}}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc1,"Top group skipped"}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc3,"Top group skipped"}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc4,"Top group skipped"}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc2,"Top group skipped"}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,{end_per_group,ptop1},
+ "Top group skipped"}}]},
+
+ {parallel,
+ [{?eh,tc_start,{user_skip_6_SUITE,{init_per_group,ptop2,[parallel]}}},
+ {?eh,tc_done,{user_skip_6_SUITE,{init_per_group,ptop2,[parallel]},ok}},
+ {?eh,tc_start,{user_skip_6_SUITE,tc1}},
+ {?eh,tc_done,{user_skip_6_SUITE,tc1,ok}},
+
+ {parallel,
+ [{?eh,tc_start,{user_skip_6_SUITE,{init_per_group,psub2,[parallel]}}},
+ {?eh,tc_done,{user_skip_6_SUITE,
+ {init_per_group,psub2,[parallel]},
+ {skipped,"Sub group skipped"}}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc3,"Sub group skipped"}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc4,"Sub group skipped"}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,{end_per_group,psub2},
+ "Sub group skipped"}}]},
+
+ {?eh,tc_start,{user_skip_6_SUITE,tc2}},
+ {?eh,tc_done,{user_skip_6_SUITE,tc2,ok}},
+ {?eh,test_stats,{8,0,{27,0}}},
+ {?eh,tc_start,{user_skip_6_SUITE,{end_per_group,ptop2,[parallel]}}},
+ {?eh,tc_done,{user_skip_6_SUITE,{end_per_group,ptop2,[parallel]},ok}}]},
+
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}
+ ];
+
+test_events(testspec_skip) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,4}},
+ {?eh,tc_start,{ct_framework,init_per_suite}},
+ {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+ {parallel,
+ [{?eh,tc_start,
+ {user_skip_7_SUITE,{init_per_group,ptop1,[parallel]}}},
+ {?eh,tc_done,
+ {user_skip_7_SUITE,{init_per_group,ptop1,[parallel]},ok}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc1,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{1,0}}},
+ {parallel,
+ [{?eh,tc_start,
+ {user_skip_7_SUITE,{init_per_group,psub1,[parallel]}}},
+ {?eh,tc_done,
+ {user_skip_7_SUITE,{init_per_group,psub1,[parallel]},ok}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc3,"SKIPPED"}},
+ {?eh,tc_start,{user_skip_7_SUITE,tc4}},
+ {?eh,tc_done,{user_skip_7_SUITE,tc4,ok}},
+ {?eh,test_stats,{1,0,{2,0}}},
+ {?eh,tc_start,
+ {user_skip_7_SUITE,{end_per_group,psub1,[parallel]}}},
+ {?eh,tc_done,
+ {user_skip_7_SUITE,{end_per_group,psub1,[parallel]},ok}}]},
+ {?eh,tc_start,{user_skip_7_SUITE,tc2}},
+ {?eh,tc_done,{user_skip_7_SUITE,tc2,ok}},
+ {?eh,test_stats,{2,0,{2,0}}},
+ {?eh,tc_start,
+ {user_skip_7_SUITE,{end_per_group,ptop1,[parallel]}}},
+ {?eh,tc_done,
+ {user_skip_7_SUITE,{end_per_group,ptop1,[parallel]},ok}}]},
+ {?eh,tc_start,{ct_framework,end_per_suite}},
+ {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,4}},
+ {?eh,tc_start,{ct_framework,init_per_suite}},
+ {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,{init_per_group,ptop1},"SKIPPED"}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc1,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{1,0}}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc3,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{2,0}}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc4,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{3,0}}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc2,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{4,0}}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,{end_per_group,ptop1},"SKIPPED"}},
+ {?eh,tc_start,{ct_framework,end_per_suite}},
+ {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,4}},
+ {?eh,tc_start,{ct_framework,init_per_suite}},
+ {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+ {parallel,
+ [{?eh,tc_start,
+ {user_skip_7_SUITE,{init_per_group,ptop1,[parallel]}}},
+ {?eh,tc_done,
+ {user_skip_7_SUITE,{init_per_group,ptop1,[parallel]},ok}},
+ {?eh,tc_user_skip,
+ {user_skip_7_SUITE,{init_per_group,psub1},"SKIPPED"}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc3,"SKIPPED"}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc4,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{2,0}}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,{end_per_group,psub1},"SKIPPED"}},
+ {?eh,tc_start,{user_skip_7_SUITE,tc1}},
+ {?eh,tc_done,{user_skip_7_SUITE,tc1,ok}},
+ {?eh,tc_start,{user_skip_7_SUITE,tc2}},
+ {?eh,tc_done,{user_skip_7_SUITE,tc2,ok}},
+ {?eh,test_stats,{2,0,{2,0}}},
+ {?eh,tc_start,{user_skip_7_SUITE,{end_per_group,ptop1,[parallel]}}},
+ {?eh,tc_done,{user_skip_7_SUITE,{end_per_group,ptop1,[parallel]},ok}}]},
+ {?eh,tc_start,{ct_framework,end_per_suite}},
+ {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,0}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,all,"SKIPPED"}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,4}},
+ {?eh,tc_start,{ct_framework,init_per_suite}},
+ {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+ {parallel,
+ [{?eh,tc_start,{user_skip_6_SUITE,{init_per_group,ptop1,[parallel]}}},
+ {?eh,tc_done,{user_skip_6_SUITE,
+ {init_per_group,ptop1,[parallel]},
+ {skipped,"Top group skipped"}}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc1,"Top group skipped"}},
+ {?eh,test_stats,{0,0,{1,0}}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc3,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{2,0}}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc4,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{3,0}}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc2,"Top group skipped"}},
+ {?eh,test_stats,{0,0,{4,0}}},
+ {?eh,tc_user_skip,
+ {user_skip_6_SUITE,{end_per_group,ptop1},"Top group skipped"}}]},
+ {?eh,tc_start,{ct_framework,end_per_suite}},
+ {?eh,tc_done,{ct_framework,end_per_suite,ok}},
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}
].
+
+
diff --git a/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_12_SUITE.erl b/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_12_SUITE.erl
new file mode 100644
index 0000000000..a168afd386
--- /dev/null
+++ b/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_12_SUITE.erl
@@ -0,0 +1,121 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(auto_skip_12_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% Function: suite() -> Info
+%% Info = [tuple()]
+%%--------------------------------------------------------------------
+suite() ->
+ [].
+
+%%--------------------------------------------------------------------
+%% Function: init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% Function: end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% Function: group(Name) -> Info
+%% Info = [tuple()]
+%%--------------------------------------------------------------------
+group(g1) ->
+ [{require,unknown_variable_g1}];
+group(g4) ->
+ [{require,unknown_variable_g4}];
+group(_) ->
+ [].
+
+%%--------------------------------------------------------------------
+%% Function: init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% Function: end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% Function: groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%%--------------------------------------------------------------------
+groups() ->
+ [{g1,[],[tc1,tc2,{g2,[],[tc3]}]},
+ {g1,[],[tc1,tc2,{g2,[],[tc3]}]},
+ {g3,[],[tc1,tc2,{g4,[],[tc3]}]}].
+
+%%--------------------------------------------------------------------
+%% Function: all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%%--------------------------------------------------------------------
+all() ->
+ [{group,g1},
+ {group,g3}].
+
+%%--------------------------------------------------------------------
+%% Function: TestCase(Config0) ->
+%% ok | exit() | {skip,Reason} | {comment,Comment} |
+%% {save_config,Config1} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% Comment = term()
+%%--------------------------------------------------------------------
+tc1(_Config) ->
+ ok.
+
+tc2(_Config) ->
+ ok.
+
+tc3(_Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_3_SUITE.erl b/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_3_SUITE.erl
index cb64cb76c5..4ef9f50514 100644
--- a/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_3_SUITE.erl
+++ b/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_3_SUITE.erl
@@ -72,7 +72,7 @@ end_per_group(_GroupName, _Config) ->
%% Reason = term()
%%--------------------------------------------------------------------
init_per_testcase(tc1, _Config) ->
- exit({init_per_testcase,tc1,failed});
+ exit('init_per_testcase for tc1 failed');
init_per_testcase(_TestCase, Config) ->
Config.
diff --git a/lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_6_SUITE.erl b/lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_6_SUITE.erl
new file mode 100644
index 0000000000..5f0e5db6f2
--- /dev/null
+++ b/lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_6_SUITE.erl
@@ -0,0 +1,118 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(user_skip_6_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% Function: suite() -> Info
+%% Info = [tuple()]
+%%--------------------------------------------------------------------
+suite() ->
+ [{timetrap,{seconds,30}}].
+
+%%--------------------------------------------------------------------
+%% Function: init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_group(ptop1, Config) ->
+ {skip,"Top group skipped"};
+init_per_group(psub2, Config) ->
+ {skip,"Sub group skipped"};
+init_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% Function: end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% Function: init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% Function: end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% Function: groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%%--------------------------------------------------------------------
+groups() ->
+ [{ptop1,[parallel],[tc1,{psub1,[parallel],[tc3,tc4]},tc2]},
+ {ptop2,[parallel],[tc1,{psub2,[parallel],[tc3,tc4]},tc2]}].
+
+%%--------------------------------------------------------------------
+%% Function: all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%%--------------------------------------------------------------------
+all() ->
+ [{group,ptop1},{group,ptop2}].
+
+%%--------------------------------------------------------------------
+%% Function: TestCase(Config0) ->
+%% ok | exit() | {skip,Reason} | {comment,Comment} |
+%% {save_config,Config1} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% Comment = term()
+%%--------------------------------------------------------------------
+tc1(_) ->
+ ok.
+
+tc2(_) ->
+ ok.
+
+tc3(_) ->
+ ok.
+
+tc4(_) ->
+ ok.
diff --git a/lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_7_SUITE.erl b/lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_7_SUITE.erl
new file mode 100644
index 0000000000..82ce536a79
--- /dev/null
+++ b/lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_7_SUITE.erl
@@ -0,0 +1,113 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(user_skip_7_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% Function: suite() -> Info
+%% Info = [tuple()]
+%%--------------------------------------------------------------------
+suite() ->
+ [{timetrap,{seconds,30}}].
+
+%%--------------------------------------------------------------------
+%% Function: init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% Function: end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% Function: init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% Function: end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% Function: groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%%--------------------------------------------------------------------
+groups() ->
+ [{ptop1,[parallel],[tc1,{psub1,[parallel],[tc3,tc4]},tc2]}].
+
+%%--------------------------------------------------------------------
+%% Function: all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%%--------------------------------------------------------------------
+all() ->
+ [{group,ptop1}].
+
+%%--------------------------------------------------------------------
+%% Function: TestCase(Config0) ->
+%% ok | exit() | {skip,Reason} | {comment,Comment} |
+%% {save_config,Config1} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% Comment = term()
+%%--------------------------------------------------------------------
+tc1(_) ->
+ ok.
+
+tc2(_) ->
+ ok.
+
+tc3(_) ->
+ ok.
+
+tc4(_) ->
+ ok.
diff --git a/lib/common_test/test/ct_surefire_SUITE.erl b/lib/common_test/test/ct_surefire_SUITE.erl
index b86b47f0a2..c5e44682b0 100644
--- a/lib/common_test/test/ct_surefire_SUITE.erl
+++ b/lib/common_test/test/ct_surefire_SUITE.erl
@@ -182,7 +182,7 @@ test_events(_) ->
{?eh,test_stats,{1,1,{1,0}}},
{?eh,tc_start,{surefire_SUITE,tc_autoskip_require}},
{?eh,tc_done,{surefire_SUITE,tc_autoskip_require,
- {skipped,{require_failed,'_'}}}},
+ {auto_skipped,{require_failed,'_'}}}},
{?eh,test_stats,{1,1,{1,1}}},
[{?eh,tc_start,{surefire_SUITE,{init_per_group,g,[]}}},
{?eh,tc_done,{surefire_SUITE,{init_per_group,g,[]},ok}},
@@ -198,7 +198,7 @@ test_events(_) ->
{?eh,test_stats,{2,2,{2,1}}},
{?eh,tc_start,{surefire_SUITE,tc_autoskip_require}},
{?eh,tc_done,{surefire_SUITE,tc_autoskip_require,
- {skipped,{require_failed,'_'}}}},
+ {auto_skipped,{require_failed,'_'}}}},
{?eh,test_stats,{2,2,{2,2}}},
{?eh,tc_start,{surefire_SUITE,{end_per_group,g,[]}}},
{?eh,tc_done,{surefire_SUITE,{end_per_group,g,[]},ok}}],
@@ -210,7 +210,7 @@ test_events(_) ->
{surefire_SUITE,init_per_group,
{'EXIT',all_cases_should_be_skipped}}}}},
{?eh,test_stats,{2,2,{2,3}}},
- {?eh,tc_auto_skip,{surefire_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{surefire_SUITE,{end_per_group,g_fail},
{failed,
{surefire_SUITE,init_per_group,
{'EXIT',all_cases_should_be_skipped}}}}}],
@@ -328,6 +328,7 @@ events_to_result([]) ->
result(ok) ->[];
result({skipped,_}) -> [s];
+result({auto_skipped,_}) -> [s];
result({failed,_}) -> [f].
%% Using the expected events' last test_stats element to produce the
diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE.erl
index 8e4852369d..9882fa980c 100644
--- a/lib/common_test/test/ct_test_server_if_1_SUITE.erl
+++ b/lib/common_test/test/ct_test_server_if_1_SUITE.erl
@@ -65,12 +65,12 @@ groups() ->
[].
init_per_group(_GroupName, Config) ->
- Config.
+ Config.
end_per_group(_GroupName, Config) ->
- Config.
+ Config.
+
-
%%--------------------------------------------------------------------
%% TEST CASES
@@ -104,7 +104,7 @@ ts_if_1(Config) when is_list(Config) ->
TestEvents = events_to_check(ts_if_1),
ok = ct_test_support:verify_events(TestEvents, Events, Config).
-
+
%%%-----------------------------------------------------------------
%%% HELP FUNCTIONS
@@ -112,17 +112,17 @@ ts_if_1(Config) when is_list(Config) ->
setup(Test, Config) ->
Opts0 = ct_test_support:get_opts(Config),
-% Level = ?config(trace_level, Config),
-% EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
-% Opts = Opts0 ++ [Test,{event_handler,{?eh,EvHArgs}}],
+ % Level = ?config(trace_level, Config),
+ % EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+ % Opts = Opts0 ++ [Test,{event_handler,{?eh,EvHArgs}}],
Opts = [Test | Opts0],
ERPid = ct_test_support:start_event_receiver(Config),
{Opts,ERPid}.
reformat(Events, EH) ->
ct_test_support:reformat(Events, EH).
-%reformat(Events, _EH) ->
-% Events.
+ %reformat(Events, _EH) ->
+ % Events.
%%%-----------------------------------------------------------------
%%% TEST EVENTS
@@ -140,14 +140,15 @@ test_events(ts_if_1) ->
[
{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
- {?eh,start_info,{10,6,26}},
+ {?eh,start_info,{10,8,25}},
+
{?eh,tc_start,{ts_if_1_SUITE,init_per_suite}},
{?eh,tc_done,{ts_if_1_SUITE,init_per_suite,ok}},
{?eh,tc_start,{ts_if_1_SUITE,tc1}},
- {?eh,tc_done,{ts_if_1_SUITE,tc1,{skipped,
- {failed,
- {ts_if_1_SUITE,init_per_testcase,
- {timetrap_timeout,2000}}}}}},
+ {?eh,tc_done,{ts_if_1_SUITE,tc1,
+ {auto_skipped,
+ {failed,
+ {ts_if_1_SUITE,init_per_testcase,{timetrap_timeout,2000}}}}}},
{?eh,test_stats,{0,0,{0,1}}},
{?eh,tc_start,{ts_if_1_SUITE,tc2}},
{?eh,tc_done,{ts_if_1_SUITE,tc2,
@@ -159,115 +160,162 @@ test_events(ts_if_1) ->
{?eh,tc_start,{ts_if_1_SUITE,tc4}},
{?eh,tc_done,{ts_if_1_SUITE,tc4,{failed,{error,failed_on_purpose}}}},
{?eh,test_stats,{1,2,{0,1}}},
- {?eh,tc_done,{ts_if_1_SUITE,tc5,{skipped,{sequence_failed,seq1,tc4}}}},
- {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_done,{ts_if_1_SUITE,tc5,{auto_skipped,{sequence_failed,seq1,tc4}}}},
+ {?eh,test_stats,{1,2,{0,2}}},
[{?eh,tc_start,{ts_if_1_SUITE,{init_per_group,seq2,[sequence]}}},
{?eh,tc_done,{ts_if_1_SUITE,{init_per_group,seq2,[sequence]},ok}},
{?eh,tc_start,{ts_if_1_SUITE,tc4}},
{?eh,tc_done,{ts_if_1_SUITE,tc4,{failed,{error,failed_on_purpose}}}},
- {?eh,test_stats,{1,3,{1,1}}},
+ {?eh,test_stats,{1,3,{0,2}}},
{?eh,tc_auto_skip,{ts_if_1_SUITE,tc5,{failed,{ts_if_1_SUITE,tc4}}}},
- {?eh,test_stats,{1,3,{1,2}}},
+ {?eh,test_stats,{1,3,{0,3}}},
{?eh,tc_start,{ts_if_1_SUITE,{end_per_group,seq2,[sequence]}}},
{?eh,tc_done,{ts_if_1_SUITE,{end_per_group,seq2,[sequence]},ok}}],
{?eh,tc_start,{ts_if_1_SUITE,tc6}},
- {?eh,tc_done,{ts_if_1_SUITE,tc6,{skipped,{require_failed,{not_available,void}}}}},
- {?eh,test_stats,{1,3,{1,3}}},
- {?eh,tc_start,{ts_if_1_SUITE,tc7}},
- {?eh,tc_done,{ts_if_1_SUITE,tc7,ok}},
- {?eh,test_stats,{2,3,{1,3}}},
+ {?eh,tc_done,{ts_if_1_SUITE,tc6,{auto_skipped,{require_failed,
+ {not_available,void}}}}},
+ {?eh,test_stats,{1,3,{0,4}}},
+ {?eh,tc_done,{ts_if_1_SUITE,tc7,{auto_skipped,
+ {testcase0_failed,bad_return_value}}}},
+ {?eh,test_stats,{1,3,{0,5}}},
{?eh,tc_start,{ts_if_1_SUITE,tc8}},
{?eh,tc_done,{ts_if_1_SUITE,tc8,{skipped,"tc8 skipped"}}},
- {?eh,test_stats,{2,3,{2,3}}},
+ {?eh,test_stats,{1,3,{1,5}}},
{?eh,tc_start,{ts_if_1_SUITE,tc9}},
{?eh,tc_done,{ts_if_1_SUITE,tc9,{skipped,'tc9 skipped'}}},
- {?eh,test_stats,{2,3,{3,3}}},
+ {?eh,test_stats,{1,3,{2,5}}},
{?eh,tc_start,{ts_if_1_SUITE,tc10}},
- {?eh,tc_done,{ts_if_1_SUITE,tc10,{failed,{error,{function_clause,'_'}}}}},
- {?eh,test_stats,{2,4,{3,3}}},
+ {?eh,tc_done,{ts_if_1_SUITE,tc10,
+ {failed,{error,{function_clause,'_'}}}}},
+ {?eh,test_stats,{1,4,{2,5}}},
{?eh,tc_start,{ts_if_1_SUITE,tc11}},
{?eh,tc_done,{ts_if_1_SUITE,tc11,
- {skipped,{failed,{ts_if_1_SUITE,init_per_testcase,bad_return}}}}},
- {?eh,test_stats,{2,4,{3,4}}},
+ {auto_skipped,
+ {failed,{ts_if_1_SUITE,init_per_testcase,bad_return}}}}},
+ {?eh,test_stats,{1,4,{2,6}}},
[{?eh,tc_start,{ts_if_1_SUITE,{init_per_group,g1,[]}}},
- {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g1,[]},{skipped,g1_got_skipped}}},
- {?eh,tc_auto_skip,{ts_if_1_SUITE,gtc1,g1_got_skipped}},
- {?eh,test_stats,{2,4,{3,5}}},
- {?eh,tc_auto_skip,{ts_if_1_SUITE,end_per_group,g1_got_skipped}}],
-
+ {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g1,[]},
+ {skipped,g1_got_skipped}}},
+ {?eh,tc_user_skip,{ts_if_1_SUITE,gtc1,g1_got_skipped}},
+ {?eh,test_stats,{1,4,{3,6}}},
+ {?eh,tc_user_skip,{ts_if_1_SUITE,{end_per_group,g1},g1_got_skipped}}],
+
{parallel,
[{?eh,tc_start,{ts_if_1_SUITE,{init_per_group,g2,[parallel]}}},
{?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g2,[parallel]},ok}},
[{?eh,tc_start,{ts_if_1_SUITE,{init_per_group,g3,[]}}},
- {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g3,[]},{skipped,g3_got_skipped}}},
- {?eh,tc_auto_skip,{ts_if_1_SUITE,gtc2,g3_got_skipped}},
- {?eh,test_stats,{2,4,{3,6}}},
- {?eh,tc_auto_skip,{ts_if_1_SUITE,end_per_group,g3_got_skipped}}],
+ {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g3,[]},{skipped,g3_got_skipped}}},
+ {?eh,tc_user_skip,{ts_if_1_SUITE,gtc2,g3_got_skipped}},
+ {?eh,test_stats,{1,4,{4,6}}},
+ {?eh,tc_user_skip,{ts_if_1_SUITE,{end_per_group,g3},g3_got_skipped}}],
{?eh,tc_start,{ts_if_1_SUITE,{end_per_group,g2,[parallel]}}},
{?eh,tc_done,{ts_if_1_SUITE,{end_per_group,g2,[parallel]},ok}}]},
{?eh,tc_start,{ts_if_1_SUITE,tc12}},
{?eh,tc_done,{ts_if_1_SUITE,tc12,{failed,{testcase_aborted,'stopping tc12'}}}},
- {?eh,test_stats,{2,5,{3,6}}},
+ {?eh,test_stats,{1,5,{4,6}}},
{?eh,tc_start,{ts_if_1_SUITE,tc13}},
{?eh,tc_done,{ts_if_1_SUITE,tc13,ok}},
- {?eh,test_stats,{3,5,{3,6}}},
+ {?eh,test_stats,{2,5,{4,6}}},
{?eh,tc_start,{ts_if_1_SUITE,end_per_suite}},
{?eh,tc_done,{ts_if_1_SUITE,end_per_suite,ok}},
+
{?eh,tc_start,{ts_if_2_SUITE,init_per_suite}},
- {?eh,tc_done,{ts_if_2_SUITE,init_per_suite,
- {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}},
- {?eh,tc_auto_skip,{ts_if_2_SUITE,my_test_case,
- {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}},
- {?eh,test_stats,{3,5,{3,7}}},
- {?eh,tc_auto_skip,{ts_if_2_SUITE,end_per_suite,
- {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}},
+ {?eh,tc_done,
+ {ts_if_2_SUITE,init_per_suite,
+ {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}},
+ {?eh,tc_auto_skip,
+ {ts_if_2_SUITE,my_test_case,
+ {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}},
+ {?eh,test_stats,{2,5,{4,7}}},
+ {?eh,tc_auto_skip,
+ {ts_if_2_SUITE,end_per_suite,
+ {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}},
+
{?eh,tc_start,{ct_framework,error_in_suite}},
- {?eh,test_stats,{3,5,{4,7}}},
+ {?eh,test_stats,{2,6,{4,7}}},
+
{?eh,tc_start,{ct_framework,error_in_suite}},
- {?eh,test_stats,{3,5,{5,7}}},
+ {?eh,test_stats,{2,7,{4,7}}},
+
{?eh,tc_start,{ts_if_5_SUITE,init_per_suite}},
{?eh,tc_done,{ts_if_5_SUITE,init_per_suite,
- {skipped,{require_failed_in_suite0,{not_available,undef_variable}}}}},
- {?eh,tc_auto_skip,{ts_if_5_SUITE,my_test_case,
- {require_failed_in_suite0,{not_available,undef_variable}}}},
- {?eh,test_stats,{3,5,{5,8}}},
- {?eh,tc_auto_skip,{ts_if_5_SUITE,end_per_suite,
- {require_failed_in_suite0,{not_available,undef_variable}}}},
+ {auto_skipped,
+ {require_failed_in_suite0,{not_available,undef_variable}}}}},
+ {?eh,tc_auto_skip,
+ {ts_if_5_SUITE,my_test_case,
+ {require_failed_in_suite0,{not_available,undef_variable}}}},
+ {?eh,test_stats,{2,7,{4,8}}},
+ {?eh,tc_auto_skip,
+ {ts_if_5_SUITE,end_per_suite,
+ {require_failed_in_suite0,{not_available,undef_variable}}}},
+
{?eh,tc_start,{ct_framework,init_per_suite}},
{?eh,tc_done,{ct_framework,init_per_suite,
{failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}},
- {?eh,tc_auto_skip,{ts_if_6_SUITE,tc1,
- {failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}},
- {?eh,test_stats,{3,5,{5,9}}},
+ {?eh,tc_auto_skip,
+ {ts_if_6_SUITE,tc1,
+ {failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}},
+ {?eh,test_stats,{2,7,{4,9}}},
+ {?eh,tc_auto_skip,
+ {ct_framework,end_per_suite,
+ {failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}},
- {?eh,tc_start,{ts_if_7_SUITE,tc1}},
- {?eh,tc_done,{ts_if_7_SUITE,tc1,ok}},
- {?eh,test_stats,{4,5,{5,9}}},
+ {?eh,tc_start,{ct_framework,init_per_suite}},
+ {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+ {?eh,tc_done,
+ {ts_if_7_SUITE,tc1,{auto_skipped,{testcase0_failed,bad_return_value}}}},
+ {?eh,test_stats,{2,7,{4,10}}},
+ {?eh,tc_done,{ts_if_7_SUITE,
+ {init_per_group,g1,[]},
+ {auto_skipped,{group0_failed,bad_return_value}}}},
+ {?eh,tc_auto_skip,
+ {ts_if_7_SUITE,tc2,{group0_failed,bad_return_value}}},
+ {?eh,test_stats,{2,7,{4,11}}},
+ {?eh,tc_auto_skip,
+ {ts_if_7_SUITE,{end_per_group,g1},{group0_failed,bad_return_value}}},
+
+ [{?eh,tc_start,{ts_if_7_SUITE,{init_per_group,g2,[]}}},
+ {?eh,tc_done,{ts_if_7_SUITE,{init_per_group,g2,[]},ok}},
+ {?eh,tc_done,{ts_if_7_SUITE,tc2,
+ {auto_skipped,{testcase0_failed,bad_return_value}}}},
+ {?eh,test_stats,{2,7,{4,12}}},
+ {?eh,tc_start,{ts_if_7_SUITE,{end_per_group,g2,[]}}},
+ {?eh,tc_done,{ts_if_7_SUITE,{end_per_group,g2,[]},ok}}],
+
+ {?eh,tc_start,{ct_framework,end_per_suite}},
+ {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+
+
+ {?eh,tc_start,{ct_framework,init_per_suite}},
+ {?eh,tc_done,{ct_framework,init_per_suite,ok}},
{?eh,tc_start,{ts_if_8_SUITE,tc1}},
{?eh,tc_done,{ts_if_8_SUITE,tc1,{failed,{error,failed_on_purpose}}}},
- {?eh,test_stats,{4,6,{5,9}}},
-
+ {?eh,test_stats,{2,8,{4,12}}},
+ {?eh,tc_start,{ct_framework,end_per_suite}},
+ {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+
+
{?eh,tc_user_skip,{skipped_by_spec_1_SUITE,all,"should be skipped"}},
- {?eh,test_stats,{4,6,{6,9}}},
-
+ {?eh,test_stats,{2,8,{5,12}}},
{?eh,tc_start,{skipped_by_spec_2_SUITE,init_per_suite}},
{?eh,tc_done,{skipped_by_spec_2_SUITE,init_per_suite,ok}},
{?eh,tc_user_skip,{skipped_by_spec_2_SUITE,tc1,"should be skipped"}},
- {?eh,test_stats,{4,6,{7,9}}},
+ {?eh,test_stats,{2,8,{6,12}}},
{?eh,tc_start,{skipped_by_spec_2_SUITE,end_per_suite}},
{?eh,tc_done,{skipped_by_spec_2_SUITE,end_per_suite,ok}},
-
+
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}
].
+
diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_7_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_7_SUITE.erl
index a2254848d0..20e04c464b 100644
--- a/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_7_SUITE.erl
+++ b/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_7_SUITE.erl
@@ -73,7 +73,8 @@ end_per_testcase(_TestCase, _Config) ->
%% N = integer() | forever
%%--------------------------------------------------------------------
groups() ->
- [].
+ [{g1,[],[tc2]},
+ {g2,[],[tc2]}].
%%--------------------------------------------------------------------
%% Function: all() -> GroupsAndTestCases | {skip,Reason}
@@ -83,7 +84,12 @@ groups() ->
%% Reason = term()
%%--------------------------------------------------------------------
all() ->
- [tc1].
+ [tc1,{group,g1},{group,g2}].
+
+group(g1) ->
+ exit(g1_byebye);
+group(_) ->
+ [].
tc1() ->
exit(tc1_byebye).
@@ -91,3 +97,9 @@ tc1() ->
tc1(_) ->
done.
+tc2() ->
+ exit(tc2_byebye).
+
+tc2(_) ->
+ done.
+
diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl
index 67e430f821..772274ce7e 100644
--- a/lib/common_test/test/ct_test_support.erl
+++ b/lib/common_test/test/ct_test_support.erl
@@ -692,8 +692,10 @@ locate({parallel,TEvs}, Node, Evs, Config) ->
test_server:format("Found ~p!", [TEv]),
{Done,RemEvs2,length(RemEvs2)}
end;
- %% end_per_group auto skipped
- (TEv={TEH,tc_auto_skip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize}) ->
+ %% end_per_group auto- or user skipped
+ (TEv={TEH,AutoOrUserSkip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize})
+ when AutoOrUserSkip == tc_auto_skip;
+ AutoOrUserSkip == tc_user_skip ->
RemEvs1 =
lists:dropwhile(
fun({EH,#event{name=tc_auto_skip,
@@ -704,10 +706,18 @@ locate({parallel,TEvs}, Node, Evs, Config) ->
match -> false;
_ -> true
end;
+ ({EH,#event{name=tc_user_skip,
+ node=EvNode,
+ data={Mod,end_per_group,Reason}}}) when
+ EH == TEH, EvNode == Node, Mod == M ->
+ case match_data(R, Reason) of
+ match -> false;
+ _ -> true
+ end;
({EH,#event{name=stop_logging,
node=EvNode,data=_}}) when
EH == TEH, EvNode == Node ->
- exit({tc_auto_skip_not_found,TEv});
+ exit({tc_auto_or_user_skip_not_found,TEv});
(_) ->
true
end, RemEvs),
@@ -925,8 +935,10 @@ locate({shuffle,TEvs}, Node, Evs, Config) ->
test_server:format("Found ~p!", [TEv]),
{Done,RemEvs2,length(RemEvs2)}
end;
- %% end_per_group auto skipped
- (TEv={TEH,tc_auto_skip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize}) ->
+ %% end_per_group auto-or user skipped
+ (TEv={TEH,AutoOrUserSkip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize})
+ when AutoOrUserSkip == tc_auto_skip;
+ AutoOrUserSkip == tc_user_skip ->
RemEvs1 =
lists:dropwhile(
fun({EH,#event{name=tc_auto_skip,
@@ -934,6 +946,11 @@ locate({shuffle,TEvs}, Node, Evs, Config) ->
data={Mod,end_per_group,Reason}}}) when
EH == TEH, EvNode == Node, Mod == M, Reason == R ->
false;
+ ({EH,#event{name=tc_user_skip,
+ node=EvNode,
+ data={Mod,end_per_group,Reason}}}) when
+ EH == TEH, EvNode == Node, Mod == M, Reason == R ->
+ false;
({EH,#event{name=stop_logging,
node=EvNode,data=_}}) when
EH == TEH, EvNode == Node ->
@@ -1178,6 +1195,9 @@ log_events1([E={_EH,tc_done,{_M,{end_per_group,_GrName,Props},_R}} | Evs], Dev,
log_events1([E={_EH,tc_auto_skip,{_M,end_per_group,_Reason}} | Evs], Dev, Ind) ->
io:format(Dev, "~s~p],~n", [Ind,E]),
log_events1(Evs, Dev, Ind--" ");
+log_events1([E={_EH,tc_user_skip,{_M,end_per_group,_Reason}} | Evs], Dev, Ind) ->
+ io:format(Dev, "~s~p],~n", [Ind,E]),
+ log_events1(Evs, Dev, Ind--" ");
log_events1([E], Dev, Ind) ->
io:format(Dev, "~s~p~n].~n", [Ind,E]),
ok;
diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl
index 6a4a4acd80..187b5e6d3a 100644
--- a/lib/common_test/test/ct_testspec_1_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_1_SUITE.erl
@@ -760,16 +760,42 @@ test_events(all_groups) ->
test_events(skip_all_groups) ->
[
{?eh,start_logging,'_'},
+ {?eh,start_info,{1,1,12}},
{?eh,tc_start,{groups_11_SUITE,init_per_suite}},
- {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1a},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1a},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}},
{?eh,test_stats,{0,0,{1,0}}},
- {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}},
{?eh,test_stats,{0,0,{2,0}}},
- {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_2},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1a},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1b},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}},
{?eh,test_stats,{0,0,{3,0}}},
- {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_4},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}},
{?eh,test_stats,{0,0,{4,0}}},
- {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1b},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_2},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_2a,"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{5,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_3a,"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{6,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_3b,"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{7,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_2b,"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{8,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_2},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_4},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_5a,"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{9,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_7a,"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{10,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_7b,"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{11,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_5b,"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{12,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_4},"SKIPPED!"}},
+ {?eh,tc_start,{groups_11_SUITE,end_per_suite}},
+ {?eh,tc_done,{groups_11_SUITE,end_per_suite,init}},
{negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
];
@@ -788,21 +814,35 @@ test_events(group) ->
test_events(skip_group) ->
[
- {?eh,start_logging,'_'},
- {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
-
- {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}},
- {?eh,tc_start,{groups_11_SUITE,testcase_1a}},
- {?eh,tc_start,{groups_11_SUITE,testcase_1b}},
- {?eh,test_stats,{2,0,{0,0}}},
- {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}},
-
- {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}},
- {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_2},"SKIPPED!"}},
- %%! But not test_group_7 since it's a sub-group!
- {?eh,test_stats,{2,0,{2,0}}},
- {negative,{?eh,tc_user_skip,'_'},{?eh,stop_logging,'_'}}
- ];
+ {?eh,start_logging,'_'},
+ {?eh,start_info,{1,1,8}},
+ {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
+ [{?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}},
+ {?eh,tc_done,{groups_11_SUITE,{init_per_group,test_group_1a,[]},ok}},
+ {?eh,tc_start,{groups_11_SUITE,testcase_1a}},
+ {?eh,tc_start,{groups_11_SUITE,testcase_1b}},
+ {?eh,test_stats,{2,0,{0,0}}},
+ {?eh,tc_start,{groups_11_SUITE,{end_per_group,test_group_1a,[]}}},
+ {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},ok}}],
+ {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1b},
+ "SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}},
+ {?eh,test_stats,{2,0,{2,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1b},
+ "SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_2},
+ "SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_2a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_3a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_3b,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_2b,"SKIPPED!"}},
+ {?eh,test_stats,{2,0,{6,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_2},
+ "SKIPPED!"}},
+ {?eh,tc_done,{groups_11_SUITE,end_per_suite,init}},
+ {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+ ];
test_events(group_all_testcases) ->
[
@@ -820,11 +860,23 @@ test_events(group_all_testcases) ->
test_events(skip_group_all_testcases) ->
[
{?eh,start_logging,'_'},
+ {?eh,start_info,{1,1,4}},
{?eh,tc_start,{groups_11_SUITE,init_per_suite}},
- {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1a},"SKIPPED!"}},
- {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1a},
+ "SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}},
{?eh,test_stats,{0,0,{2,0}}},
- {?eh,tc_start,{groups_11_SUITE,end_per_suite}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1a},
+ "SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1b},
+ "SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{4,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1b},
+ "SKIPPED!"}},
+ {?eh,tc_done,{groups_11_SUITE,end_per_suite,init}},
{negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
];
@@ -934,7 +986,7 @@ test_events(subgroup) ->
[
{?eh,start_logging,'_'},
{?eh,tc_start,{groups_12_SUITE,init_per_suite}},
-
+
{parallel,
[{?eh,tc_start,
{groups_12_SUITE,{init_per_group,test_group_2,[parallel]}}},
@@ -962,36 +1014,58 @@ test_events(subgroup) ->
test_events(skip_subgroup) ->
[
{?eh,start_logging,'_'},
+ {?eh,start_info,{1,1,6}},
{?eh,tc_start,{groups_12_SUITE,init_per_suite}},
- [{?eh,tc_start,
- {groups_12_SUITE,{init_per_group,test_group_4,[]}}},
+ [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_4,[]}}},
+ {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_4,[]},ok}},
+
{parallel,
- [{?eh,tc_start,
- {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}},
- {?eh,tc_done,
- {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}},
- {parallel,
- [{?eh,tc_start,
- {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}},
- {?eh,tc_done,
- {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}},
- [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}},
- {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}],
- {?eh,tc_user_skip,
- {groups_12_SUITE,{group,test_group_8},"SKIPPED!"}},
- {?eh,tc_start,
- {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}},
- {?eh,tc_done,
- {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}}
- ]},
- {?eh,tc_start,
- {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}},
- {?eh,tc_done,
- {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]},
- {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}],
+ [{?eh,tc_start,{groups_12_SUITE,
+ {init_per_group,test_group_5,[parallel]}}},
+ {?eh,tc_done,{groups_12_SUITE,
+ {init_per_group,test_group_5,[parallel]},ok}},
+
+ {parallel,
+ [{?eh,tc_start,{groups_12_SUITE,
+ {init_per_group,test_group_6,[parallel]}}},
+ {?eh,tc_done,{groups_12_SUITE,
+ {init_per_group,test_group_6,[parallel]},ok}},
+
+ [{?eh,tc_start,{groups_12_SUITE,
+ {init_per_group,test_group_7,[sequence]}}},
+ {?eh,tc_done,{groups_12_SUITE,
+ {init_per_group,test_group_7,[sequence]},ok}},
+ {?eh,tc_done,{groups_12_SUITE,testcase_7a,ok}},
+ {?eh,tc_done,{groups_12_SUITE,testcase_7b,ok}},
+ {?eh,tc_start,{groups_12_SUITE,
+ {end_per_group,test_group_7,[sequence]}}},
+ {?eh,tc_done,{groups_12_SUITE,
+ {end_per_group,test_group_7,[sequence]},ok}}],
+
+ {?eh,tc_user_skip,{groups_12_SUITE,
+ {init_per_group,test_group_8},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_8a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_8b,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,
+ {end_per_group,test_group_8},"SKIPPED!"}},
+
+ {?eh,tc_start,{groups_12_SUITE,
+ {end_per_group,test_group_6,[parallel]}}},
+ {?eh,tc_done,{groups_12_SUITE,
+ {end_per_group,test_group_6,[parallel]},ok}}]},
+
+ {?eh,test_stats,{4,0,{2,0}}},
+ {?eh,tc_start,{groups_12_SUITE,
+ {end_per_group,test_group_5,[parallel]}}},
+ {?eh,tc_done,{groups_12_SUITE,
+ {end_per_group,test_group_5,[parallel]},ok}}]},
- {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}},
+ {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}},
+ {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_4,[]},ok}}],
+
+ {?eh,tc_start,{groups_12_SUITE,end_per_suite}},
+ {?eh,tc_done,{groups_12_SUITE,end_per_suite,init}},
{negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
];
@@ -1066,17 +1140,26 @@ test_events(subgroup_all_testcases) ->
test_events(skip_subgroup_all_testcases) ->
[
{?eh,start_logging,'_'},
+ {?eh,start_info,{1,1,6}},
{?eh,tc_start,{groups_12_SUITE,init_per_suite}},
-
- [{?eh,tc_start,
- {groups_12_SUITE,{init_per_group,test_group_4,[]}}},
- {?eh,tc_done,
- {groups_12_SUITE,{init_per_group,test_group_4,[]},ok}},
- {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_5},"SKIPPED!"}},
- {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}},
- {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_4,[]},ok}}
- ],
-
+ [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_4,[]}}},
+ {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_4,[]},ok}},
+ {?eh,tc_user_skip,{groups_12_SUITE,
+ {init_per_group,test_group_5},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_5a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_7a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_7b,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_8a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_8b,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_5b,"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{6,0}}},
+ {?eh,tc_user_skip,{groups_12_SUITE,
+ {end_per_group,test_group_5},"SKIPPED!"}},
+ {?eh,tc_start,{groups_12_SUITE,
+ {end_per_group,test_group_4,[]}}},
+ {?eh,tc_done,{groups_12_SUITE,
+ {end_per_group,test_group_4,[]},ok}}],
+ {?eh,tc_start,{groups_12_SUITE,end_per_suite}},
{?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}},
{negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
];
@@ -1194,13 +1277,30 @@ test_events(skip_subgroup_testcase) ->
test_events(sub_skipped_by_top) ->
[
{?eh,start_logging,'_'},
+ {?eh,start_info,{1,1,12}},
{?eh,tc_start,{groups_12_SUITE,init_per_suite}},
-
- {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_4},"SKIPPED!"}},
- {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_4},"SKIPPED!"}},
-
+ {?eh,tc_user_skip,{groups_12_SUITE,{init_per_group,test_group_4},
+ "SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_5a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_7a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_7b,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_8a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_8b,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_5b,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,
+ {end_per_group,test_group_4},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,
+ {init_per_group,test_group_4},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_5a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_7a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_7b,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_8a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_8b,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_5b,"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{12,0}}},
+ {?eh,tc_user_skip,{groups_12_SUITE,
+ {end_per_group,test_group_4},"SKIPPED!"}},
{?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}},
-
{negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
];
diff --git a/lib/common_test/test/ct_testspec_3_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE.erl
index 5fa187e5b4..9f8ca84e45 100644
--- a/lib/common_test/test/ct_testspec_3_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_3_SUITE.erl
@@ -1539,7 +1539,7 @@ flat_spec1_events() ->
{?eh,test_stats,{1,2,{0,0}}},
{?eh,tc_start,{t11_SUITE,autoskip_tc}},
{?eh,tc_done,
- {t11_SUITE,autoskip_tc,{skipped,
+ {t11_SUITE,autoskip_tc,{auto_skipped,
{failed,
{t11_SUITE,init_per_testcase,
{kaboom,'_'}}}}}},
@@ -1562,7 +1562,7 @@ flat_spec1_events() ->
{?eh,test_stats,{2,4,{1,1}}},
{?eh,tc_start,{t21_SUITE,autoskip_tc}},
{?eh,tc_done,
- {t21_SUITE,autoskip_tc,{skipped,
+ {t21_SUITE,autoskip_tc,{auto_skipped,
{failed,
{t21_SUITE,init_per_testcase,
{kaboom,'_'}}}}}},
@@ -1589,7 +1589,7 @@ flat_spec2_events() ->
{?eh,test_stats,{1,2,{0,0}}},
{?eh,tc_start,{t12_SUITE,autoskip_tc}},
{?eh,tc_done,
- {t12_SUITE,autoskip_tc,{skipped,
+ {t12_SUITE,autoskip_tc,{auto_skipped,
{failed,
{t12_SUITE,init_per_testcase,
{kaboom,'_'}}}}}},
@@ -1612,7 +1612,7 @@ flat_spec2_events() ->
{?eh,test_stats,{2,4,{1,1}}},
{?eh,tc_start,{t12_SUITE,autoskip_tc}},
{?eh,tc_done,
- {t12_SUITE,autoskip_tc,{skipped,
+ {t12_SUITE,autoskip_tc,{auto_skipped,
{failed,
{t12_SUITE,init_per_testcase,
{kaboom,'_'}}}}}},
@@ -1635,7 +1635,7 @@ flat_spec2_events() ->
{?eh,test_stats,{3,6,{2,2}}},
{?eh,tc_start,{t22_SUITE,autoskip_tc}},
{?eh,tc_done,
- {t22_SUITE,autoskip_tc,{skipped,
+ {t22_SUITE,autoskip_tc,{auto_skipped,
{failed,
{t22_SUITE,init_per_testcase,
{kaboom,'_'}}}}}},
diff --git a/lib/test_server/src/test_server.erl b/lib/test_server/src/test_server.erl
index 6ddb2b615f..54be6d4c72 100644
--- a/lib/test_server/src/test_server.erl
+++ b/lib/test_server/src/test_server.erl
@@ -915,6 +915,7 @@ run_test_case_eval(Mod, Func, Args0, Name, Ref, RunInit,
put(test_server_logopts, LogOpts),
Where = [{Mod,Func}],
put(test_server_loc, Where),
+
FWInitResult = test_server_sup:framework_call(init_tc,[Mod,Func,Args0],
{ok,Args0}),
set_tc_state(running),
@@ -924,7 +925,7 @@ run_test_case_eval(Mod, Func, Args0, Name, Ref, RunInit,
run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback);
Error = {error,_Reason} ->
NewResult = do_end_tc_call(Mod,Func, {Error,Args0},
- {skip,{failed,Error}}),
+ {auto_skip,{failed,Error}}),
{{0,NewResult},Where,[]};
{fail,Reason} ->
Conf = [{tc_status,{failed,Reason}} | hd(Args0)],
@@ -935,9 +936,9 @@ run_test_case_eval(Mod, Func, Args0, Name, Ref, RunInit,
Skip = {skip,_Reason} ->
NewResult = do_end_tc_call(Mod,Func, {Skip,Args0}, Skip),
{{0,NewResult},Where,[]};
- {auto_skip,Reason} ->
- NewResult = do_end_tc_call(Mod,Func, {{skip,Reason},Args0},
- {skip,Reason}),
+ AutoSkip = {auto_skip,_Reason} ->
+ %% special case where a conf case "pretends" to be skipped
+ NewResult = do_end_tc_call(Mod,Func, {AutoSkip,Args0}, AutoSkip),
{{0,NewResult},Where,[]}
end,
exit({Ref,Time,Value,Loc,Opts}).
@@ -955,7 +956,8 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) ->
{{0,NewRes},Line,[]};
{skip_and_save,Reason,SaveCfg} ->
Line = get_loc(),
- Conf = [{tc_status,{skipped,Reason}},{save_config,SaveCfg}|hd(Args)],
+ Conf = [{tc_status,{skipped,Reason}},
+ {save_config,SaveCfg}|hd(Args)],
NewRes = do_end_tc_call(Mod,Func, {{skip,Reason},[Conf]},
{skip,Reason}),
{{0,NewRes},Line,[]};
diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl
index d0f31af198..352e58f91c 100644
--- a/lib/test_server/src/test_server_ctrl.erl
+++ b/lib/test_server/src/test_server_ctrl.erl
@@ -1161,8 +1161,11 @@ init_tester(Mod, Func, Args, Dir, Name, {_,_,MinLev}=Levels,
end,
{SkippedN,SkipStr} =
case get(test_server_skipped) of
- {0,_} -> {0,""};
- {Skipped,_} -> {Skipped,io_lib:format(", ~w Skipped", [Skipped])}
+ {0,0} ->
+ {0,""};
+ {USkipped,ASkipped} ->
+ Skipped = USkipped+ASkipped,
+ {Skipped,io_lib:format(", ~w Skipped", [Skipped])}
end,
OkN = get(test_server_ok),
FailedN = get(test_server_failed),
@@ -1402,10 +1405,23 @@ remove_conf([{conf, _Ref, Props, _MF}|Cases], NoConf, Repeats) ->
end;
remove_conf([{make,_Ref,_MF}|Cases], NoConf, Repeats) ->
remove_conf(Cases, NoConf, Repeats);
+remove_conf([{skip_case,{{_M,all},_Cmt}}|Cases], NoConf, Repeats) ->
+ remove_conf(Cases, NoConf, Repeats);
remove_conf([{skip_case,{Type,_Ref,_MF,_Cmt}}|Cases],
NoConf, Repeats) when Type==conf;
Type==make ->
remove_conf(Cases, NoConf, Repeats);
+remove_conf([{skip_case,{Type,_Ref,_MF,_Cmt},_Mode}|Cases],
+ NoConf, Repeats) when Type==conf;
+ Type==make ->
+ remove_conf(Cases, NoConf, Repeats);
+remove_conf([C={Mod,error_in_suite,_}|Cases], NoConf, Repeats) ->
+ FwMod = get_fw_mod(?MODULE),
+ if Mod == FwMod ->
+ remove_conf(Cases, NoConf, Repeats);
+ true ->
+ remove_conf(Cases, [C|NoConf], Repeats)
+ end;
remove_conf([C|Cases], NoConf, Repeats) ->
remove_conf(Cases, [C|NoConf], Repeats);
remove_conf([], NoConf, true) ->
@@ -1413,6 +1429,11 @@ remove_conf([], NoConf, true) ->
remove_conf([], NoConf, false) ->
lists:reverse(NoConf).
+get_suites([{skip_case,{{Mod,_Func},_Cmt}}|Tests], Mods) when is_atom(Mod) ->
+ case add_mod(Mod, Mods) of
+ true -> get_suites(Tests, [Mod|Mods]);
+ false -> get_suites(Tests, Mods)
+ end;
get_suites([{Mod,_Case}|Tests], Mods) when is_atom(Mod) ->
case add_mod(Mod, Mods) of
true -> get_suites(Tests, [Mod|Mods]);
@@ -1478,8 +1499,10 @@ do_test_cases(TopCases, SkipCases,
end,
put(test_server_cases, N),
put(test_server_case_num, 0),
+
TestSpec =
add_init_and_end_per_suite(TestSpec0, undefined, undefined, FwMod),
+
TI = get_target_info(),
print(1, "Starting test~ts",
[print_if_known(N, {", ~w test cases",[N]},
@@ -1795,23 +1818,40 @@ downcase([], Result) ->
%%
%% Errors are silently ignored.
-html_convert_modules(TestSpec, _Config) ->
- Mods = html_isolate_modules(TestSpec),
+html_convert_modules(TestSpec, _Config, FwMod) ->
+ Mods = html_isolate_modules(TestSpec, FwMod),
html_convert_modules(Mods),
copy_html_files(get(test_server_dir), get(test_server_log_dir_base)).
%% Retrieve a list of modules out of the test spec.
-html_isolate_modules(List) -> html_isolate_modules(List, sets:new()).
-
-html_isolate_modules([], Set) -> sets:to_list(Set);
-html_isolate_modules([{skip_case,_}|Cases], Set) ->
- html_isolate_modules(Cases, Set);
-html_isolate_modules([{conf,_Ref,_Props,{Mod,_Func}}|Cases], Set) ->
- html_isolate_modules(Cases, sets:add_element(Mod, Set));
-html_isolate_modules([{Mod,_Case}|Cases], Set) ->
- html_isolate_modules(Cases, sets:add_element(Mod, Set));
-html_isolate_modules([{Mod,_Case,_Args}|Cases], Set) ->
- html_isolate_modules(Cases, sets:add_element(Mod, Set)).
+html_isolate_modules(List, FwMod) ->
+ html_isolate_modules(List, sets:new(), FwMod).
+
+html_isolate_modules([], Set, _) -> sets:to_list(Set);
+html_isolate_modules([{skip_case,_}|Cases], Set, FwMod) ->
+ html_isolate_modules(Cases, Set, FwMod);
+html_isolate_modules([{conf,_Ref,Props,{FwMod,_Func}}|Cases], Set, FwMod) ->
+ Set1 = case proplists:get_value(suite, Props) of
+ undefined -> Set;
+ Mod -> sets:add_element(Mod, Set)
+ end,
+ html_isolate_modules(Cases, Set1, FwMod);
+html_isolate_modules([{conf,_Ref,_Props,{Mod,_Func}}|Cases], Set, FwMod) ->
+ html_isolate_modules(Cases, sets:add_element(Mod, Set), FwMod);
+html_isolate_modules([{skip_case,{conf,_Ref,{FwMod,_Func},_Cmt},Mode}|Cases],
+ Set, FwMod) ->
+ Set1 = case proplists:get_value(suite, get_props(Mode)) of
+ undefined -> Set;
+ Mod -> sets:add_element(Mod, Set)
+ end,
+ html_isolate_modules(Cases, Set1, FwMod);
+html_isolate_modules([{skip_case,{conf,_Ref,{Mod,_Func},_Cmt},_Props}|Cases],
+ Set, FwMod) ->
+ html_isolate_modules(Cases, sets:add_element(Mod, Set), FwMod);
+html_isolate_modules([{Mod,_Case}|Cases], Set, FwMod) ->
+ html_isolate_modules(Cases, sets:add_element(Mod, Set), FwMod);
+html_isolate_modules([{Mod,_Case,_Args}|Cases], Set, FwMod) ->
+ html_isolate_modules(Cases, sets:add_element(Mod, Set), FwMod).
%% Given a list of modules, convert each module's source code to HTML.
html_convert_modules([Mod|Mods]) ->
@@ -1902,13 +1942,16 @@ add_init_and_end_per_suite([{skip_case,{{Mod,_},_}}=Case|Cases], LastMod,
{PreCases, NextMod, NextRef} =
do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod),
PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)];
+add_init_and_end_per_suite([{skip_case,{conf,_,{Mod,_},_},_}=Case|Cases], LastMod,
+ LastRef, FwMod) when Mod =/= LastMod ->
+ {PreCases, NextMod, NextRef} =
+ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod),
+ PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)];
add_init_and_end_per_suite([{skip_case,{conf,_,{Mod,_},_}}=Case|Cases], LastMod,
LastRef, FwMod) when Mod =/= LastMod ->
{PreCases, NextMod, NextRef} =
do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod),
PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)];
-add_init_and_end_per_suite([{skip_case,_}=Case|Cases], LastMod, LastRef, FwMod) ->
- [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)];
add_init_and_end_per_suite([{conf,Ref,Props,{FwMod,Func}}=Case|Cases], LastMod,
LastRef, FwMod) ->
%% if Mod == FwMod, this conf test is (probably) a test case group where
@@ -1918,7 +1961,8 @@ add_init_and_end_per_suite([{conf,Ref,Props,{FwMod,Func}}=Case|Cases], LastMod,
Suite when Suite =/= undefined, Suite =/= LastMod ->
{PreCases, NextMod, NextRef} =
do_add_init_and_end_per_suite(LastMod, LastRef, Suite, FwMod),
- Case1 = {conf,Ref,proplists:delete(suite,Props),{FwMod,Func}},
+ Case1 = {conf,Ref,[{suite,NextMod}|proplists:delete(suite,Props)],
+ {FwMod,Func}},
PreCases ++ [Case1|add_init_and_end_per_suite(Cases, NextMod,
NextRef, FwMod)];
_ ->
@@ -1929,6 +1973,9 @@ add_init_and_end_per_suite([{conf,_,_,{Mod,_}}=Case|Cases], LastMod,
{PreCases, NextMod, NextRef} =
do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod),
PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)];
+add_init_and_end_per_suite([SkipCase|Cases], LastMod, LastRef, FwMod)
+ when element(1,SkipCase) == skip_case ->
+ [SkipCase|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)];
add_init_and_end_per_suite([{conf,_,_,_}=Case|Cases], LastMod, LastRef, FwMod) ->
[Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)];
add_init_and_end_per_suite([{Mod,_}=Case|Cases], LastMod, LastRef, FwMod)
@@ -2043,7 +2090,8 @@ run_test_cases(TestSpec, Config, TimetrapData) ->
true ->
ok;
false ->
- html_convert_modules(TestSpec, Config)
+ FwMod = get_fw_mod(?MODULE),
+ html_convert_modules(TestSpec, Config, FwMod)
end,
run_test_cases_loop(TestSpec, [Config], TimetrapData, [], []),
@@ -2209,34 +2257,50 @@ run_test_cases(TestSpec, Config, TimetrapData) ->
%% group1_end | --->
%%
-run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases],
- Config, TimetrapData, Mode, Status) when Type==conf;
- Type==make ->
+run_test_cases_loop([{SkipTag,CaseData={Type,_Ref,_Case,_Comment}}|Cases],
+ Config, TimetrapData, Mode, Status) when
+ ((SkipTag==auto_skip_case) or (SkipTag==skip_case)) and
+ ((Type==conf) or (Type==make)) ->
+ run_test_cases_loop([{SkipTag,CaseData,Mode}|Cases],
+ Config, TimetrapData, Mode, Status);
+
+run_test_cases_loop([{SkipTag,{Type,Ref,Case,Comment},SkipMode}|Cases],
+ Config, TimetrapData, Mode, Status) when
+ ((SkipTag==auto_skip_case) or (SkipTag==skip_case)) and
+ ((Type==conf) or (Type==make)) ->
file:set_cwd(filename:dirname(get(test_server_dir))),
CurrIOHandler = get(test_server_common_io_handler),
ParentMode = tl(Mode),
+ {AutoOrUser,ReportTag} =
+ if SkipTag == auto_skip_case -> {auto,tc_auto_skip};
+ SkipTag == skip_case -> {user,tc_user_skip}
+ end,
+
%% check and update the mode for test case execution and io msg handling
case {curr_ref(Mode),check_props(parallel, Mode)} of
{Ref,Ref} ->
case check_props(parallel, ParentMode) of
false ->
- %% this is a skipped end conf for a top level parallel group,
- %% buffered io can be flushed
+ %% this is a skipped end conf for a top level parallel
+ %% group, buffered io can be flushed
handle_test_case_io_and_status(),
set_io_buffering(undefined),
- {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, false, SkipMode),
- test_server_sup:framework_call(report, [tc_auto_skip,
- {Mod,Func,Comment}]),
+ {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment,
+ false, SkipMode),
+ ConfData = {Mod,{Func,get_name(SkipMode)},Comment},
+ test_server_sup:framework_call(report,
+ [ReportTag,ConfData]),
run_test_cases_loop(Cases, Config, TimetrapData, ParentMode,
delete_status(Ref, Status));
_ ->
- %% this is a skipped end conf for a parallel group nested under a
- %% parallel group (io buffering is active)
+ %% this is a skipped end conf for a parallel group nested
+ %% under a parallel group (io buffering is active)
wait_for_cases(Ref),
- {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, true, SkipMode),
- test_server_sup:framework_call(report, [tc_auto_skip,
- {Mod,Func,Comment}]),
+ {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment,
+ true, SkipMode),
+ ConfData = {Mod,{Func,get_name(SkipMode)},Comment},
+ test_server_sup:framework_call(report, [ReportTag,ConfData]),
case CurrIOHandler of
{Ref,_} ->
%% current_io_handler was set by start conf of this
@@ -2246,18 +2310,21 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases],
_ ->
ok
end,
- run_test_cases_loop(Cases, Config, TimetrapData, ParentMode,
+ run_test_cases_loop(Cases, Config,
+ TimetrapData, ParentMode,
delete_status(Ref, Status))
end;
{Ref,false} ->
%% this is a skipped end conf for a non-parallel group that's not
%% nested under a parallel group
- {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, false, SkipMode),
- test_server_sup:framework_call(report, [tc_auto_skip,{Mod,Func,Comment}]),
-
- %% Check if this group is auto skipped because of error in the init conf.
- %% If so, check if the parent group is a sequence, and if it is, skip
- %% all proceeding tests in that group.
+ {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment,
+ false, SkipMode),
+ ConfData = {Mod,{Func,get_name(SkipMode)},Comment},
+ test_server_sup:framework_call(report, [ReportTag,ConfData]),
+
+ %% Check if this group is auto skipped because of error in the
+ %% init conf. If so, check if the parent group is a sequence,
+ %% and if it is, skip all proceeding tests in that group.
GrName = get_name(Mode),
Cases1 =
case get_tc_results(Status) of
@@ -2270,7 +2337,8 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases],
ParentRef ->
Reason = {group_result,GrName,failed},
skip_cases_upto(ParentRef, Cases,
- Reason, tc, Mode)
+ Reason, tc, Mode,
+ SkipTag)
end;
false ->
Cases
@@ -2283,8 +2351,10 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases],
{Ref,_} ->
%% this is a skipped end conf for a non-parallel group nested under
%% a parallel group (io buffering is active)
- {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, true, SkipMode),
- test_server_sup:framework_call(report, [tc_auto_skip,{Mod,Func,Comment}]),
+ {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment,
+ true, SkipMode),
+ ConfData = {Mod,{Func,get_name(SkipMode)},Comment},
+ test_server_sup:framework_call(report, [ReportTag,ConfData]),
case CurrIOHandler of
{Ref,_} ->
%% current_io_handler was set by start conf of this
@@ -2299,20 +2369,27 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases],
{_,false} ->
%% this is a skipped start conf for a group which is not nested
%% under a parallel group
- {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, false, SkipMode),
- test_server_sup:framework_call(report, [tc_auto_skip,{Mod,Func,Comment}]),
- run_test_cases_loop(Cases, Config, TimetrapData, [conf(Ref,[])|Mode], Status);
+ {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment,
+ false, SkipMode),
+ ConfData = {Mod,{Func,get_name(SkipMode)},Comment},
+ test_server_sup:framework_call(report, [ReportTag,ConfData]),
+ run_test_cases_loop(Cases, Config, TimetrapData,
+ [conf(Ref,[])|Mode], Status);
{_,Ref0} when is_reference(Ref0) ->
- %% this is a skipped start conf for a group nested under a parallel group
- %% and if this is the first nested group, io buffering must be activated
+ %% this is a skipped start conf for a group nested under a parallel
+ %% group and if this is the first nested group, io buffering must
+ %% be activated
if CurrIOHandler == undefined ->
set_io_buffering({Ref,self()});
true ->
ok
end,
- {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, true, SkipMode),
- test_server_sup:framework_call(report, [tc_auto_skip,{Mod,Func,Comment}]),
- run_test_cases_loop(Cases, Config, TimetrapData, [conf(Ref,[])|Mode], Status)
+ {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment,
+ true, SkipMode),
+ ConfData = {Mod,{Func,get_name(SkipMode)},Comment},
+ test_server_sup:framework_call(report, [ReportTag,ConfData]),
+ run_test_cases_loop(Cases, Config, TimetrapData,
+ [conf(Ref,[])|Mode], Status)
end;
run_test_cases_loop([{auto_skip_case,{Case,Comment},SkipMode}|Cases],
@@ -2323,21 +2400,12 @@ run_test_cases_loop([{auto_skip_case,{Case,Comment},SkipMode}|Cases],
run_test_cases_loop(Cases, Config, TimetrapData, Mode,
update_status(skipped, Mod, Func, Status));
-run_test_cases_loop([{skip_case,{conf,Ref,Case,Comment}}|Cases0],
+run_test_cases_loop([{skip_case,{{Mod,all}=Case,Comment}}|Cases],
Config, TimetrapData, Mode, Status) ->
- {Mod,Func} = skip_case(user, Ref, 0, Case, Comment, is_io_buffered()),
- {Cases,Config1} =
- case curr_ref(Mode) of
- Ref ->
- %% skipped end conf
- {Cases0,tl(Config)};
- _ ->
- %% skipped start conf
- {skip_cases_upto(Ref, Cases0, Comment, conf, Mode),Config}
- end,
- test_server_sup:framework_call(report, [tc_user_skip,{Mod,Func,Comment}]),
- run_test_cases_loop(Cases, Config1, TimetrapData, Mode,
- update_status(skipped, Mod, Func, Status));
+ skip_case(user, undefined, 0, Case, Comment, false, Mode),
+ test_server_sup:framework_call(report, [tc_user_skip,
+ {Mod,all,Comment}]),
+ run_test_cases_loop(Cases, Config, TimetrapData, Mode, Status);
run_test_cases_loop([{skip_case,{Case,Comment}}|Cases],
Config, TimetrapData, Mode, Status) ->
@@ -2597,7 +2665,8 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0,
"~n*** ~w returned bad elements in Config: ~p.~n",
[Func,Bad]),
Reason = {failed,{Mod,init_per_suite,bad_return}},
- Cases2 = skip_cases_upto(Ref, Cases, Reason, conf, CurrMode),
+ Cases2 = skip_cases_upto(Ref, Cases, Reason, conf, CurrMode,
+ auto_skip_case),
set_io_buffering(IOHandler),
stop_minor_log_file(),
run_test_cases_loop(Cases2, Config, TimetrapData, Mode,
@@ -2623,7 +2692,8 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0,
print(minor, "~n*** ~w failed.~n"
" Skipping all cases.", [Func]),
Reason = {failed,{Mod,Func,Fail}},
- {skip_cases_upto(Ref, Cases, Reason, conf, CurrMode),
+ {skip_cases_upto(Ref, Cases, Reason, conf, CurrMode,
+ auto_skip_case),
Config,
update_status(failed, group_result, get_name(Mode),
delete_status(Ref, Status2))};
@@ -2635,14 +2705,37 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0,
set_io_buffering(IOHandler),
stop_minor_log_file(),
run_test_cases_loop(Cases2, Config1, TimetrapData, Mode, Status3);
+
+ {_,{auto_skip,SkipReason},_} ->
+ %% this case can only happen if the framework (not the user)
+ %% decides to skip execution of a conf function
+ {Cases2,Config1,Status3} =
+ if StartConf ->
+ ReportAbortRepeat(auto_skipped),
+ print(minor, "~n*** ~w auto skipped.~n"
+ " Skipping all cases.", [Func]),
+ {skip_cases_upto(Ref, Cases, SkipReason, conf, CurrMode,
+ auto_skip_case),
+ Config,
+ delete_status(Ref, Status2)};
+ not StartConf ->
+ ReportRepeatStop(),
+ print_conf_time(ConfTime),
+ {Cases,tl(Config),delete_status(Ref, Status2)}
+ end,
+ set_io_buffering(IOHandler),
+ stop_minor_log_file(),
+ run_test_cases_loop(Cases2, Config1, TimetrapData, Mode, Status3);
+
{_,{Skip,Reason},_} when StartConf and ((Skip==skip) or (Skip==skipped)) ->
ReportAbortRepeat(skipped),
print(minor, "~n*** ~w skipped.~n"
" Skipping all cases.", [Func]),
set_io_buffering(IOHandler),
stop_minor_log_file(),
- run_test_cases_loop(skip_cases_upto(Ref, Cases, Reason, conf, CurrMode),
- Config, TimetrapData, Mode,
+ run_test_cases_loop(skip_cases_upto(Ref, Cases, Reason, conf,
+ CurrMode, skip_case),
+ [hd(Config)|Config], TimetrapData, Mode,
delete_status(Ref, Status2));
{_,{skip_and_save,Reason,_SavedConfig},_} when StartConf ->
ReportAbortRepeat(skipped),
@@ -2650,13 +2743,15 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0,
" Skipping all cases.", [Func]),
set_io_buffering(IOHandler),
stop_minor_log_file(),
- run_test_cases_loop(skip_cases_upto(Ref, Cases, Reason, conf, CurrMode),
- Config, TimetrapData, Mode,
+ run_test_cases_loop(skip_cases_upto(Ref, Cases, Reason, conf,
+ CurrMode, skip_case),
+ [hd(Config)|Config], TimetrapData, Mode,
delete_status(Ref, Status2));
{_,_Other,_} when Func == init_per_suite ->
print(minor, "~n*** init_per_suite failed to return a Config list.~n", []),
Reason = {failed,{Mod,init_per_suite,bad_return}},
- Cases2 = skip_cases_upto(Ref, Cases, Reason, conf, CurrMode),
+ Cases2 = skip_cases_upto(Ref, Cases, Reason, conf, CurrMode,
+ auto_skip_case),
set_io_buffering(IOHandler),
stop_minor_log_file(),
run_test_cases_loop(Cases2, Config, TimetrapData, Mode,
@@ -2668,7 +2763,6 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0,
stop_minor_log_file(),
run_test_cases_loop(Cases, [hd(Config)|Config], TimetrapData,
Mode, Status2);
-
{_,_EndConfRetVal,Opts} ->
%% Check if return_group_result is set (ok, skipped or failed) and
%% if so:
@@ -2683,7 +2777,8 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0,
case {curr_ref(Mode),check_prop(sequence, Mode)} of
{ParentRef,ParentRef} ->
Reason = {group_result,GrName,failed},
- {skip_cases_upto(ParentRef, Cases, Reason, tc, Mode),
+ {skip_cases_upto(ParentRef, Cases, Reason, tc,
+ Mode, auto_skip_case),
update_status(failed, group_result, GrName,
delete_status(Ref, Status2))};
_ ->
@@ -2701,16 +2796,19 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0,
ReportRepeatStop(),
set_io_buffering(IOHandler),
stop_minor_log_file(),
- run_test_cases_loop(Cases2, tl(Config), TimetrapData, Mode, Status3)
+ run_test_cases_loop(Cases2, tl(Config), TimetrapData,
+ Mode, Status3)
end;
-run_test_cases_loop([{make,Ref,{Mod,Func,Args}}|Cases0], Config, TimetrapData, Mode, Status) ->
+run_test_cases_loop([{make,Ref,{Mod,Func,Args}}|Cases0], Config, TimetrapData,
+ Mode, Status) ->
case run_test_case(Ref, 0, Mod, Func, Args, skip_init, TimetrapData) of
{_,Why={'EXIT',_},_} ->
print(minor, "~n*** ~w failed.~n"
" Skipping all cases.", [Func]),
Reason = {failed,{Mod,Func,Why}},
- Cases = skip_cases_upto(Ref, Cases0, Reason, conf, Mode),
+ Cases = skip_cases_upto(Ref, Cases0, Reason, conf, Mode,
+ auto_skip_case),
stop_minor_log_file(),
run_test_cases_loop(Cases, Config, TimetrapData, Mode, Status);
{_,_Whatever,_} ->
@@ -2735,7 +2833,14 @@ run_test_cases_loop([{Mod,Case}|Cases], Config, TimetrapData, Mode, Status) ->
TimetrapData, Mode, Status);
run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status) ->
- Num = put(test_server_case_num, get(test_server_case_num)+1),
+ {Num,RunInit} =
+ case FwMod = get_fw_mod(?MODULE) of
+ Mod when Func == error_in_suite ->
+ {-1,skip_init};
+ _ ->
+ {put(test_server_case_num, get(test_server_case_num)+1),
+ run_init}
+ end,
%% check the current execution mode and save info about the case if
%% detected that printouts to common log files is handled later
@@ -2750,7 +2855,7 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status)
end,
case run_test_case(undefined, Num+1, Mod, Func, Args,
- run_init, TimetrapData, Mode) of
+ RunInit, TimetrapData, Mode) of
%% callback to framework module failed, exit immediately
{_,{framework_error,{FwMod,FwFunc},Reason},_} ->
print(minor, "~n*** ~w failed in ~w. Reason: ~p~n",
@@ -2791,7 +2896,8 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status)
" Skipping all other cases in sequence.",
[Func]),
Reason = {failed,{Mod,Func}},
- Cases2 = skip_cases_upto(Ref, Cases, Reason, tc, Mode),
+ Cases2 = skip_cases_upto(Ref, Cases, Reason, tc,
+ Mode, auto_skip_case),
stop_minor_log_file(),
run_test_cases_loop(Cases2, Config, TimetrapData, Mode, Status1)
end
@@ -3012,13 +3118,13 @@ cases_to_shuffle(Ref, Cases) ->
cases_to_shuffle(Ref, [{conf,Ref,_,_} | _]=Cs, N, Ix) -> % end
{N-1,Ix,Cs};
-cases_to_shuffle(Ref, [{skip_case,{_,Ref,_,_}} | _]=Cs, N, Ix) -> % end
+cases_to_shuffle(Ref, [{skip_case,{_,Ref,_,_},_} | _]=Cs, N, Ix) -> % end
{N-1,Ix,Cs};
cases_to_shuffle(Ref, [{conf,Ref1,_,_}=C | Cs], N, Ix) -> % nested group
{Cs1,Rest} = get_subcases(Ref1, Cs, []),
cases_to_shuffle(Ref, Rest, N+1, [{N,[C|Cs1]} | Ix]);
-cases_to_shuffle(Ref, [{skip_case,{_,Ref1,_,_}}=C | Cs], N, Ix) -> % nested group
+cases_to_shuffle(Ref, [{skip_case,{_,Ref1,_,_},_}=C | Cs], N, Ix) -> % nested group
{Cs1,Rest} = get_subcases(Ref1, Cs, []),
cases_to_shuffle(Ref, Rest, N+1, [{N,[C|Cs1]} | Ix]);
@@ -3027,7 +3133,7 @@ cases_to_shuffle(Ref, [C | Cs], N, Ix) ->
get_subcases(SubRef, [{conf,SubRef,_,_}=C | Cs], SubCs) ->
{lists:reverse([C|SubCs]),Cs};
-get_subcases(SubRef, [{skip_case,{_,SubRef,_,_}}=C | Cs], SubCs) ->
+get_subcases(SubRef, [{skip_case,{_,SubRef,_,_},_}=C | Cs], SubCs) ->
{lists:reverse([C|SubCs]),Cs};
get_subcases(SubRef, [C|Cs], SubCs) ->
get_subcases(SubRef, Cs, [C|SubCs]).
@@ -3075,13 +3181,27 @@ skip_case1(Type, CaseNum, Mod, Func, Comment, Mode) ->
ResultCol = if Type == auto -> ?auto_skip_color;
Type == user -> ?user_skip_color
end,
-
- Comment1 = reason_to_string(Comment),
-
print(major, "~n=case ~w:~w", [Mod,Func]),
- print(major, "=started ~s", [lists:flatten(timestamp_get(""))]),
- print(major, "=result skipped: ~ts", [Comment1]),
- print(2,"*** Skipping test case #~w ~w ***", [CaseNum,{Mod,Func}]),
+ GroupName = case get_name(Mode) of
+ undefined ->
+ "";
+ GrName ->
+ GrName1 = cast_to_list(GrName),
+ print(major, "=group_props ~p", [[{name,GrName1}]]),
+ GrName1
+ end,
+ print(major, "=started ~s", [lists:flatten(timestamp_get(""))]),
+ Comment1 = reason_to_string(Comment),
+ if Type == auto ->
+ print(major, "=result auto_skipped: ~ts", [Comment1]);
+ Type == user ->
+ print(major, "=result skipped: ~ts", [Comment1])
+ end,
+ if CaseNum == 0 ->
+ print(2,"*** Skipping ~w ***", [{Mod,Func}]);
+ true ->
+ print(2,"*** Skipping test case #~w ~w ***", [CaseNum,{Mod,Func}])
+ end,
TR = xhtml("<tr valign=\"top\">", ["<tr class=\"",odd_or_even(),"\">"]),
GroupName = case get_name(Mode) of
undefined -> "";
@@ -3097,6 +3217,7 @@ skip_case1(Type, CaseNum, Mod, Func, Comment, Mode) ->
"<td><font color=\"~ts\">SKIPPED</font></td>"
"<td>~ts</td></tr>\n",
[num2str(CaseNum),fw_name(Mod),GroupName,Func,ResultCol,Comment1]),
+
if CaseNum > 0 ->
{US,AS} = get(test_server_skipped),
case Type of
@@ -3110,12 +3231,14 @@ skip_case1(Type, CaseNum, Mod, Func, Comment, Mode) ->
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%% skip_cases_upto(Ref, Cases, Reason, Origin, Mode) -> Cases1
+%% skip_cases_upto(Ref, Cases, Reason, Origin, Mode, SkipType) -> Cases1
%%
+%% SkipType = skip_case | auto_skip_case
%% Mark all cases tagged with Ref as skipped.
-skip_cases_upto(Ref, Cases, Reason, Origin, Mode) ->
- {_,Modified,Rest} = modify_cases_upto(Ref, {skip,Reason,Origin,Mode}, Cases),
+skip_cases_upto(Ref, Cases, Reason, Origin, Mode, SkipType) ->
+ {_,Modified,Rest} =
+ modify_cases_upto(Ref, {skip,Reason,Origin,Mode,SkipType}, Cases),
Modified++Rest.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -3151,6 +3274,7 @@ modify_cases_upto(Ref, ModOp, Cases, Orig, Alt) ->
%% same ref in the list, if not, this *is* an end conf case
case lists:any(fun({_,R,_,_}) when R == Ref -> true;
({_,R,_}) when R == Ref -> true;
+ ({skip_case,{_,R,_,_},_}) when R == Ref -> true;
({skip_case,{_,R,_,_}}) when R == Ref -> true;
(_) -> false
end, Cases) of
@@ -3161,25 +3285,39 @@ modify_cases_upto(Ref, ModOp, Cases, Orig, Alt) ->
end.
%% next case is a conf with same ref, must be end conf = we're done
-modify_cases_upto1(Ref, {skip,Reason,conf,Mode}, [{conf,Ref,_Props,MF}|T], Orig, Alt) ->
+modify_cases_upto1(Ref, {skip,Reason,conf,Mode,skip_case},
+ [{conf,Ref,_Props,MF}|T], Orig, Alt) ->
+ {Orig,[{skip_case,{conf,Ref,MF,Reason},Mode}|Alt],T};
+modify_cases_upto1(Ref, {skip,Reason,conf,Mode,auto_skip_case},
+ [{conf,Ref,_Props,MF}|T], Orig, Alt) ->
{Orig,[{auto_skip_case,{conf,Ref,MF,Reason},Mode}|Alt],T};
modify_cases_upto1(Ref, {copy,NewRef}, [{conf,Ref,Props,MF}=C|T], Orig, Alt) ->
{[C|Orig],[{conf,NewRef,update_repeat(Props),MF}|Alt],T};
%% we've skipped all remaining cases in a sequence
-modify_cases_upto1(Ref, {skip,_,tc,_}, [{conf,Ref,_Props,_MF}|_]=Cs, Orig, Alt) ->
+modify_cases_upto1(Ref, {skip,_,tc,_,_},
+ [{conf,Ref,_Props,_MF}|_]=Cs, Orig, Alt) ->
{Orig,Alt,Cs};
%% next is a make case
-modify_cases_upto1(Ref, {skip,Reason,_,Mode}, [{make,Ref,MF}|T], Orig, Alt) ->
- {Orig,[{auto_skip_case,{make,Ref,MF,Reason},Mode}|Alt],T};
+modify_cases_upto1(Ref, {skip,Reason,_,Mode,SkipType},
+ [{make,Ref,MF}|T], Orig, Alt) ->
+ {Orig,[{SkipType,{make,Ref,MF,Reason},Mode}|Alt],T};
modify_cases_upto1(Ref, {copy,NewRef}, [{make,Ref,MF}=M|T], Orig, Alt) ->
{[M|Orig],[{make,NewRef,MF}|Alt],T};
%% next case is a user skipped end conf with the same ref = we're done
-modify_cases_upto1(Ref, {skip,Reason,_,Mode}, [{skip_case,{Type,Ref,MF,_Cmt}}|T], Orig, Alt) ->
- {Orig,[{auto_skip_case,{Type,Ref,MF,Reason},Mode}|Alt],T};
-modify_cases_upto1(Ref, {copy,NewRef}, [{skip_case,{Type,Ref,MF,Cmt}}=C|T], Orig, Alt) ->
+modify_cases_upto1(Ref, {skip,Reason,_,Mode,SkipType},
+ [{skip_case,{Type,Ref,MF,_Cmt},_}|T], Orig, Alt) ->
+ {Orig,[{SkipType,{Type,Ref,MF,Reason},Mode}|Alt],T};
+modify_cases_upto1(Ref, {skip,Reason,_,Mode,SkipType},
+ [{skip_case,{Type,Ref,MF,_Cmt}}|T], Orig, Alt) ->
+ {Orig,[{SkipType,{Type,Ref,MF,Reason},Mode}|Alt],T};
+modify_cases_upto1(Ref, {copy,NewRef},
+ [{skip_case,{Type,Ref,MF,Cmt},Mode}=C|T], Orig, Alt) ->
+ {[C|Orig],[{skip_case,{Type,NewRef,MF,Cmt},Mode}|Alt],T};
+modify_cases_upto1(Ref, {copy,NewRef},
+ [{skip_case,{Type,Ref,MF,Cmt}}=C|T], Orig, Alt) ->
{[C|Orig],[{skip_case,{Type,NewRef,MF,Cmt}}|Alt],T};
%% next is a skip_case, could be one test case or 'all' in suite, we must proceed
@@ -3187,13 +3325,17 @@ modify_cases_upto1(Ref, ModOp, [{skip_case,{_F,_Cmt}}=MF|T], Orig, Alt) ->
modify_cases_upto1(Ref, ModOp, T, [MF|Orig], [MF|Alt]);
%% next is a normal case (possibly in a sequence), mark as skipped, or copy, and proceed
-modify_cases_upto1(Ref, {skip,Reason,_,Mode}=Op, [{_M,_F}=MF|T], Orig, Alt) ->
+modify_cases_upto1(Ref, {skip,Reason,_,_,skip_case}=Op,
+ [{_M,_F}=MF|T], Orig, Alt) ->
+ modify_cases_upto1(Ref, Op, T, Orig, [{skip_case,{MF,Reason}}|Alt]);
+modify_cases_upto1(Ref, {skip,Reason,_,Mode,auto_skip_case}=Op,
+ [{_M,_F}=MF|T], Orig, Alt) ->
modify_cases_upto1(Ref, Op, T, Orig, [{auto_skip_case,{MF,Reason},Mode}|Alt]);
modify_cases_upto1(Ref, CopyOp, [{_M,_F}=MF|T], Orig, Alt) ->
modify_cases_upto1(Ref, CopyOp, T, [MF|Orig], [MF|Alt]);
%% next is some other case, ignore or copy
-modify_cases_upto1(Ref, {skip,_,_,_}=Op, [_|T], Orig, Alt) ->
+modify_cases_upto1(Ref, {skip,_,_,_,_}=Op, [_|T], Orig, Alt) ->
modify_cases_upto1(Ref, Op, T, Orig, Alt);
modify_cases_upto1(Ref, CopyOp, [C|T], Orig, Alt) ->
modify_cases_upto1(Ref, CopyOp, T, [C|Orig], [C|Alt]).
@@ -3567,7 +3709,8 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit,
{died,Reason} ->
progress(failed, Num, Mod, Func, Loc, Reason,
Time, Comment, Style);
- {_,{'EXIT',{Skip,Reason}}} when Skip==skip; Skip==skipped ->
+ {_,{'EXIT',{Skip,Reason}}} when Skip==skip; Skip==skipped;
+ Skip==auto_skip ->
progress(skip, Num, Mod, Func, Loc, Reason,
Time, Comment, Style);
{_,{'EXIT',_Pid,{Skip,Reason}}} when Skip==skip; Skip==skipped ->
@@ -3579,10 +3722,13 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit,
{_,{'EXIT',Reason}} ->
progress(failed, Num, Mod, Func, Loc, Reason,
Time, Comment, Style);
- {_, {Fail, Reason}} when Fail =:= fail; Fail =:= failed ->
+ {_,{Fail,Reason}} when Fail =:= fail; Fail =:= failed ->
progress(failed, Num, Mod, Func, Loc, Reason,
Time, Comment, Style);
- {_, {Skip, Reason}} when Skip==skip; Skip==skipped ->
+ {_,Reason={auto_skip,_Why}} ->
+ progress(skip, Num, Mod, Func, Loc, Reason,
+ Time, Comment, Style);
+ {_,{Skip,Reason}} when Skip==skip; Skip==skipped ->
progress(skip, Num, Mod, Func, Loc, Reason,
Time, Comment, Style);
{Time,RetVal} ->
@@ -3699,15 +3845,15 @@ num2str(N) -> integer_to_list(N).
progress(skip, CaseNum, Mod, Func, Loc, Reason, Time,
Comment, {St0,St1}) ->
- {Reason1,{Color,Ret}} =
+ {Reason1,{Color,Ret,ReportTag}} =
if_auto_skip(Reason,
- fun() -> {?auto_skip_color,auto_skip} end,
- fun() -> {?user_skip_color,skip} end),
- print(major, "=result skipped", []),
- print(1, "*** SKIPPED *** ~ts",
- [get_info_str(Func, CaseNum, get(test_server_cases))]),
+ fun() -> {?auto_skip_color,auto_skip,auto_skipped} end,
+ fun() -> {?user_skip_color,skip,skipped} end),
+ print(major, "=result ~w: ~p", [ReportTag,Reason1]),
+ print(1, "*** SKIPPED ~ts ***",
+ [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]),
test_server_sup:framework_call(report, [tc_done,{Mod,Func,
- {skipped,Reason1}}]),
+ {ReportTag,Reason1}}]),
ReasonStr = reason_to_string(Reason1),
ReasonStr1 = lists:flatten([string:strip(S,left) ||
S <- string:tokens(ReasonStr,[$\n])]),
@@ -3734,8 +3880,8 @@ progress(skip, CaseNum, Mod, Func, Loc, Reason, Time,
progress(failed, CaseNum, Mod, Func, Loc, timetrap_timeout, T,
Comment0, {St0,St1}) ->
print(major, "=result failed: timeout, ~p", [Loc]),
- print(1, "*** FAILED *** ~ts",
- [get_info_str(Func, CaseNum, get(test_server_cases))]),
+ print(1, "*** FAILED ~ts ***",
+ [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]),
test_server_sup:framework_call(report,
[tc_done,{Mod,Func,
{failed,timetrap_timeout}}]),
@@ -3760,8 +3906,8 @@ progress(failed, CaseNum, Mod, Func, Loc, timetrap_timeout, T,
progress(failed, CaseNum, Mod, Func, Loc, {testcase_aborted,Reason}, _T,
Comment0, {St0,St1}) ->
print(major, "=result failed: testcase_aborted, ~p", [Loc]),
- print(1, "*** FAILED *** ~ts",
- [get_info_str(Func, CaseNum, get(test_server_cases))]),
+ print(1, "*** FAILED ~ts ***",
+ [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]),
test_server_sup:framework_call(report,
[tc_done,{Mod,Func,
{failed,testcase_aborted}}]),
@@ -3786,8 +3932,8 @@ progress(failed, CaseNum, Mod, Func, Loc, {testcase_aborted,Reason}, _T,
progress(failed, CaseNum, Mod, Func, unknown, Reason, Time,
Comment0, {St0,St1}) ->
print(major, "=result failed: ~p, ~w", [Reason,unknown]),
- print(1, "*** FAILED *** ~ts",
- [get_info_str(Func, CaseNum, get(test_server_cases))]),
+ print(1, "*** FAILED ~ts ***",
+ [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]),
test_server_sup:framework_call(report, [tc_done,{Mod,Func,
{failed,Reason}}]),
TimeStr = io_lib:format(if is_float(Time) -> "~.3fs";
@@ -3822,8 +3968,8 @@ progress(failed, CaseNum, Mod, Func, unknown, Reason, Time,
progress(failed, CaseNum, Mod, Func, Loc, Reason, Time,
Comment0, {St0,St1}) ->
print(major, "=result failed: ~p, ~p", [Reason,Loc]),
- print(1, "*** FAILED *** ~ts",
- [get_info_str(Func, CaseNum, get(test_server_cases))]),
+ print(1, "*** FAILED ~ts ***",
+ [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]),
test_server_sup:framework_call(report, [tc_done,{Mod,Func,
{failed,Reason}}]),
TimeStr = io_lib:format(if is_float(Time) -> "~.3fs";
@@ -3920,24 +4066,25 @@ fw_name(Mod) ->
if_auto_skip(Reason={failed,{_,init_per_testcase,_}}, True, _False) ->
{Reason,True()};
-if_auto_skip({_T,{skip,Reason={failed,{_,init_per_testcase,_}}},_Opts}, True, _False) ->
+if_auto_skip({skip,Reason={failed,{_,init_per_testcase,_}}}, True, _False) ->
{Reason,True()};
-if_auto_skip({fw_auto_skip,Reason}, True, _False) ->
- {Reason,True()};
-if_auto_skip({_T,{skip,{fw_auto_skip,Reason}},_Opts}, True, _False) ->
+if_auto_skip({auto_skip,Reason}, True, _False) ->
{Reason,True()};
if_auto_skip(Reason, _True, False) ->
{Reason,False()}.
-update_skip_counters(RetVal, {US,AS}) ->
- {_,Result} = if_auto_skip(RetVal, fun() -> {US,AS+1} end, fun() -> {US+1,AS} end),
+update_skip_counters({_T,Pat,_Opts}, {US,AS}) ->
+ {_,Result} = if_auto_skip(Pat, fun() -> {US,AS+1} end, fun() -> {US+1,AS} end),
+ Result;
+update_skip_counters(Pat, {US,AS}) ->
+ {_,Result} = if_auto_skip(Pat, fun() -> {US,AS+1} end, fun() -> {US+1,AS} end),
Result.
-get_info_str(Func, 0, _Cases) ->
- atom_to_list(Func);
-get_info_str(_Func, CaseNum, unknown) ->
+get_info_str(Mod,Func, 0, _Cases) ->
+ io_lib:format("~w", [{Mod,Func}]);
+get_info_str(_Mod,_Func, CaseNum, unknown) ->
"test case " ++ integer_to_list(CaseNum);
-get_info_str(_Func, CaseNum, Cases) ->
+get_info_str(_Mod,_Func, CaseNum, Cases) ->
"test case " ++ integer_to_list(CaseNum) ++
" of " ++ integer_to_list(Cases).
@@ -4396,12 +4543,27 @@ collect_cases({conf,Props,InitMF,CaseList,FinMF} = Conf, St) ->
Props1 ->
Ref = make_ref(),
Skips = St#cc.skip,
+ Props2 = [{suite,St#cc.mod} | lists:delete(suite,Props1)],
+ Mode = [{Ref,Props2,undefined}],
case in_skip_list({St#cc.mod,Conf}, Skips) of
{true,Comment} -> % conf init skipped
- {ok,[{skip_case,{conf,Ref,InitMF,Comment}} |
+ {ok,[{skip_case,{conf,Ref,InitMF,Comment},Mode} |
[] ++ [{conf,Ref,[],FinMF}]],St};
{true,Name,Comment} when is_atom(Name) -> % all cases skipped
- {ok,[{skip_case,{{St#cc.mod,{group,Name}},Comment}}],St};
+ case collect_cases(CaseList, St) of
+ {ok,[],_St} = Empty ->
+ Empty;
+ {ok,FlatCases,St1} ->
+ Cases2Skip = FlatCases ++ [{conf,Ref,
+ keep_name(Props1),
+ FinMF}],
+ Skipped = skip_cases_upto(Ref, Cases2Skip, Comment,
+ conf, Mode, skip_case),
+ {ok,[{skip_case,{conf,Ref,InitMF,Comment},Mode} |
+ Skipped],St1};
+ {error,_Reason} = Error ->
+ Error
+ end;
{true,ToSkip,_} when is_list(ToSkip) -> % some cases skipped
case collect_cases(CaseList,
St#cc{skip=ToSkip++Skips}) of
diff --git a/lib/test_server/test/test_server_SUITE.erl b/lib/test_server/test/test_server_SUITE.erl
index 8ad5fcfb5c..c4faac036b 100644
--- a/lib/test_server/test/test_server_SUITE.erl
+++ b/lib/test_server/test/test_server_SUITE.erl
@@ -117,7 +117,7 @@ test_server_shuffle01_SUITE(Config) ->
test_server_skip_SUITE(Config) ->
run_test_server_tests("test_server_skip_SUITE", [],
- 3, 0, 1, 0, 0, 1, 3, 0, 0, Config).
+ 3, 0, 1, 0, 1, 0, 3, 0, 0, Config).
test_server_conf01_SUITE(Config) ->
run_test_server_tests("test_server_conf01_SUITE", [],
@@ -248,11 +248,13 @@ run_test_server_tests(SuiteName, Skip, NCases, NFail, NExpected, NSucc,
{NActualSkip,NActualFail,NActualSucc} =
lists:foldl(fun(#tc{ result = skip },{S,F,Su}) ->
{S+1,F,Su};
- (#tc{ result = ok },{S,F,Su}) ->
- {S,F,Su+1};
- (#tc{ result = failed },{S,F,Su}) ->
- {S,F+1,Su}
- end,{0,0,0},Data#suite.cases),
+ (#tc{ result = auto_skip },{S,F,Su}) ->
+ {S+1,F,Su};
+ (#tc{ result = ok },{S,F,Su}) ->
+ {S,F,Su+1};
+ (#tc{ result = failed },{S,F,Su}) ->
+ {S,F+1,Su}
+ end,{0,0,0},Data#suite.cases),
Data.
translate_filename(Filename,EncodingOnTestNode) ->
diff --git a/lib/test_server/test/test_server_test_lib.erl b/lib/test_server/test/test_server_test_lib.erl
index cd6804f7ad..82a702d59f 100644
--- a/lib/test_server/test/test_server_test_lib.erl
+++ b/lib/test_server/test/test_server_test_lib.erl
@@ -17,6 +17,7 @@
%% %CopyrightEnd%
%%
-module(test_server_test_lib).
+
-export([parse_suite/1]).
-export([init/2, pre_init_per_testcase/3, post_end_per_testcase/4]).
@@ -185,7 +186,9 @@ parse_case("=result" ++ Result, _, Tc) ->
"failed" ++ _ ->
{ok, Tc#tc{ result = failed } };
"skipped" ++ _ ->
- {ok, Tc#tc{ result = skip } }
+ {ok, Tc#tc{ result = skip } };
+ "auto_skipped" ++ _ ->
+ {ok, Tc#tc{ result = auto_skip } }
end;
parse_case("=finished" ++ _ , _Fd, #tc{ name = undefined }) ->
finished;