aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test
diff options
context:
space:
mode:
Diffstat (limited to 'lib/common_test')
-rw-r--r--lib/common_test/doc/src/ct_hooks.xml5
-rw-r--r--lib/common_test/src/ct_framework.erl274
-rw-r--r--lib/common_test/src/ct_logs.erl6
-rw-r--r--lib/common_test/src/ct_netconfc.erl8
-rw-r--r--lib/common_test/src/cth_log_redirect.erl12
-rw-r--r--lib/common_test/src/cth_surefire.erl5
-rw-r--r--lib/common_test/test/common_test.cover1
-rw-r--r--lib/common_test/test/ct_config_SUITE.erl34
-rw-r--r--lib/common_test/test/ct_config_info_SUITE.erl6
-rw-r--r--lib/common_test/test/ct_error_SUITE.erl52
-rw-r--r--lib/common_test/test/ct_group_info_SUITE.erl156
-rw-r--r--lib/common_test/test/ct_groups_test_2_SUITE.erl12
-rw-r--r--lib/common_test/test/ct_hooks_SUITE.erl26
-rw-r--r--lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl25
-rw-r--r--lib/common_test/test/ct_netconfc_SUITE_data/ns.erl9
-rw-r--r--lib/common_test/test/ct_pre_post_test_io_SUITE.erl43
-rw-r--r--lib/common_test/test/ct_pre_post_test_io_SUITE_data/cth_ctrl.erl3
-rw-r--r--lib/common_test/test/ct_repeat_1_SUITE.erl26
-rw-r--r--lib/common_test/test/ct_repeat_testrun_SUITE.erl21
-rw-r--r--lib/common_test/test/ct_sequence_1_SUITE.erl4
-rw-r--r--lib/common_test/test/ct_skip_SUITE.erl433
-rw-r--r--lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_12_SUITE.erl121
-rw-r--r--lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_3_SUITE.erl2
-rw-r--r--lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_6_SUITE.erl118
-rw-r--r--lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_7_SUITE.erl113
-rw-r--r--lib/common_test/test/ct_surefire_SUITE.erl7
-rw-r--r--lib/common_test/test/ct_test_server_if_1_SUITE.erl182
-rw-r--r--lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_7_SUITE.erl16
-rw-r--r--lib/common_test/test/ct_test_support.erl30
-rw-r--r--lib/common_test/test/ct_testspec_1_SUITE.erl230
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE.erl10
31 files changed, 1483 insertions, 507 deletions
diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml
index b3e713c77f..859ff9df14 100644
--- a/lib/common_test/doc/src/ct_hooks.xml
+++ b/lib/common_test/doc/src/ct_hooks.xml
@@ -485,8 +485,9 @@
NewCTHState</name>
<fsummary>Called after the CTH scope ends</fsummary>
<type>
- <v>TestcaseName = end_per_suite | init_per_group |
- end_per_group | atom()</v>
+ <v>TestcaseName = end_per_suite | {init_per_group,GroupName} |
+ {end_per_group,GroupName} | atom()</v>
+ <v>GroupName = atom()</v>
<v>Reason = {tc_auto_skip | tc_user_skip, term()}</v>
<v>CTHState = NewCTHState = term()</v>
</type>
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index 276f902b05..e81b69a1b5 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -69,13 +69,13 @@ init_tc(Mod,Func,Config) ->
andalso Func=/=end_per_group
andalso ct_util:get_testdata(skip_rest) of
true ->
- {skip,"Repeated test stopped by force_stop option"};
+ {auto_skip,"Repeated test stopped by force_stop option"};
_ ->
case ct_util:get_testdata(curr_tc) of
{Suite,{suite0_failed,{require,Reason}}} ->
- {skip,{require_failed_in_suite0,Reason}};
+ {auto_skip,{require_failed_in_suite0,Reason}};
{Suite,{suite0_failed,_}=Failure} ->
- {skip,Failure};
+ {fail,Failure};
_ ->
ct_util:update_testdata(curr_tc,
fun(undefined) ->
@@ -103,7 +103,7 @@ init_tc(Mod,Func,Config) ->
end,
init_tc1(Mod,Suite,Func,Config);
{failed,Seq,BadFunc} ->
- {skip,{sequence_failed,Seq,BadFunc}}
+ {auto_skip,{sequence_failed,Seq,BadFunc}}
end
end
end.
@@ -115,9 +115,9 @@ init_tc1(?MODULE,_,error_in_suite,[Config0]) when is_list(Config0) ->
data={?MODULE,error_in_suite}}),
case ?val(error, Config0) of
undefined ->
- {skip,"unknown_error_in_suite"};
+ {fail,"unknown_error_in_suite"};
Reason ->
- {skip,Reason}
+ {fail,Reason}
end;
init_tc1(Mod,Suite,Func,[Config0]) when is_list(Config0) ->
@@ -159,22 +159,28 @@ init_tc1(Mod,Suite,Func,[Config0]) when is_list(Config0) ->
true ->
ct_config:delete_default_config(testcase)
end,
+ Initialize = fun() ->
+ ct_logs:init_tc(false),
+ ct_event:notify(#event{name=tc_start,
+ node=node(),
+ data={Mod,FuncSpec}})
+ end,
case add_defaults(Mod,Func,AllGroups) of
Error = {suite0_failed,_} ->
- ct_logs:init_tc(false),
- ct_event:notify(#event{name=tc_start,
- node=node(),
- data={Mod,FuncSpec}}),
+ Initialize(),
ct_util:set_testdata({curr_tc,{Suite,Error}}),
{error,Error};
+ Error = {group0_failed,_} ->
+ Initialize(),
+ {auto_skip,Error};
+ Error = {testcase0_failed,_} ->
+ Initialize(),
+ {auto_skip,Error};
{SuiteInfo,MergeResult} ->
case MergeResult of
{error,Reason} ->
- ct_logs:init_tc(false),
- ct_event:notify(#event{name=tc_start,
- node=node(),
- data={Mod,FuncSpec}}),
- {skip,Reason};
+ Initialize(),
+ {fail,Reason};
_ ->
init_tc2(Mod,Suite,Func,SuiteInfo,MergeResult,Config)
end
@@ -222,11 +228,11 @@ init_tc2(Mod,Suite,Func,SuiteInfo,MergeResult,Config) ->
{suite0_failed,Reason} ->
ct_util:set_testdata({curr_tc,{Mod,{suite0_failed,
{require,Reason}}}}),
- {skip,{require_failed_in_suite0,Reason}};
+ {auto_skip,{require_failed_in_suite0,Reason}};
{error,Reason} ->
{auto_skip,{require_failed,Reason}};
{'EXIT',Reason} ->
- {auto_skip,Reason};
+ {fail,Reason};
{ok,PostInitHook,Config1} ->
case get('$test_server_framework_test') of
undefined ->
@@ -272,6 +278,8 @@ add_defaults(Mod,Func, GroupPath) ->
SuiteInfo = merge_with_suite_defaults(Suite,[]),
SuiteInfoNoCTH = [I || I <- SuiteInfo, element(1,I) =/= ct_hooks],
case add_defaults1(Mod,Func, GroupPath, SuiteInfoNoCTH) of
+ Error = {group0_failed,_} -> Error;
+ Error = {testcase0_failed,_} -> Error;
Error = {error,_} -> {SuiteInfo,Error};
MergedInfo -> {SuiteInfo,MergedInfo}
end;
@@ -292,13 +300,16 @@ add_defaults(Mod,Func, GroupPath) ->
element(1,I) =/= ct_hooks],
case add_defaults1(Mod,Func, GroupPath,
SuiteInfoNoCTH) of
+ Error = {group0_failed,_} -> Error;
+ Error = {testcase0_failed,_} -> Error;
Error = {error,_} -> {SuiteInfo1,Error};
MergedInfo -> {SuiteInfo1,MergedInfo}
end;
false ->
ErrStr = io_lib:format("~n*** ERROR *** "
"Invalid return value from "
- "~w:suite/0: ~p~n", [Suite,SuiteInfo]),
+ "~w:suite/0: ~p~n",
+ [Suite,SuiteInfo]),
io:format(ErrStr, []),
io:format(user, ErrStr, []),
{suite0_failed,bad_return_value}
@@ -318,36 +329,69 @@ add_defaults1(Mod,Func, GroupPath, SuiteInfo) ->
%% [LevelXGroupInfo, LevelX-1GroupInfo, ..., TopLevelGroupInfo]
GroupPathInfo =
lists:map(fun(GroupProps) ->
- Name = ?val(name, GroupProps),
- case catch Suite:group(Name) of
- GrInfo when is_list(GrInfo) -> GrInfo;
- _ -> []
+ case ?val(name, GroupProps) of
+ undefined ->
+ [];
+ Name ->
+ case catch Suite:group(Name) of
+ GrInfo when is_list(GrInfo) -> GrInfo;
+ {'EXIT',{undef,_}} -> [];
+ BadGr0 -> {error,BadGr0,Name}
+ end
end
end, GroupPath),
- Args = if Func == init_per_group ; Func == end_per_group ->
- [?val(name, hd(GroupPath))];
- true ->
- []
- end,
- TestCaseInfo =
- case catch apply(Mod,Func,Args) of
- TCInfo when is_list(TCInfo) -> TCInfo;
- _ -> []
- end,
- %% let test case info (also for all config funcs) override group info,
- %% and lower level group info override higher level info
- TCAndGroupInfo = [TestCaseInfo | remove_info_in_prev(TestCaseInfo,
- GroupPathInfo)],
- %% find and save require terms found in suite info
- SuiteReqs =
- [SDDef || SDDef <- SuiteInfo,
- ((require == element(1,SDDef)) or
- (default_config == element(1,SDDef)))],
- case check_for_clashes(TestCaseInfo, GroupPathInfo, SuiteReqs) of
- [] ->
- add_defaults2(Mod,Func, TCAndGroupInfo,SuiteInfo,SuiteReqs);
- Clashes ->
- {error,{config_name_already_in_use,Clashes}}
+ case lists:keysearch(error, 1, GroupPathInfo) of
+ {value,{error,BadGr0Val,GrName}} ->
+ Gr0ErrStr = io_lib:format("~n*** ERROR *** "
+ "Invalid return value from "
+ "~w:group(~w): ~p~n",
+ [Mod,GrName,BadGr0Val]),
+ io:format(Gr0ErrStr, []),
+ io:format(user, Gr0ErrStr, []),
+ {group0_failed,bad_return_value};
+ _ ->
+ Args = if Func == init_per_group ; Func == end_per_group ->
+ [?val(name, hd(GroupPath))];
+ true ->
+ []
+ end,
+ TestCaseInfo =
+ case catch apply(Mod,Func,Args) of
+ TCInfo when is_list(TCInfo) -> TCInfo;
+ {'EXIT',{undef,_}} -> [];
+ BadTC0 -> {error,BadTC0}
+ end,
+
+ case TestCaseInfo of
+ {error,BadTC0Val} ->
+ TC0ErrStr = io_lib:format("~n*** ERROR *** "
+ "Invalid return value from "
+ "~w:~w/0: ~p~n",
+ [Mod,Func,BadTC0Val]),
+ io:format(TC0ErrStr, []),
+ io:format(user, TC0ErrStr, []),
+ {testcase0_failed,bad_return_value};
+ _ ->
+ %% let test case info (also for all config funcs) override
+ %% group info, and lower level group info override higher
+ %% level info
+ TCAndGroupInfo =
+ [TestCaseInfo | remove_info_in_prev(TestCaseInfo,
+ GroupPathInfo)],
+ %% find and save require terms found in suite info
+ SuiteReqs =
+ [SDDef || SDDef <- SuiteInfo,
+ ((require == element(1,SDDef))
+ or (default_config == element(1,SDDef)))],
+ case check_for_clashes(TestCaseInfo, GroupPathInfo,
+ SuiteReqs) of
+ [] ->
+ add_defaults2(Mod,Func, TCAndGroupInfo,
+ SuiteInfo,SuiteReqs);
+ Clashes ->
+ {error,{config_name_already_in_use,Clashes}}
+ end
+ end
end.
get_suite_name(?MODULE, [Cfg|_]) when is_list(Cfg), Cfg /= [] ->
@@ -621,31 +665,34 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) ->
_ -> Func
end,
- case get('$test_server_framework_test') of
- undefined ->
- {FinalResult,FinalNotify} =
- case ct_hooks:end_tc(
- Suite, FuncSpec, Args, Result, Return) of
- '$ct_no_change' ->
- {ok,Result};
- FinalResult1 ->
- {FinalResult1,FinalResult1}
- end,
- %% send sync notification so that event handlers may print
- %% in the log file before it gets closed
- ct_event:sync_notify(#event{name=tc_done,
- node=node(),
- data={Mod,FuncSpec,
- tag_cth(FinalNotify)}});
- Fun ->
- %% send sync notification so that event handlers may print
- %% in the log file before it gets closed
- ct_event:sync_notify(#event{name=tc_done,
- node=node(),
- data={Mod,FuncSpec,tag(Result)}}),
- FinalResult = Fun(end_tc, Return)
- end,
-
+ {Result1,FinalNotify} =
+ case ct_hooks:end_tc(
+ Suite, FuncSpec, Args, Result, Return) of
+ '$ct_no_change' ->
+ {ok,Result};
+ HookResult ->
+ {HookResult,HookResult}
+ end,
+ FinalResult =
+ case get('$test_server_framework_test') of
+ undefined ->
+ %% send sync notification so that event handlers may print
+ %% in the log file before it gets closed
+ ct_event:sync_notify(#event{name=tc_done,
+ node=node(),
+ data={Mod,FuncSpec,
+ tag_cth(FinalNotify)}}),
+ Result1;
+ Fun ->
+ %% send sync notification so that event handlers may print
+ %% in the log file before it gets closed
+ ct_event:sync_notify(#event{name=tc_done,
+ node=node(),
+ data={Mod,FuncSpec,
+ tag(FinalNotify)}}),
+ Fun(end_tc, Return)
+ end,
+
case FuncSpec of
{_,GroupName,_Props} ->
if Func == end_per_group ->
@@ -685,10 +732,10 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) ->
(Unexpected) ->
exit({error,{reset_curr_tc,{Mod,Func},Unexpected}})
end,
- ct_util:update_testdata(curr_tc,ClearCurrTC),
+ ct_util:update_testdata(curr_tc, ClearCurrTC),
case FinalResult of
- {skip,{sequence_failed,_,_}} ->
+ {auto_skip,{sequence_failed,_,_}} ->
%% ct_logs:init_tc is never called for a skipped test case
%% in a failing sequence, so neither should end_tc
ok;
@@ -711,8 +758,13 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) ->
%% {error,Reason} | {skip,Reason} | {timetrap_timeout,TVal} |
%% {testcase_aborted,Reason} | testcase_aborted_or_killed |
%% {'EXIT',Reason} | Other (ignored return value, e.g. 'ok')
-tag({STag,Reason}) when STag == skip; STag == skipped ->
- {skipped,Reason};
+tag({STag,Reason}) when STag == skip; STag == skipped ->
+ case Reason of
+ {failed,{_,init_per_testcase,_}} -> {auto_skipped,Reason};
+ _ -> {skipped,Reason}
+ end;
+tag({auto_skip,Reason}) ->
+ {auto_skipped,Reason};
tag(E = {ETag,_}) when ETag == error; ETag == 'EXIT';
ETag == timetrap_timeout;
ETag == testcase_aborted ->
@@ -722,13 +774,20 @@ tag(E = testcase_aborted_or_killed) ->
tag(Other) ->
Other.
+tag_cth({skipped,Reason={failed,{_,init_per_testcase,_}}}) ->
+ {auto_skipped,Reason};
tag_cth({STag,Reason}) when STag == skip; STag == skipped ->
- {skipped,Reason};
-tag_cth({fail, Reason}) ->
- {failed, {error,Reason}};
+ case Reason of
+ {failed,{_,init_per_testcase,_}} -> {auto_skipped,Reason};
+ _ -> {skipped,Reason}
+ end;
+tag_cth({auto_skip,Reason}) ->
+ {auto_skipped,Reason};
+tag_cth({fail,Reason}) ->
+ {failed,{error,Reason}};
tag_cth(E = {ETag,_}) when ETag == error; ETag == 'EXIT';
- ETag == timetrap_timeout;
- ETag == testcase_aborted ->
+ ETag == timetrap_timeout;
+ ETag == testcase_aborted ->
{failed,E};
tag_cth(E = testcase_aborted_or_killed) ->
{failed,E};
@@ -1229,6 +1288,8 @@ report(What,Data) ->
ct_hooks:on_tc_skip(tc_auto_skip, Data);
{skipped,_} ->
ct_hooks:on_tc_skip(tc_user_skip, Data);
+ {auto_skipped,_} ->
+ ct_hooks:on_tc_skip(tc_auto_skip, Data);
_Else ->
ok
end,
@@ -1253,31 +1314,54 @@ report(What,Data) ->
add_to_stats(auto_skipped);
{_,{skipped,_}} ->
add_to_stats(user_skipped);
+ {_,{auto_skipped,_}} ->
+ add_to_stats(auto_skipped);
{_,{SkipOrFail,_Reason}} ->
add_to_stats(SkipOrFail)
end;
- tc_user_skip ->
- %% test case specified as skipped in testspec
- %% Data = {Suite,Case,Comment}
+ tc_user_skip ->
+ %% test case or config function specified as skipped in testspec,
+ %% or init config func for suite/group has returned {skip,Reason}
+ %% Data = {Suite,Case,Comment} |
+ %% {Suite,{GroupConfigFunc,GroupName},Comment}
+ {Func,Data1} = case Data of
+ {Suite,{ConfigFunc,undefined},Cmt} ->
+ {ConfigFunc,{Suite,ConfigFunc,Cmt}};
+ {_,{ConfigFunc,_},_} -> {ConfigFunc,Data};
+ {_,Case,_} -> {Case,Data}
+ end,
+
ct_event:sync_notify(#event{name=tc_user_skip,
node=node(),
- data=Data}),
- ct_hooks:on_tc_skip(What, Data),
- add_to_stats(user_skipped);
- tc_auto_skip ->
- %% test case skipped because of error in init_per_suite
- %% Data = {Suite,Case,Comment}
-
- {_Suite,Case,_Result} = Data,
+ data=Data1}),
+ ct_hooks:on_tc_skip(What, Data1),
+ if Func /= init_per_suite, Func /= init_per_group,
+ Func /= end_per_suite, Func /= end_per_group ->
+ add_to_stats(user_skipped);
+ true ->
+ ok
+ end;
+ tc_auto_skip ->
+ %% test case skipped because of error in config function, or
+ %% config function skipped because of error in info function
+ %% Data = {Suite,Case,Comment} |
+ %% {Suite,{GroupConfigFunc,GroupName},Comment}
+ {Func,Data1} = case Data of
+ {Suite,{ConfigFunc,undefined},Cmt} ->
+ {ConfigFunc,{Suite,ConfigFunc,Cmt}};
+ {_,{ConfigFunc,_},_} -> {ConfigFunc,Data};
+ {_,Case,_} -> {Case,Data}
+ end,
%% this test case does not have a log, so printouts
%% from event handlers should end up in the main log
ct_event:sync_notify(#event{name=tc_auto_skip,
node=node(),
- data=Data}),
- ct_hooks:on_tc_skip(What, Data),
- if Case /= end_per_suite,
- Case /= end_per_group ->
+ data=Data1}),
+ ct_hooks:on_tc_skip(What, Data1),
+
+ if Func /= end_per_suite,
+ Func /= end_per_group ->
add_to_stats(auto_skipped);
true ->
ok
diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl
index 1a6e4d31a8..a7fb45a4e4 100644
--- a/lib/common_test/src/ct_logs.erl
+++ b/lib/common_test/src/ct_logs.erl
@@ -1249,7 +1249,8 @@ make_one_index_entry1(SuiteName, Link, Label, Success, Fail, UserSkip, AutoSkip,
integer_to_list(NotBuilt),"</a></td>\n"]
end,
FailStr =
- if Fail > 0 ->
+ if (Fail > 0) or (NotBuilt > 0) or
+ ((Success+Fail+UserSkip+AutoSkip) == 0) ->
["<font color=\"red\">",
integer_to_list(Fail),"</font>"];
true ->
@@ -1904,7 +1905,8 @@ runentry(Dir, undefined, _) ->
runentry(Dir, Totals={Node,Label,Logs,
{TotSucc,TotFail,UserSkip,AutoSkip,NotBuilt}}, Index) ->
TotFailStr =
- if TotFail > 0 ->
+ if (TotFail > 0) or (NotBuilt > 0) or
+ ((TotSucc+TotFail+UserSkip+AutoSkip) == 0) ->
["<font color=\"red\">",
integer_to_list(TotFail),"</font>"];
true ->
diff --git a/lib/common_test/src/ct_netconfc.erl b/lib/common_test/src/ct_netconfc.erl
index 7f10e1db09..64fe8b4bb0 100644
--- a/lib/common_test/src/ct_netconfc.erl
+++ b/lib/common_test/src/ct_netconfc.erl
@@ -1129,10 +1129,14 @@ handle_msg({Ref,timeout},
ct_gen_conn:return(Caller,{error,{hello_session_failed,timeout}}),
{stop,State#state{hello_status={error,timeout}}};
handle_msg({Ref,timeout},#state{pending=Pending} = State) ->
- {value,#pending{caller=Caller},Pending1} =
+ {value,#pending{op=Op,caller=Caller},Pending1} =
lists:keytake(Ref,#pending.ref,Pending),
ct_gen_conn:return(Caller,{error,timeout}),
- {noreply,State#state{pending=Pending1}}.
+ R = case Op of
+ close_session -> stop;
+ _ -> noreply
+ end,
+ {R,State#state{pending=Pending1}}.
%% @private
%% Called by ct_util_server to close registered connections before terminate.
diff --git a/lib/common_test/src/cth_log_redirect.erl b/lib/common_test/src/cth_log_redirect.erl
index 4ee7e48a67..8fed341600 100644
--- a/lib/common_test/src/cth_log_redirect.erl
+++ b/lib/common_test/src/cth_log_redirect.erl
@@ -34,13 +34,15 @@
%% Event handler Callbacks
-export([init/1,
handle_event/2, handle_call/2, handle_info/2,
- terminate/1]).
+ terminate/1, terminate/2, code_change/3]).
%% Other
-export([handle_remote_events/1]).
-include("ct.hrl").
+-behaviour(gen_event).
+
-record(eh_state, {log_func,
curr_suite,
curr_group,
@@ -184,10 +186,13 @@ handle_call({handle_remote_events,Bool}, State) ->
handle_call(_Query, _State) ->
{error, bad_query}.
-terminate(_State) ->
+terminate(_) ->
error_logger:delete_report_handler(?MODULE),
[].
+terminate(_Arg, _State) ->
+ ok.
+
tag_event(Event) ->
{calendar:local_time(), Event}.
@@ -236,3 +241,6 @@ format_header(#eh_state{curr_suite = Suite,
curr_func = TC}) ->
io_lib:format("System report during ~w:~w/1 in ~w",
[Suite,TC,Group]).
+
+code_change(_OldVsn, State, _Extra) ->
+ {ok, State}.
diff --git a/lib/common_test/src/cth_surefire.erl b/lib/common_test/src/cth_surefire.erl
index 1a38b6584b..7ed2018bdf 100644
--- a/lib/common_test/src/cth_surefire.erl
+++ b/lib/common_test/src/cth_surefire.erl
@@ -138,6 +138,9 @@ on_tc_fail(_TC, Res, State) ->
{fail,lists:flatten(io_lib:format("~p",[Res]))} },
State#state{ test_cases = [NewTC | tl(TCs)]}.
+on_tc_skip({ConfigFunc,_GrName},{Type,_Reason} = Res, State0)
+ when Type == tc_auto_skip; Type == tc_user_skip ->
+ on_tc_skip(ConfigFunc, Res, State0);
on_tc_skip(Tc,{Type,_Reason} = Res, State0) when Type == tc_auto_skip ->
TcStr = atom_to_list(Tc),
State =
@@ -330,5 +333,7 @@ count_tcs([#testcase{result={fail,_}}|TCs],Ok,F,S) ->
count_tcs(TCs,Ok,F+1,S);
count_tcs([#testcase{result={skipped,_}}|TCs],Ok,F,S) ->
count_tcs(TCs,Ok,F,S+1);
+count_tcs([#testcase{result={auto_skipped,_}}|TCs],Ok,F,S) ->
+ count_tcs(TCs,Ok,F,S+1);
count_tcs([],Ok,F,S) ->
{Ok+F+S,F,S}.
diff --git a/lib/common_test/test/common_test.cover b/lib/common_test/test/common_test.cover
index 3aa49623e7..87d00c420f 100644
--- a/lib/common_test/test/common_test.cover
+++ b/lib/common_test/test/common_test.cover
@@ -4,7 +4,6 @@
test_server,
test_server_ctrl,
test_server_gl,
- test_server_h,
test_server_io,
test_server_node,
test_server_sup]}]}.
diff --git a/lib/common_test/test/ct_config_SUITE.erl b/lib/common_test/test/ct_config_SUITE.erl
index d92be9ec6e..284f836517 100644
--- a/lib/common_test/test/ct_config_SUITE.erl
+++ b/lib/common_test/test/ct_config_SUITE.erl
@@ -234,25 +234,25 @@ expected_events(config_static_SUITE)->
?sok(test_get_config_deep_nested,{3,0,{0,0}}),
?sok(test_default_suitewide,{4,0,{0,0}}),
?snok(test_config_name_already_in_use1,
- {skipped,{config_name_already_in_use,[x1]}},{4,0,{1,0}}),
- ?sok(test_default_tclocal,{5,0,{1,0}}),
+ {failed,{error,{config_name_already_in_use,[x1]}}},{4,1,{0,0}}),
+ ?sok(test_default_tclocal,{5,1,{0,0}}),
?snok(test_config_name_already_in_use2,
- {skipped,{config_name_already_in_use,[alias,x1]}},{5,0,{2,0}}),
- ?sok(test_alias_tclocal,{6,0,{2,0}}),
- ?sok(test_get_config_undefined,{7,0,{2,0}}),
- ?sok(test_require_subvals,{8,0,{2,0}}),
+ {failed,{error,{config_name_already_in_use,[alias,x1]}}},{5,2,{0,0}}),
+ ?sok(test_alias_tclocal,{6,2,{0,0}}),
+ ?sok(test_get_config_undefined,{7,2,{0,0}}),
+ ?sok(test_require_subvals,{8,2,{0,0}}),
?snok(test_require_subvals2,
- {skipped,{require_failed,
- {not_available,{gen_cfg,[a,b,c,d]}}}},{8,0,{2,1}}),
- ?sok(test_require_deep_config,{9,0,{2,1}}),
- ?sok(test_shadow_all,{10,0,{2,1}}),
- ?sok(test_element,{11,0,{2,1}}),
- ?sok(test_shadow_all_element,{12,0,{2,1}}),
- ?sok(test_internal_deep,{13,0,{2,1}}),
- ?sok(test_alias_tclocal_nested,{14,0,{2,1}}),
- ?sok(test_alias_tclocal_nested_backward_compat,{15,0,{2,1}}),
- ?sok(test_alias_tclocal_nested_backward_compat_subvals,{16,0,{2,1}}),
- ?sok(test_config_same_name_already_in_use,{17,0,{2,1}}),
+ {auto_skipped,{require_failed,
+ {not_available,{gen_cfg,[a,b,c,d]}}}},{8,2,{0,1}}),
+ ?sok(test_require_deep_config,{9,2,{0,1}}),
+ ?sok(test_shadow_all,{10,2,{0,1}}),
+ ?sok(test_element,{11,2,{0,1}}),
+ ?sok(test_shadow_all_element,{12,2,{0,1}}),
+ ?sok(test_internal_deep,{13,2,{0,1}}),
+ ?sok(test_alias_tclocal_nested,{14,2,{0,1}}),
+ ?sok(test_alias_tclocal_nested_backward_compat,{15,2,{0,1}}),
+ ?sok(test_alias_tclocal_nested_backward_compat_subvals,{16,2,{0,1}}),
+ ?sok(test_config_same_name_already_in_use,{17,2,{0,1}}),
{?eh,tc_start,{config_static_SUITE,end_per_suite}},
{?eh,tc_done,{config_static_SUITE,end_per_suite,ok}},
{?eh,test_done,{'DEF','STOP_TIME'}},
diff --git a/lib/common_test/test/ct_config_info_SUITE.erl b/lib/common_test/test/ct_config_info_SUITE.erl
index 10fe8286dd..8f2f0eb75f 100644
--- a/lib/common_test/test/ct_config_info_SUITE.erl
+++ b/lib/common_test/test/ct_config_info_SUITE.erl
@@ -127,7 +127,7 @@ test_events(config_info) ->
{failed,{timetrap_timeout,350}}}},
{?eh,tc_auto_skip,{config_info_1_SUITE,t11,
{failed,{config_info_1_SUITE,init_per_group,{timetrap_timeout,350}}}}},
- {?eh,tc_auto_skip,{config_info_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{config_info_1_SUITE,{end_per_group,g1},
{failed,{config_info_1_SUITE,init_per_group,
{timetrap_timeout,350}}}}}],
@@ -145,12 +145,12 @@ test_events(config_info) ->
{?eh,tc_auto_skip,{config_info_1_SUITE,t41,
{failed,{config_info_1_SUITE,init_per_group,
{timetrap_timeout,400}}}}},
- {?eh,tc_auto_skip,{config_info_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{config_info_1_SUITE,{end_per_group,g4},
{failed,{config_info_1_SUITE,init_per_group,
{timetrap_timeout,400}}}}}],
{?eh,tc_start,{config_info_1_SUITE,t31}},
{?eh,tc_done,{config_info_1_SUITE,t31,
- {skipped,{failed,{config_info_1_SUITE,init_per_testcase,
+ {auto_skipped,{failed,{config_info_1_SUITE,init_per_testcase,
{timetrap_timeout,250}}}}}},
{?eh,tc_start,{config_info_1_SUITE,t32}},
{?eh,tc_done,{config_info_1_SUITE,t32,
diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl
index 3881ced17d..28f0494d20 100644
--- a/lib/common_test/test/ct_error_SUITE.erl
+++ b/lib/common_test/test/ct_error_SUITE.erl
@@ -465,7 +465,7 @@ test_events(cfg_error) ->
{'EXIT',{init_per_group_fails,g1}}}}}},
{?eh,test_stats,{4,0,{0,11}}},
{?eh,tc_auto_skip,
- {cfg_error_8_SUITE,end_per_group,
+ {cfg_error_8_SUITE,{end_per_group,g1},
{failed,{cfg_error_8_SUITE,init_per_group,
{'EXIT',{init_per_group_fails,g1}}}}}}],
@@ -476,7 +476,7 @@ test_events(cfg_error) ->
{failed,{cfg_error_8_SUITE,init_per_group,
{timetrap_timeout,2000}}}}},
{?eh,test_stats,{4,0,{0,12}}},
- {?eh,tc_auto_skip,{cfg_error_8_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{cfg_error_8_SUITE,{end_per_group,g2},
{failed,{cfg_error_8_SUITE,init_per_group,
{timetrap_timeout,2000}}}}}],
@@ -490,7 +490,7 @@ test_events(cfg_error) ->
{'EXIT',{{badmatch,42},'_'}}}}}},
{?eh,test_stats,{4,0,{0,13}}},
{?eh,tc_auto_skip,
- {cfg_error_8_SUITE,end_per_group,
+ {cfg_error_8_SUITE,{end_per_group,g3},
{failed,{cfg_error_8_SUITE,init_per_group,
{'EXIT',{{badmatch,42},'_'}}}}}}],
@@ -516,7 +516,7 @@ test_events(cfg_error) ->
{'EXIT',{sub_group_failed,g6}}}}}},
{?eh,test_stats,{6,0,{0,14}}},
{?eh,tc_auto_skip,
- {cfg_error_8_SUITE,end_per_group,
+ {cfg_error_8_SUITE,{end_per_group,g6},
{failed,{cfg_error_8_SUITE,init_per_group,
{'EXIT',{sub_group_failed,g6}}}}}}],
{?eh,tc_start,{cfg_error_8_SUITE,tc3}},
@@ -550,23 +550,23 @@ test_events(cfg_error) ->
{?eh,tc_done,{cfg_error_9_SUITE,init_per_suite,ok}},
{?eh,tc_start,{cfg_error_9_SUITE,tc1}},
{?eh,tc_done,{cfg_error_9_SUITE,tc1,
- {skipped,{failed,{cfg_error_9_SUITE,init_per_testcase,
- {tc1_should_be_skipped,'_'}}}}}},
+ {auto_skipped,{failed,{cfg_error_9_SUITE,init_per_testcase,
+ {tc1_should_be_skipped,'_'}}}}}},
{?eh,test_stats,{9,0,{0,15}}},
{?eh,tc_start,{cfg_error_9_SUITE,tc2}},
{?eh,tc_done,{cfg_error_9_SUITE,tc2,
- {skipped,{failed,{cfg_error_9_SUITE,init_per_testcase,
- {timetrap_timeout,2000}}}}}},
+ {auto_skipped,{failed,{cfg_error_9_SUITE,init_per_testcase,
+ {timetrap_timeout,2000}}}}}},
{?eh,test_stats,{9,0,{0,16}}},
{?eh,tc_start,{cfg_error_9_SUITE,tc3}},
{?eh,tc_done,{cfg_error_9_SUITE,tc3,
- {skipped,{failed,{cfg_error_9_SUITE,init_per_testcase,
- {{badmatch,undefined},'_'}}}}}},
+ {auto_skipped,{failed,{cfg_error_9_SUITE,init_per_testcase,
+ {{badmatch,undefined},'_'}}}}}},
{?eh,test_stats,{9,0,{0,17}}},
{?eh,tc_start,{cfg_error_9_SUITE,tc4}},
{?eh,tc_done,
{cfg_error_9_SUITE,tc4,
- {skipped,{failed,{cfg_error_9_SUITE,init_per_testcase,bad_return}}}}},
+ {auto_skipped,{failed,{cfg_error_9_SUITE,init_per_testcase,bad_return}}}}},
{?eh,test_stats,{9,0,{0,18}}},
{?eh,tc_start,{cfg_error_9_SUITE,tc5}},
{?eh,tc_done,
@@ -622,41 +622,41 @@ test_events(cfg_error) ->
{?eh,tc_done,{cfg_error_11_SUITE,init_per_suite,ok}},
{?eh,tc_start,{cfg_error_11_SUITE,tc1}},
{?eh,tc_done, {cfg_error_11_SUITE,tc1,
- {skipped,{config_name_already_in_use,[dummy_alias]}}}},
- {?eh,test_stats,{12,6,{1,19}}},
+ {failed,{error,{config_name_already_in_use,[dummy_alias]}}}}},
+ {?eh,test_stats,{12,7,{0,19}}},
{?eh,tc_start,{cfg_error_11_SUITE,tc2}},
{?eh,tc_done,{cfg_error_11_SUITE,tc2,ok}},
- {?eh,test_stats,{13,6,{1,19}}},
+ {?eh,test_stats,{13,7,{0,19}}},
{?eh,tc_start,{cfg_error_11_SUITE,end_per_suite}},
{?eh,tc_done,{cfg_error_11_SUITE,end_per_suite,ok}},
{?eh,tc_start,{cfg_error_12_SUITE,tc1}},
{?eh,tc_done,{ct_framework,init_tc,{framework_error,{timetrap,500}}}},
- {?eh,test_stats,{13,7,{1,19}}},
+ {?eh,test_stats,{13,8,{0,19}}},
{?eh,tc_start,{cfg_error_12_SUITE,tc2}},
{?eh,tc_done,{cfg_error_12_SUITE,tc2,{failed,
{cfg_error_12_SUITE,end_per_testcase,
{timetrap_timeout,500}}}}},
- {?eh,test_stats,{14,7,{1,19}}},
+ {?eh,test_stats,{14,8,{0,19}}},
{?eh,tc_start,{cfg_error_12_SUITE,tc3}},
{?eh,tc_done,{cfg_error_12_SUITE,tc3,ok}},
- {?eh,test_stats,{15,7,{1,19}}},
+ {?eh,test_stats,{15,8,{0,19}}},
{?eh,tc_start,{cfg_error_12_SUITE,tc4}},
{?eh,tc_done,{cfg_error_12_SUITE,tc4,{failed,
{cfg_error_12_SUITE,end_per_testcase,
{timetrap_timeout,500}}}}},
- {?eh,test_stats,{16,7,{1,19}}},
+ {?eh,test_stats,{16,8,{0,19}}},
{?eh,tc_start,{cfg_error_13_SUITE,init_per_suite}},
{?eh,tc_done,{cfg_error_13_SUITE,init_per_suite,ok}},
{?eh,tc_start,{cfg_error_13_SUITE,tc1}},
{?eh,tc_done,{cfg_error_13_SUITE,tc1,ok}},
- {?eh,test_stats,{17,7,{1,19}}},
+ {?eh,test_stats,{17,8,{0,19}}},
{?eh,tc_start,{cfg_error_13_SUITE,end_per_suite}},
{?eh,tc_done,{cfg_error_13_SUITE,end_per_suite,ok}},
{?eh,tc_start,{cfg_error_14_SUITE,init_per_suite}},
{?eh,tc_done,{cfg_error_14_SUITE,init_per_suite,ok}},
{?eh,tc_start,{cfg_error_14_SUITE,tc1}},
{?eh,tc_done,{cfg_error_14_SUITE,tc1,ok}},
- {?eh,test_stats,{18,7,{1,19}}},
+ {?eh,test_stats,{18,8,{0,19}}},
{?eh,tc_start,{cfg_error_14_SUITE,end_per_suite}},
{?eh,tc_done,{cfg_error_14_SUITE,end_per_suite,
{comment,
@@ -1102,7 +1102,7 @@ test_events(timetrap_fun_group) ->
{failed,{timetrap_8_SUITE,init_per_group,
{user_timetrap_error,{kaboom,'_'}}}}}},
{?eh,test_stats,{0,11,{0,2}}},
- {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,g4},
{failed,{timetrap_8_SUITE,init_per_group,
{user_timetrap_error,{kaboom,'_'}}}}}}],
@@ -1117,7 +1117,7 @@ test_events(timetrap_fun_group) ->
{failed,{timetrap_8_SUITE,init_per_group,
{user_timetrap_error,{kaboom,'_'}}}}}},
{?eh,test_stats,{0,11,{0,4}}},
- {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,g5},
{failed,{timetrap_8_SUITE,init_per_group,
{user_timetrap_error,{kaboom,'_'}}}}}}],
@@ -1132,7 +1132,7 @@ test_events(timetrap_fun_group) ->
{failed,{timetrap_8_SUITE,init_per_group,
{timetrap_timeout,'_'}}}}},
{?eh,test_stats,{0,11,{0,6}}},
- {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,g6},
{failed,{timetrap_8_SUITE,init_per_group,
{timetrap_timeout,'_'}}}}}],
@@ -1285,7 +1285,7 @@ test_events(timetrap_fun_group) ->
{failed,{timetrap_8_SUITE,init_per_group,
{user_timetrap_error,{kaboom,'_'}}}}}},
{?eh,test_stats,{4,26,{0,8}}},
- {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,pg4},
{failed,{timetrap_8_SUITE,init_per_group,
{user_timetrap_error,{kaboom,'_'}}}}}}]},
@@ -1301,7 +1301,7 @@ test_events(timetrap_fun_group) ->
{failed,{timetrap_8_SUITE,init_per_group,
{user_timetrap_error,{kaboom,'_'}}}}}},
{?eh,test_stats,{4,26,{0,10}}},
- {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,pg5},
{failed,{timetrap_8_SUITE,init_per_group,
{user_timetrap_error,{kaboom,'_'}}}}}}]},
@@ -1317,7 +1317,7 @@ test_events(timetrap_fun_group) ->
{failed,{timetrap_8_SUITE,init_per_group,
{timetrap_timeout,'_'}}}}},
{?eh,test_stats,{4,26,{0,12}}},
- {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,pg6},
{failed,{timetrap_8_SUITE,init_per_group,
{timetrap_timeout,'_'}}}}}]},
diff --git a/lib/common_test/test/ct_group_info_SUITE.erl b/lib/common_test/test/ct_group_info_SUITE.erl
index c56fa952e8..e7bc5baaa1 100644
--- a/lib/common_test/test/ct_group_info_SUITE.erl
+++ b/lib/common_test/test/ct_group_info_SUITE.erl
@@ -269,13 +269,19 @@ test_events(timetrap_all) ->
{?eh,tc_done,{group_timetrap_1_SUITE,{end_per_group,g10,[]},ok}}],
[{?eh,tc_start,{group_timetrap_1_SUITE,{init_per_group,g11,[]}}},
- {?eh,tc_done,{group_timetrap_1_SUITE,{init_per_group,g11,[]},ok}},
- {?eh,tc_done,{group_timetrap_1_SUITE,t111,{failed,{timetrap_timeout,1000}}}},
- {?eh,test_stats,{0,14,{0,0}}},
- {?eh,tc_start,{group_timetrap_1_SUITE,{end_per_group,g11,[]}}},
- {?eh,tc_done,{group_timetrap_1_SUITE,{end_per_group,g11,[]},ok}}],
-
+ {?eh,tc_done,{group_timetrap_1_SUITE,
+ {init_per_group,g11,[]},
+ {auto_skipped,{group0_failed,bad_return_value}}}},
+ {?eh,tc_auto_skip,
+ {group_timetrap_1_SUITE,t111,{group0_failed,bad_return_value}}},
+ {?eh,test_stats,{0,13,{0,1}}},
+ {?eh,tc_auto_skip,{group_timetrap_1_SUITE,
+ {end_per_group,g11},
+ {group0_failed,bad_return_value}}}],
+
+ {?eh,tc_start,{group_timetrap_1_SUITE,end_per_suite}},
{?eh,tc_done,{group_timetrap_1_SUITE,end_per_suite,ok}},
+
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}
];
@@ -422,13 +428,15 @@ test_events(timetrap_all_no_ips) ->
{?eh,tc_done,{group_timetrap_2_SUITE,{end_per_group,g10,[]},ok}}],
[{?eh,tc_start,{group_timetrap_2_SUITE,{init_per_group,g11,[]}}},
- {?eh,tc_done,{group_timetrap_2_SUITE,{init_per_group,g11,[]},ok}},
- {?eh,tc_done,{group_timetrap_2_SUITE,t111,{failed,{timetrap_timeout,1000}}}},
- {?eh,test_stats,{0,14,{0,0}}},
- {?eh,tc_start,{group_timetrap_2_SUITE,{end_per_group,g11,[]}}},
- {?eh,tc_done,{group_timetrap_2_SUITE,{end_per_group,g11,[]},ok}}],
-
- {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,tc_done,{group_timetrap_2_SUITE,
+ {init_per_group,g11,[]},
+ {auto_skipped,{group0_failed,bad_return_value}}}},
+ {?eh,tc_auto_skip,{group_timetrap_2_SUITE,t111,
+ {group0_failed,bad_return_value}}},
+ {?eh,test_stats,{0,13,{0,1}}},
+ {?eh,tc_auto_skip,{group_timetrap_2_SUITE,
+ {end_per_group,g11},
+ {group0_failed,bad_return_value}}}],
{?eh,stop_logging,[]}
];
@@ -501,11 +509,13 @@ test_events(timetrap_all_no_ipg) ->
{?eh,tc_done,{ct_framework,{end_per_group,g10,[{suite,group_timetrap_3_SUITE}]},ok}}],
[{?eh,tc_start,{ct_framework,{init_per_group,g11,[{suite,group_timetrap_3_SUITE}]}}},
- {?eh,tc_done,{ct_framework,{init_per_group,g11,[{suite,group_timetrap_3_SUITE}]},ok}},
- {?eh,tc_done,{group_timetrap_3_SUITE,t111,{failed,{timetrap_timeout,1000}}}},
- {?eh,test_stats,{0,14,{0,0}}},
- {?eh,tc_start,{ct_framework,{end_per_group,g11,[{suite,group_timetrap_3_SUITE}]}}},
- {?eh,tc_done,{ct_framework,{end_per_group,g11,[{suite,group_timetrap_3_SUITE}]},ok}}],
+ {?eh,tc_done,{ct_framework,
+ {init_per_group,g11,[{suite,group_timetrap_3_SUITE}]},
+ {auto_skipped,{group0_failed,bad_return_value}}}},
+ {?eh,tc_auto_skip,{group_timetrap_3_SUITE,t111,{group0_failed,bad_return_value}}},
+ {?eh,test_stats,{0,13,{0,1}}},
+ {?eh,tc_auto_skip,{ct_framework,{end_per_group,g11},
+ {group0_failed,bad_return_value}}}],
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}
@@ -539,11 +549,13 @@ test_events(require) ->
[{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g4,[]}}},
{?eh,tc_done,{group_require_1_SUITE,{init_per_group,g4,[]},
- {skipped,{require_failed,{name_in_use,common2_alias,common2}}}}},
+ {auto_skipped,{require_failed,
+ {name_in_use,common2_alias,common2}}}}},
{?eh,tc_auto_skip,{group_require_1_SUITE,t41,
- {require_failed,{name_in_use,common2_alias,common2}}}},
+ {require_failed,
+ {name_in_use,common2_alias,common2}}}},
{?eh,test_stats,{4,0,{0,1}}},
- {?eh,tc_auto_skip,{group_require_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{group_require_1_SUITE,{end_per_group,g4},
{require_failed,{name_in_use,common2_alias,common2}}}}],
[{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g5,[]}}},
@@ -566,16 +578,18 @@ test_events(require) ->
[{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g8,[]}}},
{?eh,tc_done,{group_require_1_SUITE,
{init_per_group,g8,[]},
- {skipped,{require_failed,{not_available,non_existing}}}}},
+ {auto_skipped,{require_failed,
+ {not_available,non_existing}}}}},
{?eh,tc_auto_skip,{group_require_1_SUITE,t81,
{require_failed,{not_available,non_existing}}}},
{?eh,test_stats,{8,0,{0,2}}},
- {?eh,tc_auto_skip,{group_require_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{group_require_1_SUITE,{end_per_group,g8},
{require_failed,{not_available,non_existing}}}}],
[{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g9,[]}}},
{?eh,tc_done,{group_require_1_SUITE,{init_per_group,g9,[]},ok}},
{?eh,tc_done,{group_require_1_SUITE,t91,
- {skipped,{require_failed,{not_available,non_existing}}}}},
+ {auto_skipped,{require_failed,
+ {not_available,non_existing}}}}},
{?eh,test_stats,{8,0,{0,3}}},
{?eh,tc_start,{group_require_1_SUITE,{end_per_group,g9,[]}}},
{?eh,tc_done,{group_require_1_SUITE,{end_per_group,g9,[]},ok}}],
@@ -587,12 +601,16 @@ test_events(require) ->
{?eh,tc_done,{group_require_1_SUITE,{end_per_group,g10,[]},ok}}],
[{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g11,[]}}},
- {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g11,[]},ok}},
- {?eh,tc_done,{group_require_1_SUITE,t111,ok}},
- {?eh,test_stats,{10,0,{0,3}}},
- {?eh,tc_start,{group_require_1_SUITE,{end_per_group,g11,[]}}},
- {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g11,[]},ok}}],
-
+ {?eh,tc_done,{group_require_1_SUITE,
+ {init_per_group,g11,[]},
+ {auto_skipped,{group0_failed,bad_return_value}}}},
+ {?eh,tc_auto_skip,{group_require_1_SUITE,t111,
+ {group0_failed,bad_return_value}}},
+ {?eh,test_stats,{9,0,{0,4}}},
+ {?eh,tc_auto_skip,{group_require_1_SUITE,
+ {end_per_group,g11},
+ {group0_failed,bad_return_value}}}],
+
{?eh,tc_done,{group_require_1_SUITE,end_per_suite,ok}},
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}
@@ -627,11 +645,11 @@ test_events(require_default) ->
[{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g4,[]}}},
{?eh,tc_done,{group_require_1_SUITE,
{init_per_group,g4,[]},
- {skipped,{require_failed,{not_available,common3}}}}},
+ {auto_skipped,{require_failed,{not_available,common3}}}}},
{?eh,tc_auto_skip,{group_require_1_SUITE,t41,
{require_failed,{not_available,common3}}}},
{?eh,test_stats,{4,0,{0,1}}},
- {?eh,tc_auto_skip,{group_require_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{group_require_1_SUITE,{end_per_group,g4},
{require_failed,{not_available,common3}}}}],
[{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g5,[]}}},
@@ -654,17 +672,19 @@ test_events(require_default) ->
[{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g8,[]}}},
{?eh,tc_done,{group_require_1_SUITE,
{init_per_group,g8,[]},
- {skipped,{require_failed,{not_available,non_existing}}}}},
+ {auto_skipped,{require_failed,
+ {not_available,non_existing}}}}},
{?eh,tc_auto_skip,{group_require_1_SUITE,t81,
{require_failed,{not_available,non_existing}}}},
{?eh,test_stats,{8,0,{0,2}}},
- {?eh,tc_auto_skip,{group_require_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{group_require_1_SUITE,{end_per_group,g8},
{require_failed,{not_available,non_existing}}}}],
[{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g9,[]}}},
{?eh,tc_done,{group_require_1_SUITE,{init_per_group,g9,[]},ok}},
{?eh,tc_done,{group_require_1_SUITE,t91,
- {skipped,{require_failed,{not_available,non_existing}}}}},
+ {auto_skipped,{require_failed,
+ {not_available,non_existing}}}}},
{?eh,test_stats,{8,0,{0,3}}},
{?eh,tc_start,{group_require_1_SUITE,{end_per_group,g9,[]}}},
{?eh,tc_done,{group_require_1_SUITE,{end_per_group,g9,[]},ok}}],
@@ -676,11 +696,15 @@ test_events(require_default) ->
{?eh,tc_done,{group_require_1_SUITE,{end_per_group,g10,[]},ok}}],
[{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g11,[]}}},
- {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g11,[]},ok}},
- {?eh,tc_done,{group_require_1_SUITE,t111,ok}},
- {?eh,test_stats,{10,0,{0,3}}},
- {?eh,tc_start,{group_require_1_SUITE,{end_per_group,g11,[]}}},
- {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g11,[]},ok}}],
+ {?eh,tc_done,{group_require_1_SUITE,
+ {init_per_group,g11,[]},
+ {auto_skipped,{group0_failed,bad_return_value}}}},
+ {?eh,tc_auto_skip,{group_require_1_SUITE,t111,
+ {group0_failed,bad_return_value}}},
+ {?eh,test_stats,{9,0,{0,4}}},
+ {?eh,tc_auto_skip,{group_require_1_SUITE,
+ {end_per_group,g11},
+ {group0_failed,bad_return_value}}}],
{?eh,tc_done,{group_require_1_SUITE,end_per_suite,ok}},
{?eh,test_done,{'DEF','STOP_TIME'}},
@@ -714,11 +738,12 @@ test_events(require_no_ips) ->
[{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g4,[]}}},
{?eh,tc_done,{group_require_2_SUITE,{init_per_group,g4,[]},
- {skipped,{require_failed,{name_in_use,common2_alias,common2}}}}},
+ {auto_skipped,{require_failed,
+ {name_in_use,common2_alias,common2}}}}},
{?eh,tc_auto_skip,{group_require_2_SUITE,t41,
{require_failed,{name_in_use,common2_alias,common2}}}},
{?eh,test_stats,{4,0,{0,1}}},
- {?eh,tc_auto_skip,{group_require_2_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{group_require_2_SUITE,{end_per_group,g4},
{require_failed,{name_in_use,common2_alias,common2}}}}],
[{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g5,[]}}},
@@ -741,16 +766,18 @@ test_events(require_no_ips) ->
[{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g8,[]}}},
{?eh,tc_done,{group_require_2_SUITE,
{init_per_group,g8,[]},
- {skipped,{require_failed,{not_available,non_existing}}}}},
+ {auto_skipped,{require_failed,
+ {not_available,non_existing}}}}},
{?eh,tc_auto_skip,{group_require_2_SUITE,t81,
{require_failed,{not_available,non_existing}}}},
{?eh,test_stats,{8,0,{0,2}}},
- {?eh,tc_auto_skip,{group_require_2_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{group_require_2_SUITE,{end_per_group,g8},
{require_failed,{not_available,non_existing}}}}],
[{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g9,[]}}},
{?eh,tc_done,{group_require_2_SUITE,{init_per_group,g9,[]},ok}},
{?eh,tc_done,{group_require_2_SUITE,t91,
- {skipped,{require_failed,{not_available,non_existing}}}}},
+ {auto_skipped,{require_failed,
+ {not_available,non_existing}}}}},
{?eh,test_stats,{8,0,{0,3}}},
{?eh,tc_start,{group_require_2_SUITE,{end_per_group,g9,[]}}},
{?eh,tc_done,{group_require_2_SUITE,{end_per_group,g9,[]},ok}}],
@@ -760,13 +787,17 @@ test_events(require_no_ips) ->
{?eh,tc_done,{group_require_2_SUITE,t101,ok}},
{?eh,tc_start,{group_require_2_SUITE,{end_per_group,g10,[]}}},
{?eh,tc_done,{group_require_2_SUITE,{end_per_group,g10,[]},ok}}],
-
+
[{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g11,[]}}},
- {?eh,tc_done,{group_require_2_SUITE,{init_per_group,g11,[]},ok}},
- {?eh,tc_done,{group_require_2_SUITE,t111,ok}},
- {?eh,test_stats,{10,0,{0,3}}},
- {?eh,tc_start,{group_require_2_SUITE,{end_per_group,g11,[]}}},
- {?eh,tc_done,{group_require_2_SUITE,{end_per_group,g11,[]},ok}}],
+ {?eh,tc_done,{group_require_2_SUITE,
+ {init_per_group,g11,[]},
+ {auto_skipped,{group0_failed,bad_return_value}}}},
+ {?eh,tc_auto_skip,{group_require_2_SUITE,t111,
+ {group0_failed,bad_return_value}}},
+ {?eh,test_stats,{9,0,{0,4}}},
+ {?eh,tc_auto_skip,{group_require_2_SUITE,
+ {end_per_group,g11},
+ {group0_failed,bad_return_value}}}],
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}
@@ -799,11 +830,11 @@ test_events(require_no_ipg) ->
[{?eh,tc_start,{ct_framework,{init_per_group,g4,[{suite,group_require_3_SUITE}]}}},
{?eh,tc_done,{ct_framework,{init_per_group,g4,[{suite,group_require_3_SUITE}]},
- {skipped,{require_failed,{name_in_use,common2_alias,common2}}}}},
+ {auto_skipped,{require_failed,{name_in_use,common2_alias,common2}}}}},
{?eh,tc_auto_skip,{group_require_3_SUITE,t41,
{require_failed,{name_in_use,common2_alias,common2}}}},
{?eh,test_stats,{4,0,{0,1}}},
- {?eh,tc_auto_skip,{ct_framework,end_per_group,
+ {?eh,tc_auto_skip,{ct_framework,{end_per_group,g4},
{require_failed,{name_in_use,common2_alias,common2}}}}],
[{?eh,tc_start,{ct_framework,{init_per_group,g5,[{suite,group_require_3_SUITE}]}}},
@@ -825,16 +856,16 @@ test_events(require_no_ipg) ->
[{?eh,tc_start,{ct_framework,{init_per_group,g8,[{suite,group_require_3_SUITE}]}}},
{?eh,tc_done,{ct_framework,{init_per_group,g8,[{suite,group_require_3_SUITE}]},
- {skipped,{require_failed,{not_available,non_existing}}}}},
+ {auto_skipped,{require_failed,{not_available,non_existing}}}}},
{?eh,tc_auto_skip,{group_require_3_SUITE,t81,
{require_failed,{not_available,non_existing}}}},
{?eh,test_stats,{8,0,{0,2}}},
- {?eh,tc_auto_skip,{ct_framework,end_per_group,
+ {?eh,tc_auto_skip,{ct_framework,{end_per_group,g8},
{require_failed,{not_available,non_existing}}}}],
[{?eh,tc_start,{ct_framework,{init_per_group,g9,[{suite,group_require_3_SUITE}]}}},
{?eh,tc_done,{ct_framework,{init_per_group,g9,[{suite,group_require_3_SUITE}]},ok}},
{?eh,tc_done,{group_require_3_SUITE,t91,
- {skipped,{require_failed,{not_available,non_existing}}}}},
+ {auto_skipped,{require_failed,{not_available,non_existing}}}}},
{?eh,test_stats,{8,0,{0,3}}},
{?eh,tc_start,{ct_framework,{end_per_group,g9,[{suite,group_require_3_SUITE}]}}},
{?eh,tc_done,{ct_framework,{end_per_group,g9,[{suite,group_require_3_SUITE}]},ok}}],
@@ -844,13 +875,14 @@ test_events(require_no_ipg) ->
{?eh,tc_done,{group_require_3_SUITE,t101,ok}},
{?eh,tc_start,{ct_framework,{end_per_group,g10,[{suite,group_require_3_SUITE}]}}},
{?eh,tc_done,{ct_framework,{end_per_group,g10,[{suite,group_require_3_SUITE}]},ok}}],
-
+
[{?eh,tc_start,{ct_framework,{init_per_group,g11,[{suite,group_require_3_SUITE}]}}},
- {?eh,tc_done,{ct_framework,{init_per_group,g11,[{suite,group_require_3_SUITE}]},ok}},
- {?eh,tc_done,{group_require_3_SUITE,t111,ok}},
- {?eh,test_stats,{10,0,{0,3}}},
- {?eh,tc_start,{ct_framework,{end_per_group,g11,[{suite,group_require_3_SUITE}]}}},
- {?eh,tc_done,{ct_framework,{end_per_group,g11,[{suite,group_require_3_SUITE}]},ok}}],
+ {?eh,tc_done,{ct_framework,{init_per_group,g11,[{suite,group_require_3_SUITE}]},
+ {auto_skipped,{group0_failed,bad_return_value}}}},
+ {?eh,tc_auto_skip,{group_require_3_SUITE,t111,{group0_failed,bad_return_value}}},
+ {?eh,test_stats,{9,0,{0,4}}},
+ {?eh,tc_auto_skip,{ct_framework,{end_per_group,g11},
+ {group0_failed,bad_return_value}}}],
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}
diff --git a/lib/common_test/test/ct_groups_test_2_SUITE.erl b/lib/common_test/test/ct_groups_test_2_SUITE.erl
index 1b2ad12e2f..8b0de98709 100644
--- a/lib/common_test/test/ct_groups_test_2_SUITE.erl
+++ b/lib/common_test/test/ct_groups_test_2_SUITE.erl
@@ -171,16 +171,20 @@ test_events(missing_conf) ->
{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
{?eh,start_info,{1,1,2}},
- {?eh,tc_start,{ct_framework,{init_per_group,group1,[]}}},
- {?eh,tc_done,{ct_framework,{init_per_group,group1,[]},ok}},
+ {?eh,tc_start,{ct_framework,{init_per_group,group1,
+ [{suite,missing_conf_SUITE}]}}},
+ {?eh,tc_done,{ct_framework,{init_per_group,group1,
+ [{suite,missing_conf_SUITE}]},ok}},
{?eh,tc_start,{missing_conf_SUITE,tc1}},
{?eh,tc_done,{missing_conf_SUITE,tc1,ok}},
{?eh,test_stats,{1,0,{0,0}}},
{?eh,tc_start,{missing_conf_SUITE,tc2}},
{?eh,tc_done,{missing_conf_SUITE,tc2,ok}},
{?eh,test_stats,{2,0,{0,0}}},
- {?eh,tc_start,{ct_framework,{end_per_group,group1,[]}}},
- {?eh,tc_done,{ct_framework,{end_per_group,group1,[]},ok}},
+ {?eh,tc_start,{ct_framework,{end_per_group,group1,
+ [{suite,missing_conf_SUITE}]}}},
+ {?eh,tc_done,{ct_framework,{end_per_group,group1,
+ [{suite,missing_conf_SUITE}]},ok}},
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}
];
diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl
index 596bfe3ff0..083c87b49e 100644
--- a/lib/common_test/test/ct_hooks_SUITE.erl
+++ b/lib/common_test/test/ct_hooks_SUITE.erl
@@ -749,11 +749,10 @@ test_events(skip_pre_suite_cth) ->
{?eh,cth,{'_',on_tc_skip,
[init_per_suite,{tc_user_skip,{skipped,"Test skip"}},[]]}},
- {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}},
- {?eh,cth,{'_',on_tc_skip,[test_case,{tc_auto_skip,"Test skip"},[]]}},
+ {?eh,tc_user_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}},
+ {?eh,cth,{'_',on_tc_skip,[test_case,{tc_user_skip,"Test skip"},[]]}},
- {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}},
- {?eh,cth,{'_',on_tc_skip,[end_per_suite,{tc_auto_skip,"Test skip"},[]]}},
+ {?eh,tc_user_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}},
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,cth, {'_',terminate,[[]]}},
@@ -773,11 +772,10 @@ test_events(skip_post_suite_cth) ->
{?eh,cth,{'_',on_tc_skip,
[init_per_suite,{tc_user_skip,{skipped,"Test skip"}},[]]}},
- {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}},
- {?eh,cth,{'_',on_tc_skip,[test_case,{tc_auto_skip,"Test skip"},[]]}},
+ {?eh,tc_user_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}},
+ {?eh,cth,{'_',on_tc_skip,[test_case,{tc_user_skip,"Test skip"},[]]}},
- {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}},
- {?eh,cth,{'_',on_tc_skip,[end_per_suite,{tc_auto_skip,"Test skip"},[]]}},
+ {?eh,tc_user_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}},
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,cth,{'_',terminate,[[]]}},
@@ -1219,11 +1217,15 @@ test_events(cth_log) ->
{?eh,tc_start,{cth_log_SUITE,init_per_suite}},
{parallel,
- [{?eh,tc_start,{ct_framework,{init_per_group,g1,[parallel]}}},
- {?eh,tc_done,{ct_framework,{init_per_group,g1,[parallel]},ok}},
+ [{?eh,tc_start,{ct_framework,{init_per_group,g1,
+ [{suite,cth_log_SUITE},parallel]}}},
+ {?eh,tc_done,{ct_framework,{init_per_group,g1,
+ [{suite,cth_log_SUITE},parallel]},ok}},
{?eh,test_stats,{30,0,{0,0}}},
- {?eh,tc_start,{ct_framework,{end_per_group,g1,[parallel]}}},
- {?eh,tc_done,{ct_framework,{end_per_group,g1,[parallel]},ok}}]},
+ {?eh,tc_start,{ct_framework,{end_per_group,g1,
+ [{suite,cth_log_SUITE},parallel]}}},
+ {?eh,tc_done,{ct_framework,{end_per_group,g1,
+ [{suite,cth_log_SUITE},parallel]},ok}}]},
{?eh,tc_done,{cth_log_SUITE,end_per_suite,ok}},
{?eh,test_done,{'DEF','STOP_TIME'}},
diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl
index 6ee7fdd6f6..2bcfeeec0c 100644
--- a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl
+++ b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl
@@ -84,7 +84,9 @@ all() ->
no_host,
no_port,
invalid_opt,
+ timeout_close_session,
get,
+ timeout_get,
get_xpath,
get_config,
get_config_xpath,
@@ -143,11 +145,10 @@ init_per_suite(Config) ->
end.
end_per_suite(Config) ->
- PrivDir = ?config(priv_dir, Config),
?NS:stop(?config(server,Config)),
ssh:stop(),
crypto:stop(),
- remove_id_keys(PrivDir),
+ remove_id_keys(Config),
Config.
hello(Config) ->
@@ -344,6 +345,15 @@ invalid_opt(Config) ->
{error,{invalid_option,{some_other_opt,true}}} = ct_netconfc:open(Opts2),
ok.
+timeout_close_session(Config) ->
+ DataDir = ?config(data_dir,Config),
+ {ok,Client} = open_success(DataDir),
+ ?NS:expect('close-session'),
+ true = erlang:is_process_alive(Client),
+ {error,timeout} = ct_netconfc:close_session(Client,1000),
+ false = erlang:is_process_alive(Client),
+ ok.
+
get(Config) ->
DataDir = ?config(data_dir,Config),
{ok,Client} = open_success(DataDir),
@@ -354,6 +364,15 @@ get(Config) ->
?ok = ct_netconfc:close_session(Client),
ok.
+timeout_get(Config) ->
+ DataDir = ?config(data_dir,Config),
+ {ok,Client} = open_success(DataDir),
+ ?NS:expect('get'),
+ {error,timeout} = ct_netconfc:get(Client,{server,[{xmlns,"myns"}],[]},1000),
+ ?NS:expect_do_reply('close-session',close,ok),
+ ?ok = ct_netconfc:close_session(Client),
+ ok.
+
get_xpath(Config) ->
DataDir = ?config(data_dir,Config),
{ok,Client} = open_success(DataDir),
@@ -698,7 +717,7 @@ timeout_receive_chunked_data(Config) ->
?ok = ct_netconfc:close_session(Client),
ok.
-%% Same as receive_chunked_data, but timeout waiting for last part.
+%% Same as receive_chunked_data, but close while waiting for last part.
close_while_waiting_for_chunked_data(Config) ->
DataDir = ?config(data_dir,Config),
{ok,Client} = open_success(DataDir),
diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl
index 09217f60a3..fb0734d48e 100644
--- a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl
+++ b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl
@@ -98,8 +98,9 @@ start(Dir) ->
%% Stop the netconf server
stop(Pid) ->
- Pid ! {stop,self()},
- receive stopped -> ok end.
+ Ref = erlang:monitor(process,Pid),
+ Pid ! stop,
+ receive {'DOWN',Ref,process,Pid,_} -> ok end.
%% Set the session id for the hello message.
%% If this is not called prior to starting the session, no hello
@@ -177,9 +178,9 @@ init_server(Dir) ->
loop(Daemon) ->
receive
- {stop,From} ->
+ stop ->
ssh:stop_daemon(Daemon),
- From ! stopped;
+ ok;
{table_trans,Fun,Args,From} ->
%% Simple transaction mechanism for ets table
R = apply(Fun,Args),
diff --git a/lib/common_test/test/ct_pre_post_test_io_SUITE.erl b/lib/common_test/test/ct_pre_post_test_io_SUITE.erl
index 84341a0b99..5de1ecc2bd 100644
--- a/lib/common_test/test/ct_pre_post_test_io_SUITE.erl
+++ b/lib/common_test/test/ct_pre_post_test_io_SUITE.erl
@@ -89,28 +89,31 @@ pre_post_io(Config) ->
%%!--------------------------------------------------------------------
spawn(fun() ->
+ ct:pal("CONTROLLER: Started!", []),
%% --- test run 1 ---
- ct:sleep(3000),
- ct_test_support:ct_rpc({cth_log_redirect,
- handle_remote_events,
- [true]}, Config),
- ct:sleep(2000),
- io:format(user, "Starting test run!~n", []),
- ct_test_support:ct_rpc({cth_ctrl,proceed,[]}, Config),
- ct:sleep(6000),
- io:format(user, "Finishing off!~n", []),
- ct_test_support:ct_rpc({cth_ctrl,proceed,[]}, Config),
+ timer:sleep(3000),
+ ct:pal("CONTROLLER: Handle remote events = true", []),
+ ok = ct_test_support:ct_rpc({cth_log_redirect,
+ handle_remote_events,
+ [true]}, Config),
+ timer:sleep(2000),
+ ct:pal("CONTROLLER: Proceeding with test run #1!", []),
+ ok = ct_test_support:ct_rpc({cth_ctrl,proceed,[]}, Config),
+ timer:sleep(6000),
+ ct:pal("CONTROLLER: Proceeding with shutdown #1!", []),
+ ok = ct_test_support:ct_rpc({cth_ctrl,proceed,[]}, Config),
%% --- test run 2 ---
- ct:sleep(3000),
- ct_test_support:ct_rpc({cth_log_redirect,
- handle_remote_events,
- [true]}, Config),
- ct:sleep(2000),
- io:format(user, "Starting test run!~n", []),
- ct_test_support:ct_rpc({cth_ctrl,proceed,[]}, Config),
- ct:sleep(6000),
- io:format(user, "Finishing off!~n", []),
- ct_test_support:ct_rpc({cth_ctrl,proceed,[]}, Config)
+ timer:sleep(3000),
+ ct:pal("CONTROLLER: Handle remote events = true", []),
+ ok = ct_test_support:ct_rpc({cth_log_redirect,
+ handle_remote_events,
+ [true]}, Config),
+ timer:sleep(2000),
+ ct:pal("CONTROLLER: Proceeding with test run #2!", []),
+ ok = ct_test_support:ct_rpc({cth_ctrl,proceed,[]}, Config),
+ timer:sleep(6000),
+ ct:pal("CONTROLLER: Proceeding with shutdown #2!", []),
+ ok = ct_test_support:ct_rpc({cth_ctrl,proceed,[]}, Config)
end),
ct_test_support:run(Opts, Config),
Events = ct_test_support:get_events(ERPid, Config),
diff --git a/lib/common_test/test/ct_pre_post_test_io_SUITE_data/cth_ctrl.erl b/lib/common_test/test/ct_pre_post_test_io_SUITE_data/cth_ctrl.erl
index c8c08a5735..2ba991fc61 100644
--- a/lib/common_test/test/ct_pre_post_test_io_SUITE_data/cth_ctrl.erl
+++ b/lib/common_test/test/ct_pre_post_test_io_SUITE_data/cth_ctrl.erl
@@ -28,7 +28,8 @@
%%%===================================================================
proceed() ->
- ?MODULE ! proceed.
+ ?MODULE ! proceed,
+ ok.
%%--------------------------------------------------------------------
%% Hook functions
diff --git a/lib/common_test/test/ct_repeat_1_SUITE.erl b/lib/common_test/test/ct_repeat_1_SUITE.erl
index 090002d0c2..98eaa28763 100644
--- a/lib/common_test/test/ct_repeat_1_SUITE.erl
+++ b/lib/common_test/test/ct_repeat_1_SUITE.erl
@@ -229,7 +229,7 @@ test_events(repeat_cs_and_grs) ->
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',fails_on_purpose}}}}},
{?eh,test_stats,{3,1,{0,1}}},
- {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init},
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',fails_on_purpose}}}}}],
{?eh,test_stats,{4,1,{0,1}}},
@@ -251,7 +251,7 @@ test_events(repeat_cs_and_grs) ->
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',fails_on_purpose}}}}},
{?eh,test_stats,{7,2,{0,2}}},
- {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init},
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',fails_on_purpose}}}}}],
{?eh,test_stats,{8,2,{0,2}}},
@@ -311,7 +311,7 @@ test_events(repeat_seq) ->
{init_per_group,gr_fail_init,[]},
{failed,{error,fails_on_purpose}}}},
{?eh,test_stats,{7,2,{0,5}}},
- {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init},
{failed,
{repeat_1_SUITE,init_per_group,
{'EXIT',fails_on_purpose}}}}}],
@@ -410,7 +410,7 @@ test_events(repeat_gr_until_any_ok) ->
{init_per_group,gr_fail_init,[]},
{failed,{error,fails_on_purpose}}}},
{?eh,test_stats,{1,1,{0,1}}},
- {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init},
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',fails_on_purpose}}}}}],
{?eh,test_stats,{1,2,{0,1}}},
@@ -452,7 +452,7 @@ test_events(repeat_gr_until_any_ok) ->
{init_per_group,gr_fail_init,[]},
{failed,{error,fails_on_purpose}}}},
{?eh,test_stats,{5,6,{0,3}}},
- {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init},
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',fails_on_purpose}}}}}],
{?eh,tc_done,{repeat_1_SUITE,
@@ -469,7 +469,7 @@ test_events(repeat_gr_until_any_ok) ->
{init_per_group,gr_fail_init,[]},
{failed,{error,fails_on_purpose}}}},
{?eh,test_stats,{7,7,{0,4}}},
- {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init},
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',fails_on_purpose}}}}}],
{?eh,tc_done,{repeat_1_SUITE,
@@ -770,7 +770,7 @@ test_events(repeat_gr_until_any_fail) ->
{'EXIT',failing_this_time}}}}},
{?eh,test_stats,{14,0,{0,1}}},
{?eh,tc_auto_skip,
- {repeat_1_SUITE,end_per_group,
+ {repeat_1_SUITE,{end_per_group,gr_ok_then_fail_init},
{failed,
{repeat_1_SUITE,init_per_group,
{'EXIT',failing_this_time}}}}}],
@@ -967,7 +967,7 @@ test_events(repeat_gr_until_all_ok) ->
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',failing_this_time}}}}},
{?eh,test_stats,{7,1,{0,1}}},
- {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init_then_ok},
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',failing_this_time}}}}}],
{?eh,tc_done,{repeat_1_SUITE,tc_ok_1,ok}},
@@ -1083,7 +1083,7 @@ test_events(repeat_gr_until_all_fail) ->
{init_per_group,gr_fail_init,[]},
{failed,{error,fails_on_purpose}}}},
{?eh,test_stats,{0,1,{0,1}}},
- {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init},
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',fails_on_purpose}}}}}],
{?eh,tc_done,{repeat_1_SUITE,tc_ok_then_fail_1,ok}},
@@ -1102,7 +1102,7 @@ test_events(repeat_gr_until_all_fail) ->
{init_per_group,gr_fail_init,[]},
{failed,{error,fails_on_purpose}}}},
{?eh,test_stats,{2,2,{0,2}}},
- {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init},
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',fails_on_purpose}}}}}],
{?eh,tc_done,{repeat_1_SUITE,tc_ok_then_fail_1,
@@ -1134,7 +1134,7 @@ test_events(repeat_gr_until_all_fail) ->
{init_per_group,gr_ok_then_fail_init,[]},
{failed,{error,failing_this_time}}}},
{?eh,test_stats,{4,4,{0,3}}},
- {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_ok_then_fail_init},
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',failing_this_time}}}}}],
{?eh,tc_start,{repeat_1_SUITE,tc_fail_1}},
@@ -1293,7 +1293,7 @@ test_events(repeat_seq_until_any_fail) ->
{init_per_group,gr_ok_then_fail_init,[]},
{failed,{error,failing_this_time}}}},
{?eh,test_stats,{24,1,{0,5}}},
- {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_ok_then_fail_init},
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',failing_this_time}}}}}],
{?eh,tc_auto_skip,{repeat_1_SUITE,tc_ok_1,
@@ -1516,7 +1516,7 @@ test_events(repeat_shuffled_seq_until_any_fail) ->
{init_per_group,repeat_shuffled_seq_until_any_fail_5,
[{shuffle,'_'},{repeat_until_any_fail,2},
sequence]},ok}},
- [{?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group,
+ [{?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_ok_then_fail_init},
{failed,{repeat_1_SUITE,init_per_group,
{'EXIT',failing_this_time}}}}}],
{?eh,tc_start,{repeat_1_SUITE,
diff --git a/lib/common_test/test/ct_repeat_testrun_SUITE.erl b/lib/common_test/test/ct_repeat_testrun_SUITE.erl
index 35d67a10f2..bb2aba2c5a 100644
--- a/lib/common_test/test/ct_repeat_testrun_SUITE.erl
+++ b/lib/common_test/test/ct_repeat_testrun_SUITE.erl
@@ -36,7 +36,7 @@
-define(eh, ct_test_support_eh).
-define(skip_reason, "Repeated test stopped by force_stop option").
--define(skipped, {skipped, ?skip_reason}).
+-define(skipped, {auto_skipped, ?skip_reason}).
%% Timers used in this test.
@@ -78,7 +78,7 @@ init_per_suite(Config0) ->
{1,0,{0,0}} = ct_test_support:run(ct,run_test,[Opts2],Config),
%% Time the shortest testcase to use for offset
- {T0,{1,0,{0,0}}} = timer:tc(ct_test_support,run,[ct,run_test,[Opts1],Config]),
+ {_T0,{1,0,{0,0}}} = timer:tc(ct_test_support,run,[ct,run_test,[Opts1],Config]),
%% -2 is to ensure we hit inside the target test case and not after
% T = round(T0/1000000)-2,
@@ -222,7 +222,7 @@ until_force_stop_skip_rest_group(Config) when is_list(Config) ->
fun() ->
[_] = ct_test_support:run_ct_run_test(
Opts++[{until,until_str(?t3,1,Config)}],Config),
- 0 = ct_test_support:run_ct_script_start(
+ 1 = ct_test_support:run_ct_script_start(
Opts++[{until,until_str(?t3,1,Config)}],Config)
end,
ok = execute(ExecuteFun,
@@ -341,19 +341,18 @@ skip_first_tc1(Suite) ->
{?eh,tc_done,{Suite,tc1,ok}},
{?eh,test_stats,{'_',0,{0,0}}},
{?eh,tc_done,{Suite,tc2,?skipped}},
- {?eh,test_stats,{'_',0,{1,0}}},
+ {?eh,test_stats,{'_',0,{0,1}}},
{?eh,tc_done,{Suite,{init_per_group,g,[]},?skipped}},
{?eh,tc_auto_skip,{Suite,tc1,?skip_reason}},
- {?eh,test_stats,{'_',0,{1,1}}},
+ {?eh,test_stats,{'_',0,{0,2}}},
{?eh,tc_auto_skip,{Suite,tc2,?skip_reason}},
- {?eh,test_stats,{'_',0,{1,2}}},
- {?eh,tc_auto_skip,{Suite,end_per_group,?skip_reason}},
+ {?eh,test_stats,{'_',0,{0,3}}},
+ {?eh,tc_auto_skip,{Suite,{end_per_group,g},?skip_reason}},
{?eh,tc_done,{Suite,tc2,?skipped}},
- {?eh,test_stats,{'_',0,{2,2}}},
+ {?eh,test_stats,{'_',0,{0,4}}},
{?eh,tc_start,{Suite,end_per_suite}},
{?eh,tc_done,{Suite,end_per_suite,ok}}].
-
skip_tc1_in_group(Suite) ->
[{?eh,tc_start,{Suite,init_per_suite}},
{?eh,tc_done,{Suite,init_per_suite,ok}},
@@ -369,10 +368,10 @@ skip_tc1_in_group(Suite) ->
{?eh,tc_done,{Suite,tc1,ok}},
{?eh,test_stats,{'_',0,{0,0}}},
{?eh,tc_done,{Suite,tc2,?skipped}},
- {?eh,test_stats,{'_',0,{1,0}}},
+ {?eh,test_stats,{'_',0,{0,1}}},
{?eh,tc_start,{Suite,{end_per_group,g,[]}}},
{?eh,tc_done,{Suite,{end_per_group,g,[]},ok}}],
{?eh,tc_done,{Suite,tc2,?skipped}},
- {?eh,test_stats,{'_',0,{2,0}}},
+ {?eh,test_stats,{'_',0,{0,2}}},
{?eh,tc_start,{Suite,end_per_suite}},
{?eh,tc_done,{Suite,end_per_suite,ok}}].
diff --git a/lib/common_test/test/ct_sequence_1_SUITE.erl b/lib/common_test/test/ct_sequence_1_SUITE.erl
index 5facf90656..8c87236838 100644
--- a/lib/common_test/test/ct_sequence_1_SUITE.erl
+++ b/lib/common_test/test/ct_sequence_1_SUITE.erl
@@ -212,7 +212,7 @@ test_events(subgroup_init_fail) ->
{failed,{subgroups_1_SUITE,init_per_group,
{'EXIT',init_per_group_fails_on_purpose}}}}},
{?eh,test_stats,{0,0,{0,1}}},
- {?eh,tc_auto_skip,{subgroups_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{subgroups_1_SUITE,{end_per_group,fail_init},
{failed,{subgroups_1_SUITE,init_per_group,
{'EXIT',init_per_group_fails_on_purpose}}}}}],
{?eh,tc_auto_skip,{subgroups_1_SUITE,ok_tc,{group_result,fail_init,failed}}},
@@ -294,7 +294,7 @@ test_events(case_after_subgroup_fail_init) ->
{subgroups_1_SUITE,init_per_group,
{'EXIT',init_per_group_fails_on_purpose}}}}},
{?eh,test_stats,{0,0,{0,1}}},
- {?eh,tc_auto_skip,{subgroups_1_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{subgroups_1_SUITE,{end_per_group,fail_init},
{failed,
{subgroups_1_SUITE,init_per_group,
{'EXIT',init_per_group_fails_on_purpose}}}}}],
diff --git a/lib/common_test/test/ct_skip_SUITE.erl b/lib/common_test/test/ct_skip_SUITE.erl
index b8be55f43a..b0a6c839a2 100644
--- a/lib/common_test/test/ct_skip_SUITE.erl
+++ b/lib/common_test/test/ct_skip_SUITE.erl
@@ -59,7 +59,7 @@ end_per_testcase(TestCase, Config) ->
suite() -> [{ct_hooks,[ts_install_cth]}].
all() ->
- [auto_skip, user_skip].
+ [auto_skip, user_skip, testspec_skip].
groups() ->
[].
@@ -91,7 +91,8 @@ auto_skip(Config) when is_list(Config) ->
Join(DataDir, "auto_skip_8_SUITE"),
Join(DataDir, "auto_skip_9_SUITE"),
Join(DataDir, "auto_skip_10_SUITE"),
- Join(DataDir, "auto_skip_11_SUITE")
+ Join(DataDir, "auto_skip_11_SUITE"),
+ Join(DataDir, "auto_skip_12_SUITE")
],
{Opts,ERPid} = setup({suite,Suites}, Config),
@@ -116,7 +117,8 @@ user_skip(Config) when is_list(Config) ->
Join(DataDir, "user_skip_2_SUITE"),
Join(DataDir, "user_skip_3_SUITE"),
Join(DataDir, "user_skip_4_SUITE"),
- Join(DataDir, "user_skip_5_SUITE")],
+ Join(DataDir, "user_skip_5_SUITE"),
+ Join(DataDir, "user_skip_6_SUITE")],
{Opts,ERPid} = setup({suite,Suites}, Config),
ok = ct_test_support:run(Opts, Config),
@@ -131,8 +133,56 @@ user_skip(Config) when is_list(Config) ->
ok = ct_test_support:verify_events(TestEvents, Events, Config).
%%%-----------------------------------------------------------------
+%%%
+testspec_skip(Config) when is_list(Config) ->
+ TestDir = filename:join(?config(data_dir, Config),
+ filename:join("skip", "test")),
+ TestSpec1 = [{suites, TestDir, user_skip_7_SUITE},
+ {skip_cases, TestDir, user_skip_7_SUITE, [tc1,tc3], "SKIPPED"}],
+
+ TestSpec2 = [{suites, TestDir, user_skip_7_SUITE},
+ {skip_groups, TestDir, user_skip_7_SUITE, ptop1, "SKIPPED"}],
+
+ TestSpec3 = [{suites, TestDir, user_skip_7_SUITE},
+ {skip_groups, TestDir, user_skip_7_SUITE, psub1, "SKIPPED"}],
+
+ TestSpec4 = [{suites, TestDir, user_skip_7_SUITE},
+ {skip_suites, TestDir, user_skip_7_SUITE, "SKIPPED"}],
+
+ TestSpec5 = [{groups, TestDir, user_skip_6_SUITE, ptop1},
+ {skip_groups, TestDir, user_skip_6_SUITE, psub1, "SKIPPED"}],
+
+ {Opts,ERPid} = setup_testspec([{ts1,TestSpec1},
+ {ts2,TestSpec2},
+ {ts3,TestSpec3},
+ {ts4,TestSpec4},
+ {ts5,TestSpec5}], Config),
+
+ ok = ct_test_support:run(Opts, Config),
+
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(testspec_skip,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config),
+ Opts),
+
+ TestEvents = events_to_check(testspec_skip),
+ ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
+%%%-----------------------------------------------------------------
%%% HELP FUNCTIONS
%%%-----------------------------------------------------------------
+setup_testspec(TestSpecs, Config) ->
+ SpecFiles =
+ [begin SpecFile = filename:join(?config(priv_dir, Config),
+ atom_to_list(SpecName)++".spec"),
+ {ok,Dev} = file:open(SpecFile, [write]),
+ [io:format(Dev, "~p.~n", [Term]) || Term <- TestSpec],
+ file:close(Dev),
+ SpecFile
+ end || {SpecName,TestSpec} <- TestSpecs],
+ setup({spec,SpecFiles}, Config).
setup(Test, Config) ->
Opts0 = ct_test_support:get_opts(Config),
@@ -163,7 +213,7 @@ test_events(auto_skip) ->
[
{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
- {?eh,start_info,{11,11,34}},
+ {?eh,start_info,{12,12,43}},
{?eh,tc_start,{auto_skip_1_SUITE,init_per_suite}},
{?eh,tc_done,
@@ -196,8 +246,8 @@ test_events(auto_skip) ->
{?eh,tc_start,{auto_skip_3_SUITE,tc1}},
{?eh,tc_done,
{auto_skip_3_SUITE,tc1,
- {skipped,{failed,{auto_skip_3_SUITE,init_per_testcase,
- {{init_per_testcase,tc1,failed},'_'}}}}}},
+ {auto_skipped,{failed,{auto_skip_3_SUITE,init_per_testcase,
+ {'init_per_testcase for tc1 failed','_'}}}}}},
{?eh,test_stats,{0,0,{0,4}}},
{?eh,tc_start,{auto_skip_3_SUITE,tc2}},
{?eh,tc_done,{auto_skip_3_SUITE,tc2,ok}},
@@ -209,7 +259,7 @@ test_events(auto_skip) ->
{?eh,tc_done,{auto_skip_4_SUITE,init_per_suite,ok}},
{?eh,tc_start,{auto_skip_4_SUITE,tc1}},
{?eh,tc_done,{auto_skip_4_SUITE,tc1,
- {skipped,{failed,{auto_skip_4_SUITE,init_per_testcase,
+ {auto_skipped,{failed,{auto_skip_4_SUITE,init_per_testcase,
{timetrap_timeout,1000}}}}}},
{?eh,test_stats,{1,0,{0,5}}},
{?eh,tc_start,{auto_skip_4_SUITE,tc2}},
@@ -232,8 +282,9 @@ test_events(auto_skip) ->
{'EXIT',{group,g1,failed}}}}}},
{?eh,test_stats,{2,0,{0,7}}},
{?eh,tc_auto_skip,
- {auto_skip_5_SUITE,end_per_group,{failed,{auto_skip_5_SUITE,init_per_group,
- {'EXIT',{group,g1,failed}}}}}}],
+ {auto_skip_5_SUITE,{end_per_group,g1},
+ {failed,{auto_skip_5_SUITE,init_per_group,
+ {'EXIT',{group,g1,failed}}}}}}],
{?eh,tc_start,{auto_skip_5_SUITE,end_per_suite}},
{?eh,tc_done,{auto_skip_5_SUITE,end_per_suite,ok}},
@@ -260,8 +311,9 @@ test_events(auto_skip) ->
{'EXIT',{group,g1,failed}}}}}},
{?eh,test_stats,{2,0,{0,11}}},
{?eh,tc_auto_skip,
- {auto_skip_6_SUITE,end_per_group,{failed,{auto_skip_6_SUITE,init_per_group,
- {'EXIT',{group,g1,failed}}}}}}],
+ {auto_skip_6_SUITE,{end_per_group,g1},
+ {failed,{auto_skip_6_SUITE,init_per_group,
+ {'EXIT',{group,g1,failed}}}}}}],
[{?eh,tc_start,{auto_skip_6_SUITE,{init_per_group,g3,[]}}},
{?eh,tc_done,{auto_skip_6_SUITE,{init_per_group,g3,[]},ok}},
@@ -280,8 +332,9 @@ test_events(auto_skip) ->
{'EXIT',{group,g4,failed}}}}}},
{?eh,test_stats,{3,0,{0,13}}},
{?eh,tc_auto_skip,
- {auto_skip_6_SUITE,end_per_group,{failed,{auto_skip_6_SUITE,init_per_group,
- {'EXIT',{group,g4,failed}}}}}}],
+ {auto_skip_6_SUITE,{end_per_group,g4},
+ {failed,{auto_skip_6_SUITE,init_per_group,
+ {'EXIT',{group,g4,failed}}}}}}],
{?eh,tc_start,{auto_skip_6_SUITE,tc2}},
{?eh,tc_done,{auto_skip_6_SUITE,tc2,ok}},
{?eh,test_stats,{4,0,{0,13}}},
@@ -320,7 +373,7 @@ test_events(auto_skip) ->
{?eh,test_stats,{5,0,{0,17}}},
{?eh,tc_start,{auto_skip_9_SUITE,tc2}},
{?eh,tc_done,{auto_skip_9_SUITE,tc2,
- {skipped,{failed,{auto_skip_9_SUITE,init_per_testcase,bad_return}}}}},
+ {auto_skipped,{failed,{auto_skip_9_SUITE,init_per_testcase,bad_return}}}}},
{?eh,test_stats,{5,0,{0,18}}},
[{?eh,tc_start,{auto_skip_9_SUITE,{init_per_group,g1,[]}}},
@@ -343,7 +396,7 @@ test_events(auto_skip) ->
{?eh,tc_done,{auto_skip_9_SUITE,{init_per_group,g3,[]},ok}},
{?eh,tc_start,{auto_skip_9_SUITE,tc5}},
{?eh,tc_done,{auto_skip_9_SUITE,tc5,
- {skipped,{failed,{auto_skip_9_SUITE,init_per_testcase,bad_return}}}}},
+ {auto_skipped,{failed,{auto_skip_9_SUITE,init_per_testcase,bad_return}}}}},
{?eh,test_stats,{7,0,{0,19}}},
{?eh,tc_start,{auto_skip_9_SUITE,{end_per_group,g3,[]}}},
{?eh,tc_done,{auto_skip_9_SUITE,{end_per_group,g3,[]},ok}}],
@@ -363,7 +416,7 @@ test_events(auto_skip) ->
{?eh,tc_start,{auto_skip_9_SUITE,tc8}},
{?eh,tc_done,
{auto_skip_9_SUITE,tc8,
- {skipped,{failed,{auto_skip_9_SUITE,init_per_testcase,
+ {auto_skipped,{failed,{auto_skip_9_SUITE,init_per_testcase,
{{badmatch,undefined},'_'}}}}}},
{?eh,tc_start,
{auto_skip_9_SUITE,{end_per_group,g5,[parallel]}}},
@@ -383,26 +436,26 @@ test_events(auto_skip) ->
{?eh,tc_start,{auto_skip_10_SUITE,init_per_suite}},
{?eh,tc_done,{auto_skip_10_SUITE,init_per_suite,
- {skipped,
- {require_failed_in_suite0,
- {not_available,undefined_config_variable}}}}},
- {?eh,tc_auto_skip,
- {auto_skip_10_SUITE,tc1,
- {require_failed_in_suite0,{not_available,undefined_config_variable}}}},
+ {auto_skipped,{require_failed_in_suite0,
+ {not_available,undefined_config_variable}}}}},
+ {?eh,tc_auto_skip,{auto_skip_10_SUITE,tc1,
+ {require_failed_in_suite0,
+ {not_available,undefined_config_variable}}}},
{?eh,test_stats,{9,0,{0,21}}},
- {?eh,tc_auto_skip,
- {auto_skip_10_SUITE,tc2,
- {require_failed_in_suite0,{not_available,undefined_config_variable}}}},
+ {?eh,tc_auto_skip,{auto_skip_10_SUITE,tc2,
+ {require_failed_in_suite0,
+ {not_available,undefined_config_variable}}}},
{?eh,test_stats,{9,0,{0,22}}},
- {?eh,tc_auto_skip,
- {auto_skip_10_SUITE,end_per_suite,
- {require_failed_in_suite0,{not_available,undefined_config_variable}}}},
+ {?eh,tc_auto_skip,{auto_skip_10_SUITE,end_per_suite,
+ {require_failed_in_suite0,
+ {not_available,undefined_config_variable}}}},
{?eh,tc_start,{auto_skip_11_SUITE,init_per_suite}},
{?eh,tc_done,{auto_skip_11_SUITE,init_per_suite,ok}},
{?eh,tc_start,{auto_skip_11_SUITE,tc1}},
{?eh,tc_done,{auto_skip_11_SUITE,tc1,
- {skipped,{require_failed,{not_available,undefined_config_variable}}}}},
+ {auto_skipped,{require_failed,
+ {not_available,undefined_config_variable}}}}},
{?eh,test_stats,{9,0,{0,23}}},
{?eh,tc_start,{auto_skip_11_SUITE,tc2}},
{?eh,tc_done,{auto_skip_11_SUITE,tc2,ok}},
@@ -421,7 +474,8 @@ test_events(auto_skip) ->
{?eh,tc_start,{auto_skip_11_SUITE,tc3}},
{?eh,tc_done,
{auto_skip_11_SUITE,tc3,
- {skipped,{require_failed,{not_available,undefined_config_variable}}}}},
+ {auto_skipped,{require_failed,
+ {not_available,undefined_config_variable}}}}},
{?eh,test_stats,{10,0,{0,24}}},
{?eh,tc_start,
{auto_skip_11_SUITE,{end_per_group,g2,[parallel]}}},
@@ -434,6 +488,75 @@ test_events(auto_skip) ->
{?eh,tc_start,{auto_skip_11_SUITE,end_per_suite}},
{?eh,tc_done,{auto_skip_11_SUITE,end_per_suite,ok}},
+
+ {?eh,tc_start,{auto_skip_12_SUITE,init_per_suite}},
+ {?eh,tc_done,{auto_skip_12_SUITE,init_per_suite,ok}},
+
+ [{?eh,tc_start,{ct_framework,{init_per_group,g1,
+ [{suite,auto_skip_12_SUITE}]}}},
+ {?eh,tc_done,{ct_framework,{init_per_group,g1,
+ [{suite,auto_skip_12_SUITE}]},
+ {auto_skipped,
+ {require_failed,{not_available,unknown_variable_g1}}}}},
+ {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc1,
+ {require_failed,{not_available,unknown_variable_g1}}}},
+ {?eh,test_stats,{10,0,{0,25}}},
+ {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc2,
+ {require_failed,{not_available,unknown_variable_g1}}}},
+ {?eh,test_stats,{10,0,{0,26}}},
+ {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3,
+ {require_failed,{not_available,unknown_variable_g1}}}},
+ {?eh,test_stats,{10,0,{0,27}}},
+ {?eh,tc_auto_skip,{ct_framework,{end_per_group,g1},
+ {require_failed,{not_available,unknown_variable_g1}}}}],
+
+ [{?eh,tc_start,{ct_framework,{init_per_group,g1,
+ [{suite,auto_skip_12_SUITE}]}}},
+ {?eh,tc_done,{ct_framework,{init_per_group,g1,
+ [{suite,auto_skip_12_SUITE}]},
+ {auto_skipped,
+ {require_failed,{not_available,unknown_variable_g1}}}}},
+ {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc1,
+ {require_failed,{not_available,unknown_variable_g1}}}},
+ {?eh,test_stats,{10,0,{0,28}}},
+ {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc2,
+ {require_failed,{not_available,unknown_variable_g1}}}},
+ {?eh,test_stats,{10,0,{0,29}}},
+ {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3,
+ {require_failed,{not_available,unknown_variable_g1}}}},
+ {?eh,test_stats,{10,0,{0,30}}},
+ {?eh,tc_auto_skip,{ct_framework,{end_per_group,g1},
+ {require_failed,{not_available,unknown_variable_g1}}}}],
+
+ [{?eh,tc_start,{ct_framework,{init_per_group,g3,
+ [{suite,auto_skip_12_SUITE}]}}},
+ {?eh,tc_done,{ct_framework,{init_per_group,g3,
+ [{suite,auto_skip_12_SUITE}]},ok}},
+ {?eh,tc_start,{auto_skip_12_SUITE,tc1}},
+ {?eh,tc_done,{auto_skip_12_SUITE,tc1,ok}},
+ {?eh,test_stats,{11,0,{0,30}}},
+ {?eh,tc_start,{auto_skip_12_SUITE,tc2}},
+ {?eh,tc_done,{auto_skip_12_SUITE,tc2,ok}},
+ {?eh,test_stats,{12,0,{0,30}}},
+ [{?eh,tc_start,{ct_framework,{init_per_group,g4,
+ [{suite,auto_skip_12_SUITE}]}}},
+ {?eh,tc_done,{ct_framework,{init_per_group,g4,
+ [{suite,auto_skip_12_SUITE}]},
+ {auto_skipped,
+ {require_failed,{not_available,unknown_variable_g4}}}}},
+ {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3,
+ {require_failed,{not_available,unknown_variable_g4}}}},
+ {?eh,test_stats,{12,0,{0,31}}},
+ {?eh,tc_auto_skip,{ct_framework,{end_per_group,g4},
+ {require_failed,{not_available,unknown_variable_g4}}}}],
+
+ {?eh,tc_start,{ct_framework,{end_per_group,g3,
+ [{suite,auto_skip_12_SUITE}]}}},
+ {?eh,tc_done,{ct_framework,{end_per_group,g3,
+ [{suite,auto_skip_12_SUITE}]},ok}}],
+
+ {?eh,tc_start,{auto_skip_12_SUITE,end_per_suite}},
+ {?eh,tc_done,{auto_skip_12_SUITE,end_per_suite,ok}},
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}
@@ -442,46 +565,46 @@ test_events(auto_skip) ->
test_events(user_skip) ->
[{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
- {?eh,start_info,{5,5,27}},
+ {?eh,start_info,{6,6,35}},
{?eh,tc_start,{user_skip_1_SUITE,init_per_suite}},
{?eh,tc_done,
{user_skip_1_SUITE,init_per_suite,{skipped,"Whole suite skipped"}}},
- {?eh,tc_auto_skip,
+ {?eh,tc_user_skip,
{user_skip_1_SUITE,tc1,"Whole suite skipped"}},
- {?eh,test_stats,{0,0,{0,1}}},
- {?eh,tc_auto_skip,
+ {?eh,test_stats,{0,0,{1,0}}},
+ {?eh,tc_user_skip,
{user_skip_1_SUITE,tc2,"Whole suite skipped"}},
- {?eh,test_stats,{0,0,{0,2}}},
- {?eh,tc_auto_skip,
+ {?eh,test_stats,{0,0,{2,0}}},
+ {?eh,tc_user_skip,
{user_skip_1_SUITE,tc3,"Whole suite skipped"}},
- {?eh,test_stats,{0,0,{0,3}}},
- {?eh,tc_auto_skip,
+ {?eh,test_stats,{0,0,{3,0}}},
+ {?eh,tc_user_skip,
{user_skip_1_SUITE,tc4,"Whole suite skipped"}},
- {?eh,test_stats,{0,0,{0,4}}},
- {?eh,tc_auto_skip,
+ {?eh,test_stats,{0,0,{4,0}}},
+ {?eh,tc_user_skip,
{user_skip_1_SUITE,end_per_suite,"Whole suite skipped"}},
{?eh,tc_start,{user_skip_2_SUITE,init_per_suite}},
{?eh,tc_done,{user_skip_2_SUITE,init_per_suite,ok}},
{?eh,tc_start,{user_skip_2_SUITE,tc1}},
{?eh,tc_done,{user_skip_2_SUITE,tc1,{skipped,{tc1,skipped}}}},
- {?eh,test_stats,{0,0,{1,4}}},
+ {?eh,test_stats,{0,0,{5,0}}},
[{?eh,tc_start,{user_skip_2_SUITE,{init_per_group,g1,[]}}},
{?eh,tc_done,{user_skip_2_SUITE,{init_per_group,g1,[]},ok}},
{?eh,tc_start,{user_skip_2_SUITE,tc2}},
{?eh,tc_done,{user_skip_2_SUITE,tc2,ok}},
- {?eh,test_stats,{1,0,{1,4}}},
+ {?eh,test_stats,{1,0,{5,0}}},
{?eh,tc_start,{user_skip_2_SUITE,tc3}},
{?eh,tc_done,{user_skip_2_SUITE,tc3,{skipped,{tc3,skipped}}}},
- {?eh,test_stats,{1,0,{2,4}}},
+ {?eh,test_stats,{1,0,{6,0}}},
{?eh,tc_start,{user_skip_2_SUITE,{end_per_group,g1,[]}}},
{?eh,tc_done,{user_skip_2_SUITE,{end_per_group,g1,[]},ok}}],
{?eh,tc_start,{user_skip_2_SUITE,tc4}},
{?eh,tc_done,{user_skip_2_SUITE,tc4,ok}},
- {?eh,test_stats,{2,0,{2,4}}},
+ {?eh,test_stats,{2,0,{6,0}}},
{?eh,tc_start,{user_skip_2_SUITE,end_per_suite}},
{?eh,tc_done,{user_skip_2_SUITE,end_per_suite,ok}},
@@ -489,16 +612,16 @@ test_events(user_skip) ->
{?eh,tc_done,{user_skip_3_SUITE,init_per_suite,ok}},
{?eh,tc_start,{user_skip_3_SUITE,tc1}},
{?eh,tc_done,{user_skip_3_SUITE,tc1,{skipped,"Test case skipped"}}},
- {?eh,test_stats,{2,0,{3,4}}},
+ {?eh,test_stats,{2,0,{7,0}}},
[{?eh,tc_start,{user_skip_3_SUITE,{init_per_group,g1,[]}}},
{?eh,tc_done,{user_skip_3_SUITE,{init_per_group,g1,[]},ok}},
{?eh,tc_start,{user_skip_3_SUITE,tc2}},
{?eh,tc_done,{user_skip_3_SUITE,tc2,ok}},
- {?eh,test_stats,{3,0,{3,4}}},
+ {?eh,test_stats,{3,0,{7,0}}},
{?eh,tc_start,{user_skip_3_SUITE,tc3}},
{?eh,tc_done,{user_skip_3_SUITE,tc3,{skipped,"Test case skipped"}}},
- {?eh,test_stats,{3,0,{4,4}}},
+ {?eh,test_stats,{3,0,{8,0}}},
{?eh,tc_start,{user_skip_3_SUITE,{end_per_group,g1,[]}}},
{?eh,tc_done,{user_skip_3_SUITE,{end_per_group,g1,[]},ok}}],
@@ -506,7 +629,7 @@ test_events(user_skip) ->
{?eh,tc_done,{user_skip_3_SUITE,tc4,
{skipped,{proc_info,{{current_function,{user_skip_3_SUITE,tc4,1}},
{initial_call,{erlang,apply,2}}}}}}},
- {?eh,test_stats,{3,0,{5,4}}},
+ {?eh,test_stats,{3,0,{9,0}}},
{?eh,tc_start,{user_skip_3_SUITE,end_per_suite}},
{?eh,tc_done,{user_skip_3_SUITE,end_per_suite,ok}},
@@ -515,63 +638,219 @@ test_events(user_skip) ->
[{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g1,[]}}},
{?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g1,[]},{skipped,"Group skipped"}}},
- {?eh,tc_auto_skip,{user_skip_4_SUITE,tc1,"Group skipped"}},
- {?eh,test_stats,{3,0,{5,5}}},
- {?eh,tc_auto_skip,{user_skip_4_SUITE,tc2,"Group skipped"}},
- {?eh,test_stats,{3,0,{5,6}}},
- {?eh,tc_auto_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}],
+ {?eh,tc_user_skip,{user_skip_4_SUITE,tc1,"Group skipped"}},
+ {?eh,test_stats,{3,0,{10,0}}},
+ {?eh,tc_user_skip,{user_skip_4_SUITE,tc2,"Group skipped"}},
+ {?eh,test_stats,{3,0,{11,0}}},
+ {?eh,tc_user_skip,{user_skip_4_SUITE,{end_per_group,g1},"Group skipped"}}],
[{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g2,[]}}},
{?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g2,[]},ok}},
{?eh,tc_start,{user_skip_4_SUITE,tc3}},
{?eh,tc_done,{user_skip_4_SUITE,tc3,ok}},
- {?eh,test_stats,{4,0,{5,6}}},
+ {?eh,test_stats,{4,0,{11,0}}},
{?eh,tc_start,{user_skip_4_SUITE,tc4}},
{?eh,tc_done,{user_skip_4_SUITE,tc4,ok}},
- {?eh,test_stats,{5,0,{5,6}}},
+ {?eh,test_stats,{5,0,{11,0}}},
{?eh,tc_start,{user_skip_4_SUITE,{end_per_group,g2,[]}}},
{?eh,tc_done,{user_skip_4_SUITE,{end_per_group,g2,[]},ok}}],
[{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g3,[]}}},
{?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g3,[]},{skipped,"Group skipped"}}},
- {?eh,tc_auto_skip,{user_skip_4_SUITE,tc5,"Group skipped"}},
- {?eh,tc_auto_skip,{user_skip_4_SUITE,tc6,"Group skipped"}},
- {?eh,tc_auto_skip,{user_skip_4_SUITE,tc7,"Group skipped"}},
- {?eh,tc_auto_skip,{user_skip_4_SUITE,tc8,"Group skipped"}},
- {?eh,test_stats,{5,0,{5,10}}},
- {?eh,tc_auto_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}],
+ {?eh,tc_user_skip,{user_skip_4_SUITE,tc5,"Group skipped"}},
+ {?eh,tc_user_skip,{user_skip_4_SUITE,tc6,"Group skipped"}},
+ {?eh,tc_user_skip,{user_skip_4_SUITE,tc7,"Group skipped"}},
+ {?eh,tc_user_skip,{user_skip_4_SUITE,tc8,"Group skipped"}},
+ {?eh,test_stats,{5,0,{15,0}}},
+ {?eh,tc_user_skip,{user_skip_4_SUITE,{end_per_group,g3},"Group skipped"}}],
[{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g5,[]}}},
{?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g5,[]},ok}},
{?eh,tc_start,{user_skip_4_SUITE,tc9}},
{?eh,tc_done,{user_skip_4_SUITE,tc9,ok}},
- {?eh,test_stats,{6,0,{5,10}}},
+ {?eh,test_stats,{6,0,{15,0}}},
[{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g6,[]}}},
{?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g6,[]},{skipped,"Group skipped"}}},
- {?eh,tc_auto_skip,{user_skip_4_SUITE,tc10,"Group skipped"}},
- {?eh,test_stats,{6,0,{5,11}}},
- {?eh,tc_auto_skip,{user_skip_4_SUITE,tc11,"Group skipped"}},
- {?eh,test_stats,{6,0,{5,12}}},
- {?eh,tc_auto_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}],
+ {?eh,tc_user_skip,{user_skip_4_SUITE,tc10,"Group skipped"}},
+ {?eh,test_stats,{6,0,{16,0}}},
+ {?eh,tc_user_skip,{user_skip_4_SUITE,tc11,"Group skipped"}},
+ {?eh,test_stats,{6,0,{17,0}}},
+ {?eh,tc_user_skip,{user_skip_4_SUITE,{end_per_group,g6},"Group skipped"}}],
{?eh,tc_start,{user_skip_4_SUITE,{end_per_group,g5,[]}}},
{?eh,tc_done,{user_skip_4_SUITE,{end_per_group,g5,[]},ok}}],
{?eh,tc_start,{user_skip_4_SUITE,end_per_suite}},
{?eh,tc_done,{user_skip_4_SUITE,end_per_suite,ok}},
- {ct_test_support_eh,tc_start,{user_skip_5_SUITE,init_per_suite}},
+ {?eh,tc_start,{user_skip_5_SUITE,init_per_suite}},
{?eh,tc_done,{user_skip_5_SUITE,init_per_suite,
{skipped,{bad,'Whole suite skipped'}}}},
- {?eh,tc_auto_skip,{user_skip_5_SUITE,tc1,{bad,'Whole suite skipped'}}},
- {?eh,test_stats,{6,0,{5,13}}},
- {?eh,tc_auto_skip,{user_skip_5_SUITE,tc2,{bad,'Whole suite skipped'}}},
- {?eh,test_stats,{6,0,{5,14}}},
- {?eh,tc_auto_skip,{user_skip_5_SUITE,tc3,{bad,'Whole suite skipped'}}},
- {?eh,test_stats,{6,0,{5,15}}},
- {?eh,tc_auto_skip,{user_skip_5_SUITE,tc4,{bad,'Whole suite skipped'}}},
- {?eh,test_stats,{6,0,{5,16}}},
- {?eh,tc_auto_skip,{user_skip_5_SUITE,end_per_suite,{bad,'Whole suite skipped'}}},
+ {?eh,tc_user_skip,{user_skip_5_SUITE,tc1,{bad,'Whole suite skipped'}}},
+ {?eh,test_stats,{6,0,{18,0}}},
+ {?eh,tc_user_skip,{user_skip_5_SUITE,tc2,{bad,'Whole suite skipped'}}},
+ {?eh,test_stats,{6,0,{19,0}}},
+ {?eh,tc_user_skip,{user_skip_5_SUITE,tc3,{bad,'Whole suite skipped'}}},
+ {?eh,test_stats,{6,0,{20,0}}},
+ {?eh,tc_user_skip,{user_skip_5_SUITE,tc4,{bad,'Whole suite skipped'}}},
+ {?eh,test_stats,{6,0,{21,0}}},
+ {?eh,tc_user_skip,{user_skip_5_SUITE,end_per_suite,{bad,'Whole suite skipped'}}},
+
+ {parallel,
+ [{?eh,tc_start,{user_skip_6_SUITE,{init_per_group,ptop1,[parallel]}}},
+ {?eh,tc_done,{user_skip_6_SUITE,
+ {init_per_group,ptop1,[parallel]},
+ {skipped,"Top group skipped"}}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc1,"Top group skipped"}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc3,"Top group skipped"}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc4,"Top group skipped"}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc2,"Top group skipped"}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,{end_per_group,ptop1},
+ "Top group skipped"}}]},
+
+ {parallel,
+ [{?eh,tc_start,{user_skip_6_SUITE,{init_per_group,ptop2,[parallel]}}},
+ {?eh,tc_done,{user_skip_6_SUITE,{init_per_group,ptop2,[parallel]},ok}},
+ {?eh,tc_start,{user_skip_6_SUITE,tc1}},
+ {?eh,tc_done,{user_skip_6_SUITE,tc1,ok}},
+
+ {parallel,
+ [{?eh,tc_start,{user_skip_6_SUITE,{init_per_group,psub2,[parallel]}}},
+ {?eh,tc_done,{user_skip_6_SUITE,
+ {init_per_group,psub2,[parallel]},
+ {skipped,"Sub group skipped"}}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc3,"Sub group skipped"}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc4,"Sub group skipped"}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,{end_per_group,psub2},
+ "Sub group skipped"}}]},
+
+ {?eh,tc_start,{user_skip_6_SUITE,tc2}},
+ {?eh,tc_done,{user_skip_6_SUITE,tc2,ok}},
+ {?eh,test_stats,{8,0,{27,0}}},
+ {?eh,tc_start,{user_skip_6_SUITE,{end_per_group,ptop2,[parallel]}}},
+ {?eh,tc_done,{user_skip_6_SUITE,{end_per_group,ptop2,[parallel]},ok}}]},
+
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}
+ ];
+
+test_events(testspec_skip) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,4}},
+ {?eh,tc_start,{ct_framework,init_per_suite}},
+ {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+ {parallel,
+ [{?eh,tc_start,
+ {user_skip_7_SUITE,{init_per_group,ptop1,[parallel]}}},
+ {?eh,tc_done,
+ {user_skip_7_SUITE,{init_per_group,ptop1,[parallel]},ok}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc1,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{1,0}}},
+ {parallel,
+ [{?eh,tc_start,
+ {user_skip_7_SUITE,{init_per_group,psub1,[parallel]}}},
+ {?eh,tc_done,
+ {user_skip_7_SUITE,{init_per_group,psub1,[parallel]},ok}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc3,"SKIPPED"}},
+ {?eh,tc_start,{user_skip_7_SUITE,tc4}},
+ {?eh,tc_done,{user_skip_7_SUITE,tc4,ok}},
+ {?eh,test_stats,{1,0,{2,0}}},
+ {?eh,tc_start,
+ {user_skip_7_SUITE,{end_per_group,psub1,[parallel]}}},
+ {?eh,tc_done,
+ {user_skip_7_SUITE,{end_per_group,psub1,[parallel]},ok}}]},
+ {?eh,tc_start,{user_skip_7_SUITE,tc2}},
+ {?eh,tc_done,{user_skip_7_SUITE,tc2,ok}},
+ {?eh,test_stats,{2,0,{2,0}}},
+ {?eh,tc_start,
+ {user_skip_7_SUITE,{end_per_group,ptop1,[parallel]}}},
+ {?eh,tc_done,
+ {user_skip_7_SUITE,{end_per_group,ptop1,[parallel]},ok}}]},
+ {?eh,tc_start,{ct_framework,end_per_suite}},
+ {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,4}},
+ {?eh,tc_start,{ct_framework,init_per_suite}},
+ {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,{init_per_group,ptop1},"SKIPPED"}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc1,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{1,0}}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc3,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{2,0}}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc4,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{3,0}}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc2,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{4,0}}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,{end_per_group,ptop1},"SKIPPED"}},
+ {?eh,tc_start,{ct_framework,end_per_suite}},
+ {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,4}},
+ {?eh,tc_start,{ct_framework,init_per_suite}},
+ {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+ {parallel,
+ [{?eh,tc_start,
+ {user_skip_7_SUITE,{init_per_group,ptop1,[parallel]}}},
+ {?eh,tc_done,
+ {user_skip_7_SUITE,{init_per_group,ptop1,[parallel]},ok}},
+ {?eh,tc_user_skip,
+ {user_skip_7_SUITE,{init_per_group,psub1},"SKIPPED"}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc3,"SKIPPED"}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc4,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{2,0}}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,{end_per_group,psub1},"SKIPPED"}},
+ {?eh,tc_start,{user_skip_7_SUITE,tc1}},
+ {?eh,tc_done,{user_skip_7_SUITE,tc1,ok}},
+ {?eh,tc_start,{user_skip_7_SUITE,tc2}},
+ {?eh,tc_done,{user_skip_7_SUITE,tc2,ok}},
+ {?eh,test_stats,{2,0,{2,0}}},
+ {?eh,tc_start,{user_skip_7_SUITE,{end_per_group,ptop1,[parallel]}}},
+ {?eh,tc_done,{user_skip_7_SUITE,{end_per_group,ptop1,[parallel]},ok}}]},
+ {?eh,tc_start,{ct_framework,end_per_suite}},
+ {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,0}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,all,"SKIPPED"}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,4}},
+ {?eh,tc_start,{ct_framework,init_per_suite}},
+ {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+ {parallel,
+ [{?eh,tc_start,{user_skip_6_SUITE,{init_per_group,ptop1,[parallel]}}},
+ {?eh,tc_done,{user_skip_6_SUITE,
+ {init_per_group,ptop1,[parallel]},
+ {skipped,"Top group skipped"}}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc1,"Top group skipped"}},
+ {?eh,test_stats,{0,0,{1,0}}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc3,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{2,0}}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc4,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{3,0}}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc2,"Top group skipped"}},
+ {?eh,test_stats,{0,0,{4,0}}},
+ {?eh,tc_user_skip,
+ {user_skip_6_SUITE,{end_per_group,ptop1},"Top group skipped"}}]},
+ {?eh,tc_start,{ct_framework,end_per_suite}},
+ {?eh,tc_done,{ct_framework,end_per_suite,ok}},
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}
].
+
+
diff --git a/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_12_SUITE.erl b/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_12_SUITE.erl
new file mode 100644
index 0000000000..a168afd386
--- /dev/null
+++ b/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_12_SUITE.erl
@@ -0,0 +1,121 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(auto_skip_12_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% Function: suite() -> Info
+%% Info = [tuple()]
+%%--------------------------------------------------------------------
+suite() ->
+ [].
+
+%%--------------------------------------------------------------------
+%% Function: init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% Function: end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% Function: group(Name) -> Info
+%% Info = [tuple()]
+%%--------------------------------------------------------------------
+group(g1) ->
+ [{require,unknown_variable_g1}];
+group(g4) ->
+ [{require,unknown_variable_g4}];
+group(_) ->
+ [].
+
+%%--------------------------------------------------------------------
+%% Function: init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% Function: end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% Function: groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%%--------------------------------------------------------------------
+groups() ->
+ [{g1,[],[tc1,tc2,{g2,[],[tc3]}]},
+ {g1,[],[tc1,tc2,{g2,[],[tc3]}]},
+ {g3,[],[tc1,tc2,{g4,[],[tc3]}]}].
+
+%%--------------------------------------------------------------------
+%% Function: all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%%--------------------------------------------------------------------
+all() ->
+ [{group,g1},
+ {group,g3}].
+
+%%--------------------------------------------------------------------
+%% Function: TestCase(Config0) ->
+%% ok | exit() | {skip,Reason} | {comment,Comment} |
+%% {save_config,Config1} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% Comment = term()
+%%--------------------------------------------------------------------
+tc1(_Config) ->
+ ok.
+
+tc2(_Config) ->
+ ok.
+
+tc3(_Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_3_SUITE.erl b/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_3_SUITE.erl
index cb64cb76c5..4ef9f50514 100644
--- a/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_3_SUITE.erl
+++ b/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_3_SUITE.erl
@@ -72,7 +72,7 @@ end_per_group(_GroupName, _Config) ->
%% Reason = term()
%%--------------------------------------------------------------------
init_per_testcase(tc1, _Config) ->
- exit({init_per_testcase,tc1,failed});
+ exit('init_per_testcase for tc1 failed');
init_per_testcase(_TestCase, Config) ->
Config.
diff --git a/lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_6_SUITE.erl b/lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_6_SUITE.erl
new file mode 100644
index 0000000000..5f0e5db6f2
--- /dev/null
+++ b/lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_6_SUITE.erl
@@ -0,0 +1,118 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(user_skip_6_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% Function: suite() -> Info
+%% Info = [tuple()]
+%%--------------------------------------------------------------------
+suite() ->
+ [{timetrap,{seconds,30}}].
+
+%%--------------------------------------------------------------------
+%% Function: init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_group(ptop1, Config) ->
+ {skip,"Top group skipped"};
+init_per_group(psub2, Config) ->
+ {skip,"Sub group skipped"};
+init_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% Function: end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% Function: init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% Function: end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% Function: groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%%--------------------------------------------------------------------
+groups() ->
+ [{ptop1,[parallel],[tc1,{psub1,[parallel],[tc3,tc4]},tc2]},
+ {ptop2,[parallel],[tc1,{psub2,[parallel],[tc3,tc4]},tc2]}].
+
+%%--------------------------------------------------------------------
+%% Function: all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%%--------------------------------------------------------------------
+all() ->
+ [{group,ptop1},{group,ptop2}].
+
+%%--------------------------------------------------------------------
+%% Function: TestCase(Config0) ->
+%% ok | exit() | {skip,Reason} | {comment,Comment} |
+%% {save_config,Config1} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% Comment = term()
+%%--------------------------------------------------------------------
+tc1(_) ->
+ ok.
+
+tc2(_) ->
+ ok.
+
+tc3(_) ->
+ ok.
+
+tc4(_) ->
+ ok.
diff --git a/lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_7_SUITE.erl b/lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_7_SUITE.erl
new file mode 100644
index 0000000000..82ce536a79
--- /dev/null
+++ b/lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_7_SUITE.erl
@@ -0,0 +1,113 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(user_skip_7_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% Function: suite() -> Info
+%% Info = [tuple()]
+%%--------------------------------------------------------------------
+suite() ->
+ [{timetrap,{seconds,30}}].
+
+%%--------------------------------------------------------------------
+%% Function: init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% Function: end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% Function: init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% Function: end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% Function: groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%%--------------------------------------------------------------------
+groups() ->
+ [{ptop1,[parallel],[tc1,{psub1,[parallel],[tc3,tc4]},tc2]}].
+
+%%--------------------------------------------------------------------
+%% Function: all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%%--------------------------------------------------------------------
+all() ->
+ [{group,ptop1}].
+
+%%--------------------------------------------------------------------
+%% Function: TestCase(Config0) ->
+%% ok | exit() | {skip,Reason} | {comment,Comment} |
+%% {save_config,Config1} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% Comment = term()
+%%--------------------------------------------------------------------
+tc1(_) ->
+ ok.
+
+tc2(_) ->
+ ok.
+
+tc3(_) ->
+ ok.
+
+tc4(_) ->
+ ok.
diff --git a/lib/common_test/test/ct_surefire_SUITE.erl b/lib/common_test/test/ct_surefire_SUITE.erl
index b86b47f0a2..c5e44682b0 100644
--- a/lib/common_test/test/ct_surefire_SUITE.erl
+++ b/lib/common_test/test/ct_surefire_SUITE.erl
@@ -182,7 +182,7 @@ test_events(_) ->
{?eh,test_stats,{1,1,{1,0}}},
{?eh,tc_start,{surefire_SUITE,tc_autoskip_require}},
{?eh,tc_done,{surefire_SUITE,tc_autoskip_require,
- {skipped,{require_failed,'_'}}}},
+ {auto_skipped,{require_failed,'_'}}}},
{?eh,test_stats,{1,1,{1,1}}},
[{?eh,tc_start,{surefire_SUITE,{init_per_group,g,[]}}},
{?eh,tc_done,{surefire_SUITE,{init_per_group,g,[]},ok}},
@@ -198,7 +198,7 @@ test_events(_) ->
{?eh,test_stats,{2,2,{2,1}}},
{?eh,tc_start,{surefire_SUITE,tc_autoskip_require}},
{?eh,tc_done,{surefire_SUITE,tc_autoskip_require,
- {skipped,{require_failed,'_'}}}},
+ {auto_skipped,{require_failed,'_'}}}},
{?eh,test_stats,{2,2,{2,2}}},
{?eh,tc_start,{surefire_SUITE,{end_per_group,g,[]}}},
{?eh,tc_done,{surefire_SUITE,{end_per_group,g,[]},ok}}],
@@ -210,7 +210,7 @@ test_events(_) ->
{surefire_SUITE,init_per_group,
{'EXIT',all_cases_should_be_skipped}}}}},
{?eh,test_stats,{2,2,{2,3}}},
- {?eh,tc_auto_skip,{surefire_SUITE,end_per_group,
+ {?eh,tc_auto_skip,{surefire_SUITE,{end_per_group,g_fail},
{failed,
{surefire_SUITE,init_per_group,
{'EXIT',all_cases_should_be_skipped}}}}}],
@@ -328,6 +328,7 @@ events_to_result([]) ->
result(ok) ->[];
result({skipped,_}) -> [s];
+result({auto_skipped,_}) -> [s];
result({failed,_}) -> [f].
%% Using the expected events' last test_stats element to produce the
diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE.erl
index 8e4852369d..9882fa980c 100644
--- a/lib/common_test/test/ct_test_server_if_1_SUITE.erl
+++ b/lib/common_test/test/ct_test_server_if_1_SUITE.erl
@@ -65,12 +65,12 @@ groups() ->
[].
init_per_group(_GroupName, Config) ->
- Config.
+ Config.
end_per_group(_GroupName, Config) ->
- Config.
+ Config.
+
-
%%--------------------------------------------------------------------
%% TEST CASES
@@ -104,7 +104,7 @@ ts_if_1(Config) when is_list(Config) ->
TestEvents = events_to_check(ts_if_1),
ok = ct_test_support:verify_events(TestEvents, Events, Config).
-
+
%%%-----------------------------------------------------------------
%%% HELP FUNCTIONS
@@ -112,17 +112,17 @@ ts_if_1(Config) when is_list(Config) ->
setup(Test, Config) ->
Opts0 = ct_test_support:get_opts(Config),
-% Level = ?config(trace_level, Config),
-% EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
-% Opts = Opts0 ++ [Test,{event_handler,{?eh,EvHArgs}}],
+ % Level = ?config(trace_level, Config),
+ % EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+ % Opts = Opts0 ++ [Test,{event_handler,{?eh,EvHArgs}}],
Opts = [Test | Opts0],
ERPid = ct_test_support:start_event_receiver(Config),
{Opts,ERPid}.
reformat(Events, EH) ->
ct_test_support:reformat(Events, EH).
-%reformat(Events, _EH) ->
-% Events.
+ %reformat(Events, _EH) ->
+ % Events.
%%%-----------------------------------------------------------------
%%% TEST EVENTS
@@ -140,14 +140,15 @@ test_events(ts_if_1) ->
[
{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
- {?eh,start_info,{10,6,26}},
+ {?eh,start_info,{10,8,25}},
+
{?eh,tc_start,{ts_if_1_SUITE,init_per_suite}},
{?eh,tc_done,{ts_if_1_SUITE,init_per_suite,ok}},
{?eh,tc_start,{ts_if_1_SUITE,tc1}},
- {?eh,tc_done,{ts_if_1_SUITE,tc1,{skipped,
- {failed,
- {ts_if_1_SUITE,init_per_testcase,
- {timetrap_timeout,2000}}}}}},
+ {?eh,tc_done,{ts_if_1_SUITE,tc1,
+ {auto_skipped,
+ {failed,
+ {ts_if_1_SUITE,init_per_testcase,{timetrap_timeout,2000}}}}}},
{?eh,test_stats,{0,0,{0,1}}},
{?eh,tc_start,{ts_if_1_SUITE,tc2}},
{?eh,tc_done,{ts_if_1_SUITE,tc2,
@@ -159,115 +160,162 @@ test_events(ts_if_1) ->
{?eh,tc_start,{ts_if_1_SUITE,tc4}},
{?eh,tc_done,{ts_if_1_SUITE,tc4,{failed,{error,failed_on_purpose}}}},
{?eh,test_stats,{1,2,{0,1}}},
- {?eh,tc_done,{ts_if_1_SUITE,tc5,{skipped,{sequence_failed,seq1,tc4}}}},
- {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_done,{ts_if_1_SUITE,tc5,{auto_skipped,{sequence_failed,seq1,tc4}}}},
+ {?eh,test_stats,{1,2,{0,2}}},
[{?eh,tc_start,{ts_if_1_SUITE,{init_per_group,seq2,[sequence]}}},
{?eh,tc_done,{ts_if_1_SUITE,{init_per_group,seq2,[sequence]},ok}},
{?eh,tc_start,{ts_if_1_SUITE,tc4}},
{?eh,tc_done,{ts_if_1_SUITE,tc4,{failed,{error,failed_on_purpose}}}},
- {?eh,test_stats,{1,3,{1,1}}},
+ {?eh,test_stats,{1,3,{0,2}}},
{?eh,tc_auto_skip,{ts_if_1_SUITE,tc5,{failed,{ts_if_1_SUITE,tc4}}}},
- {?eh,test_stats,{1,3,{1,2}}},
+ {?eh,test_stats,{1,3,{0,3}}},
{?eh,tc_start,{ts_if_1_SUITE,{end_per_group,seq2,[sequence]}}},
{?eh,tc_done,{ts_if_1_SUITE,{end_per_group,seq2,[sequence]},ok}}],
{?eh,tc_start,{ts_if_1_SUITE,tc6}},
- {?eh,tc_done,{ts_if_1_SUITE,tc6,{skipped,{require_failed,{not_available,void}}}}},
- {?eh,test_stats,{1,3,{1,3}}},
- {?eh,tc_start,{ts_if_1_SUITE,tc7}},
- {?eh,tc_done,{ts_if_1_SUITE,tc7,ok}},
- {?eh,test_stats,{2,3,{1,3}}},
+ {?eh,tc_done,{ts_if_1_SUITE,tc6,{auto_skipped,{require_failed,
+ {not_available,void}}}}},
+ {?eh,test_stats,{1,3,{0,4}}},
+ {?eh,tc_done,{ts_if_1_SUITE,tc7,{auto_skipped,
+ {testcase0_failed,bad_return_value}}}},
+ {?eh,test_stats,{1,3,{0,5}}},
{?eh,tc_start,{ts_if_1_SUITE,tc8}},
{?eh,tc_done,{ts_if_1_SUITE,tc8,{skipped,"tc8 skipped"}}},
- {?eh,test_stats,{2,3,{2,3}}},
+ {?eh,test_stats,{1,3,{1,5}}},
{?eh,tc_start,{ts_if_1_SUITE,tc9}},
{?eh,tc_done,{ts_if_1_SUITE,tc9,{skipped,'tc9 skipped'}}},
- {?eh,test_stats,{2,3,{3,3}}},
+ {?eh,test_stats,{1,3,{2,5}}},
{?eh,tc_start,{ts_if_1_SUITE,tc10}},
- {?eh,tc_done,{ts_if_1_SUITE,tc10,{failed,{error,{function_clause,'_'}}}}},
- {?eh,test_stats,{2,4,{3,3}}},
+ {?eh,tc_done,{ts_if_1_SUITE,tc10,
+ {failed,{error,{function_clause,'_'}}}}},
+ {?eh,test_stats,{1,4,{2,5}}},
{?eh,tc_start,{ts_if_1_SUITE,tc11}},
{?eh,tc_done,{ts_if_1_SUITE,tc11,
- {skipped,{failed,{ts_if_1_SUITE,init_per_testcase,bad_return}}}}},
- {?eh,test_stats,{2,4,{3,4}}},
+ {auto_skipped,
+ {failed,{ts_if_1_SUITE,init_per_testcase,bad_return}}}}},
+ {?eh,test_stats,{1,4,{2,6}}},
[{?eh,tc_start,{ts_if_1_SUITE,{init_per_group,g1,[]}}},
- {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g1,[]},{skipped,g1_got_skipped}}},
- {?eh,tc_auto_skip,{ts_if_1_SUITE,gtc1,g1_got_skipped}},
- {?eh,test_stats,{2,4,{3,5}}},
- {?eh,tc_auto_skip,{ts_if_1_SUITE,end_per_group,g1_got_skipped}}],
-
+ {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g1,[]},
+ {skipped,g1_got_skipped}}},
+ {?eh,tc_user_skip,{ts_if_1_SUITE,gtc1,g1_got_skipped}},
+ {?eh,test_stats,{1,4,{3,6}}},
+ {?eh,tc_user_skip,{ts_if_1_SUITE,{end_per_group,g1},g1_got_skipped}}],
+
{parallel,
[{?eh,tc_start,{ts_if_1_SUITE,{init_per_group,g2,[parallel]}}},
{?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g2,[parallel]},ok}},
[{?eh,tc_start,{ts_if_1_SUITE,{init_per_group,g3,[]}}},
- {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g3,[]},{skipped,g3_got_skipped}}},
- {?eh,tc_auto_skip,{ts_if_1_SUITE,gtc2,g3_got_skipped}},
- {?eh,test_stats,{2,4,{3,6}}},
- {?eh,tc_auto_skip,{ts_if_1_SUITE,end_per_group,g3_got_skipped}}],
+ {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g3,[]},{skipped,g3_got_skipped}}},
+ {?eh,tc_user_skip,{ts_if_1_SUITE,gtc2,g3_got_skipped}},
+ {?eh,test_stats,{1,4,{4,6}}},
+ {?eh,tc_user_skip,{ts_if_1_SUITE,{end_per_group,g3},g3_got_skipped}}],
{?eh,tc_start,{ts_if_1_SUITE,{end_per_group,g2,[parallel]}}},
{?eh,tc_done,{ts_if_1_SUITE,{end_per_group,g2,[parallel]},ok}}]},
{?eh,tc_start,{ts_if_1_SUITE,tc12}},
{?eh,tc_done,{ts_if_1_SUITE,tc12,{failed,{testcase_aborted,'stopping tc12'}}}},
- {?eh,test_stats,{2,5,{3,6}}},
+ {?eh,test_stats,{1,5,{4,6}}},
{?eh,tc_start,{ts_if_1_SUITE,tc13}},
{?eh,tc_done,{ts_if_1_SUITE,tc13,ok}},
- {?eh,test_stats,{3,5,{3,6}}},
+ {?eh,test_stats,{2,5,{4,6}}},
{?eh,tc_start,{ts_if_1_SUITE,end_per_suite}},
{?eh,tc_done,{ts_if_1_SUITE,end_per_suite,ok}},
+
{?eh,tc_start,{ts_if_2_SUITE,init_per_suite}},
- {?eh,tc_done,{ts_if_2_SUITE,init_per_suite,
- {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}},
- {?eh,tc_auto_skip,{ts_if_2_SUITE,my_test_case,
- {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}},
- {?eh,test_stats,{3,5,{3,7}}},
- {?eh,tc_auto_skip,{ts_if_2_SUITE,end_per_suite,
- {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}},
+ {?eh,tc_done,
+ {ts_if_2_SUITE,init_per_suite,
+ {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}},
+ {?eh,tc_auto_skip,
+ {ts_if_2_SUITE,my_test_case,
+ {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}},
+ {?eh,test_stats,{2,5,{4,7}}},
+ {?eh,tc_auto_skip,
+ {ts_if_2_SUITE,end_per_suite,
+ {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}},
+
{?eh,tc_start,{ct_framework,error_in_suite}},
- {?eh,test_stats,{3,5,{4,7}}},
+ {?eh,test_stats,{2,6,{4,7}}},
+
{?eh,tc_start,{ct_framework,error_in_suite}},
- {?eh,test_stats,{3,5,{5,7}}},
+ {?eh,test_stats,{2,7,{4,7}}},
+
{?eh,tc_start,{ts_if_5_SUITE,init_per_suite}},
{?eh,tc_done,{ts_if_5_SUITE,init_per_suite,
- {skipped,{require_failed_in_suite0,{not_available,undef_variable}}}}},
- {?eh,tc_auto_skip,{ts_if_5_SUITE,my_test_case,
- {require_failed_in_suite0,{not_available,undef_variable}}}},
- {?eh,test_stats,{3,5,{5,8}}},
- {?eh,tc_auto_skip,{ts_if_5_SUITE,end_per_suite,
- {require_failed_in_suite0,{not_available,undef_variable}}}},
+ {auto_skipped,
+ {require_failed_in_suite0,{not_available,undef_variable}}}}},
+ {?eh,tc_auto_skip,
+ {ts_if_5_SUITE,my_test_case,
+ {require_failed_in_suite0,{not_available,undef_variable}}}},
+ {?eh,test_stats,{2,7,{4,8}}},
+ {?eh,tc_auto_skip,
+ {ts_if_5_SUITE,end_per_suite,
+ {require_failed_in_suite0,{not_available,undef_variable}}}},
+
{?eh,tc_start,{ct_framework,init_per_suite}},
{?eh,tc_done,{ct_framework,init_per_suite,
{failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}},
- {?eh,tc_auto_skip,{ts_if_6_SUITE,tc1,
- {failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}},
- {?eh,test_stats,{3,5,{5,9}}},
+ {?eh,tc_auto_skip,
+ {ts_if_6_SUITE,tc1,
+ {failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}},
+ {?eh,test_stats,{2,7,{4,9}}},
+ {?eh,tc_auto_skip,
+ {ct_framework,end_per_suite,
+ {failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}},
- {?eh,tc_start,{ts_if_7_SUITE,tc1}},
- {?eh,tc_done,{ts_if_7_SUITE,tc1,ok}},
- {?eh,test_stats,{4,5,{5,9}}},
+ {?eh,tc_start,{ct_framework,init_per_suite}},
+ {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+ {?eh,tc_done,
+ {ts_if_7_SUITE,tc1,{auto_skipped,{testcase0_failed,bad_return_value}}}},
+ {?eh,test_stats,{2,7,{4,10}}},
+ {?eh,tc_done,{ts_if_7_SUITE,
+ {init_per_group,g1,[]},
+ {auto_skipped,{group0_failed,bad_return_value}}}},
+ {?eh,tc_auto_skip,
+ {ts_if_7_SUITE,tc2,{group0_failed,bad_return_value}}},
+ {?eh,test_stats,{2,7,{4,11}}},
+ {?eh,tc_auto_skip,
+ {ts_if_7_SUITE,{end_per_group,g1},{group0_failed,bad_return_value}}},
+
+ [{?eh,tc_start,{ts_if_7_SUITE,{init_per_group,g2,[]}}},
+ {?eh,tc_done,{ts_if_7_SUITE,{init_per_group,g2,[]},ok}},
+ {?eh,tc_done,{ts_if_7_SUITE,tc2,
+ {auto_skipped,{testcase0_failed,bad_return_value}}}},
+ {?eh,test_stats,{2,7,{4,12}}},
+ {?eh,tc_start,{ts_if_7_SUITE,{end_per_group,g2,[]}}},
+ {?eh,tc_done,{ts_if_7_SUITE,{end_per_group,g2,[]},ok}}],
+
+ {?eh,tc_start,{ct_framework,end_per_suite}},
+ {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+
+
+ {?eh,tc_start,{ct_framework,init_per_suite}},
+ {?eh,tc_done,{ct_framework,init_per_suite,ok}},
{?eh,tc_start,{ts_if_8_SUITE,tc1}},
{?eh,tc_done,{ts_if_8_SUITE,tc1,{failed,{error,failed_on_purpose}}}},
- {?eh,test_stats,{4,6,{5,9}}},
-
+ {?eh,test_stats,{2,8,{4,12}}},
+ {?eh,tc_start,{ct_framework,end_per_suite}},
+ {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+
+
{?eh,tc_user_skip,{skipped_by_spec_1_SUITE,all,"should be skipped"}},
- {?eh,test_stats,{4,6,{6,9}}},
-
+ {?eh,test_stats,{2,8,{5,12}}},
{?eh,tc_start,{skipped_by_spec_2_SUITE,init_per_suite}},
{?eh,tc_done,{skipped_by_spec_2_SUITE,init_per_suite,ok}},
{?eh,tc_user_skip,{skipped_by_spec_2_SUITE,tc1,"should be skipped"}},
- {?eh,test_stats,{4,6,{7,9}}},
+ {?eh,test_stats,{2,8,{6,12}}},
{?eh,tc_start,{skipped_by_spec_2_SUITE,end_per_suite}},
{?eh,tc_done,{skipped_by_spec_2_SUITE,end_per_suite,ok}},
-
+
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}
].
+
diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_7_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_7_SUITE.erl
index a2254848d0..20e04c464b 100644
--- a/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_7_SUITE.erl
+++ b/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_7_SUITE.erl
@@ -73,7 +73,8 @@ end_per_testcase(_TestCase, _Config) ->
%% N = integer() | forever
%%--------------------------------------------------------------------
groups() ->
- [].
+ [{g1,[],[tc2]},
+ {g2,[],[tc2]}].
%%--------------------------------------------------------------------
%% Function: all() -> GroupsAndTestCases | {skip,Reason}
@@ -83,7 +84,12 @@ groups() ->
%% Reason = term()
%%--------------------------------------------------------------------
all() ->
- [tc1].
+ [tc1,{group,g1},{group,g2}].
+
+group(g1) ->
+ exit(g1_byebye);
+group(_) ->
+ [].
tc1() ->
exit(tc1_byebye).
@@ -91,3 +97,9 @@ tc1() ->
tc1(_) ->
done.
+tc2() ->
+ exit(tc2_byebye).
+
+tc2(_) ->
+ done.
+
diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl
index 67e430f821..772274ce7e 100644
--- a/lib/common_test/test/ct_test_support.erl
+++ b/lib/common_test/test/ct_test_support.erl
@@ -692,8 +692,10 @@ locate({parallel,TEvs}, Node, Evs, Config) ->
test_server:format("Found ~p!", [TEv]),
{Done,RemEvs2,length(RemEvs2)}
end;
- %% end_per_group auto skipped
- (TEv={TEH,tc_auto_skip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize}) ->
+ %% end_per_group auto- or user skipped
+ (TEv={TEH,AutoOrUserSkip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize})
+ when AutoOrUserSkip == tc_auto_skip;
+ AutoOrUserSkip == tc_user_skip ->
RemEvs1 =
lists:dropwhile(
fun({EH,#event{name=tc_auto_skip,
@@ -704,10 +706,18 @@ locate({parallel,TEvs}, Node, Evs, Config) ->
match -> false;
_ -> true
end;
+ ({EH,#event{name=tc_user_skip,
+ node=EvNode,
+ data={Mod,end_per_group,Reason}}}) when
+ EH == TEH, EvNode == Node, Mod == M ->
+ case match_data(R, Reason) of
+ match -> false;
+ _ -> true
+ end;
({EH,#event{name=stop_logging,
node=EvNode,data=_}}) when
EH == TEH, EvNode == Node ->
- exit({tc_auto_skip_not_found,TEv});
+ exit({tc_auto_or_user_skip_not_found,TEv});
(_) ->
true
end, RemEvs),
@@ -925,8 +935,10 @@ locate({shuffle,TEvs}, Node, Evs, Config) ->
test_server:format("Found ~p!", [TEv]),
{Done,RemEvs2,length(RemEvs2)}
end;
- %% end_per_group auto skipped
- (TEv={TEH,tc_auto_skip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize}) ->
+ %% end_per_group auto-or user skipped
+ (TEv={TEH,AutoOrUserSkip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize})
+ when AutoOrUserSkip == tc_auto_skip;
+ AutoOrUserSkip == tc_user_skip ->
RemEvs1 =
lists:dropwhile(
fun({EH,#event{name=tc_auto_skip,
@@ -934,6 +946,11 @@ locate({shuffle,TEvs}, Node, Evs, Config) ->
data={Mod,end_per_group,Reason}}}) when
EH == TEH, EvNode == Node, Mod == M, Reason == R ->
false;
+ ({EH,#event{name=tc_user_skip,
+ node=EvNode,
+ data={Mod,end_per_group,Reason}}}) when
+ EH == TEH, EvNode == Node, Mod == M, Reason == R ->
+ false;
({EH,#event{name=stop_logging,
node=EvNode,data=_}}) when
EH == TEH, EvNode == Node ->
@@ -1178,6 +1195,9 @@ log_events1([E={_EH,tc_done,{_M,{end_per_group,_GrName,Props},_R}} | Evs], Dev,
log_events1([E={_EH,tc_auto_skip,{_M,end_per_group,_Reason}} | Evs], Dev, Ind) ->
io:format(Dev, "~s~p],~n", [Ind,E]),
log_events1(Evs, Dev, Ind--" ");
+log_events1([E={_EH,tc_user_skip,{_M,end_per_group,_Reason}} | Evs], Dev, Ind) ->
+ io:format(Dev, "~s~p],~n", [Ind,E]),
+ log_events1(Evs, Dev, Ind--" ");
log_events1([E], Dev, Ind) ->
io:format(Dev, "~s~p~n].~n", [Ind,E]),
ok;
diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl
index 6a4a4acd80..187b5e6d3a 100644
--- a/lib/common_test/test/ct_testspec_1_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_1_SUITE.erl
@@ -760,16 +760,42 @@ test_events(all_groups) ->
test_events(skip_all_groups) ->
[
{?eh,start_logging,'_'},
+ {?eh,start_info,{1,1,12}},
{?eh,tc_start,{groups_11_SUITE,init_per_suite}},
- {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1a},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1a},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}},
{?eh,test_stats,{0,0,{1,0}}},
- {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}},
{?eh,test_stats,{0,0,{2,0}}},
- {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_2},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1a},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1b},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}},
{?eh,test_stats,{0,0,{3,0}}},
- {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_4},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}},
{?eh,test_stats,{0,0,{4,0}}},
- {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1b},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_2},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_2a,"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{5,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_3a,"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{6,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_3b,"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{7,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_2b,"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{8,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_2},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_4},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_5a,"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{9,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_7a,"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{10,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_7b,"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{11,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_5b,"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{12,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_4},"SKIPPED!"}},
+ {?eh,tc_start,{groups_11_SUITE,end_per_suite}},
+ {?eh,tc_done,{groups_11_SUITE,end_per_suite,init}},
{negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
];
@@ -788,21 +814,35 @@ test_events(group) ->
test_events(skip_group) ->
[
- {?eh,start_logging,'_'},
- {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
-
- {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}},
- {?eh,tc_start,{groups_11_SUITE,testcase_1a}},
- {?eh,tc_start,{groups_11_SUITE,testcase_1b}},
- {?eh,test_stats,{2,0,{0,0}}},
- {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}},
-
- {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}},
- {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_2},"SKIPPED!"}},
- %%! But not test_group_7 since it's a sub-group!
- {?eh,test_stats,{2,0,{2,0}}},
- {negative,{?eh,tc_user_skip,'_'},{?eh,stop_logging,'_'}}
- ];
+ {?eh,start_logging,'_'},
+ {?eh,start_info,{1,1,8}},
+ {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
+ [{?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}},
+ {?eh,tc_done,{groups_11_SUITE,{init_per_group,test_group_1a,[]},ok}},
+ {?eh,tc_start,{groups_11_SUITE,testcase_1a}},
+ {?eh,tc_start,{groups_11_SUITE,testcase_1b}},
+ {?eh,test_stats,{2,0,{0,0}}},
+ {?eh,tc_start,{groups_11_SUITE,{end_per_group,test_group_1a,[]}}},
+ {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},ok}}],
+ {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1b},
+ "SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}},
+ {?eh,test_stats,{2,0,{2,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1b},
+ "SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_2},
+ "SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_2a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_3a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_3b,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_2b,"SKIPPED!"}},
+ {?eh,test_stats,{2,0,{6,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_2},
+ "SKIPPED!"}},
+ {?eh,tc_done,{groups_11_SUITE,end_per_suite,init}},
+ {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+ ];
test_events(group_all_testcases) ->
[
@@ -820,11 +860,23 @@ test_events(group_all_testcases) ->
test_events(skip_group_all_testcases) ->
[
{?eh,start_logging,'_'},
+ {?eh,start_info,{1,1,4}},
{?eh,tc_start,{groups_11_SUITE,init_per_suite}},
- {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1a},"SKIPPED!"}},
- {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1a},
+ "SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}},
{?eh,test_stats,{0,0,{2,0}}},
- {?eh,tc_start,{groups_11_SUITE,end_per_suite}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1a},
+ "SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1b},
+ "SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{4,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1b},
+ "SKIPPED!"}},
+ {?eh,tc_done,{groups_11_SUITE,end_per_suite,init}},
{negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
];
@@ -934,7 +986,7 @@ test_events(subgroup) ->
[
{?eh,start_logging,'_'},
{?eh,tc_start,{groups_12_SUITE,init_per_suite}},
-
+
{parallel,
[{?eh,tc_start,
{groups_12_SUITE,{init_per_group,test_group_2,[parallel]}}},
@@ -962,36 +1014,58 @@ test_events(subgroup) ->
test_events(skip_subgroup) ->
[
{?eh,start_logging,'_'},
+ {?eh,start_info,{1,1,6}},
{?eh,tc_start,{groups_12_SUITE,init_per_suite}},
- [{?eh,tc_start,
- {groups_12_SUITE,{init_per_group,test_group_4,[]}}},
+ [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_4,[]}}},
+ {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_4,[]},ok}},
+
{parallel,
- [{?eh,tc_start,
- {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}},
- {?eh,tc_done,
- {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}},
- {parallel,
- [{?eh,tc_start,
- {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}},
- {?eh,tc_done,
- {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}},
- [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}},
- {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}],
- {?eh,tc_user_skip,
- {groups_12_SUITE,{group,test_group_8},"SKIPPED!"}},
- {?eh,tc_start,
- {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}},
- {?eh,tc_done,
- {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}}
- ]},
- {?eh,tc_start,
- {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}},
- {?eh,tc_done,
- {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]},
- {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}],
+ [{?eh,tc_start,{groups_12_SUITE,
+ {init_per_group,test_group_5,[parallel]}}},
+ {?eh,tc_done,{groups_12_SUITE,
+ {init_per_group,test_group_5,[parallel]},ok}},
+
+ {parallel,
+ [{?eh,tc_start,{groups_12_SUITE,
+ {init_per_group,test_group_6,[parallel]}}},
+ {?eh,tc_done,{groups_12_SUITE,
+ {init_per_group,test_group_6,[parallel]},ok}},
+
+ [{?eh,tc_start,{groups_12_SUITE,
+ {init_per_group,test_group_7,[sequence]}}},
+ {?eh,tc_done,{groups_12_SUITE,
+ {init_per_group,test_group_7,[sequence]},ok}},
+ {?eh,tc_done,{groups_12_SUITE,testcase_7a,ok}},
+ {?eh,tc_done,{groups_12_SUITE,testcase_7b,ok}},
+ {?eh,tc_start,{groups_12_SUITE,
+ {end_per_group,test_group_7,[sequence]}}},
+ {?eh,tc_done,{groups_12_SUITE,
+ {end_per_group,test_group_7,[sequence]},ok}}],
+
+ {?eh,tc_user_skip,{groups_12_SUITE,
+ {init_per_group,test_group_8},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_8a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_8b,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,
+ {end_per_group,test_group_8},"SKIPPED!"}},
+
+ {?eh,tc_start,{groups_12_SUITE,
+ {end_per_group,test_group_6,[parallel]}}},
+ {?eh,tc_done,{groups_12_SUITE,
+ {end_per_group,test_group_6,[parallel]},ok}}]},
+
+ {?eh,test_stats,{4,0,{2,0}}},
+ {?eh,tc_start,{groups_12_SUITE,
+ {end_per_group,test_group_5,[parallel]}}},
+ {?eh,tc_done,{groups_12_SUITE,
+ {end_per_group,test_group_5,[parallel]},ok}}]},
- {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}},
+ {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}},
+ {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_4,[]},ok}}],
+
+ {?eh,tc_start,{groups_12_SUITE,end_per_suite}},
+ {?eh,tc_done,{groups_12_SUITE,end_per_suite,init}},
{negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
];
@@ -1066,17 +1140,26 @@ test_events(subgroup_all_testcases) ->
test_events(skip_subgroup_all_testcases) ->
[
{?eh,start_logging,'_'},
+ {?eh,start_info,{1,1,6}},
{?eh,tc_start,{groups_12_SUITE,init_per_suite}},
-
- [{?eh,tc_start,
- {groups_12_SUITE,{init_per_group,test_group_4,[]}}},
- {?eh,tc_done,
- {groups_12_SUITE,{init_per_group,test_group_4,[]},ok}},
- {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_5},"SKIPPED!"}},
- {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}},
- {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_4,[]},ok}}
- ],
-
+ [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_4,[]}}},
+ {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_4,[]},ok}},
+ {?eh,tc_user_skip,{groups_12_SUITE,
+ {init_per_group,test_group_5},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_5a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_7a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_7b,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_8a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_8b,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_5b,"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{6,0}}},
+ {?eh,tc_user_skip,{groups_12_SUITE,
+ {end_per_group,test_group_5},"SKIPPED!"}},
+ {?eh,tc_start,{groups_12_SUITE,
+ {end_per_group,test_group_4,[]}}},
+ {?eh,tc_done,{groups_12_SUITE,
+ {end_per_group,test_group_4,[]},ok}}],
+ {?eh,tc_start,{groups_12_SUITE,end_per_suite}},
{?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}},
{negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
];
@@ -1194,13 +1277,30 @@ test_events(skip_subgroup_testcase) ->
test_events(sub_skipped_by_top) ->
[
{?eh,start_logging,'_'},
+ {?eh,start_info,{1,1,12}},
{?eh,tc_start,{groups_12_SUITE,init_per_suite}},
-
- {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_4},"SKIPPED!"}},
- {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_4},"SKIPPED!"}},
-
+ {?eh,tc_user_skip,{groups_12_SUITE,{init_per_group,test_group_4},
+ "SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_5a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_7a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_7b,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_8a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_8b,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_5b,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,
+ {end_per_group,test_group_4},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,
+ {init_per_group,test_group_4},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_5a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_7a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_7b,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_8a,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_8b,"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_12_SUITE,testcase_5b,"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{12,0}}},
+ {?eh,tc_user_skip,{groups_12_SUITE,
+ {end_per_group,test_group_4},"SKIPPED!"}},
{?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}},
-
{negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
];
diff --git a/lib/common_test/test/ct_testspec_3_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE.erl
index 5fa187e5b4..9f8ca84e45 100644
--- a/lib/common_test/test/ct_testspec_3_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_3_SUITE.erl
@@ -1539,7 +1539,7 @@ flat_spec1_events() ->
{?eh,test_stats,{1,2,{0,0}}},
{?eh,tc_start,{t11_SUITE,autoskip_tc}},
{?eh,tc_done,
- {t11_SUITE,autoskip_tc,{skipped,
+ {t11_SUITE,autoskip_tc,{auto_skipped,
{failed,
{t11_SUITE,init_per_testcase,
{kaboom,'_'}}}}}},
@@ -1562,7 +1562,7 @@ flat_spec1_events() ->
{?eh,test_stats,{2,4,{1,1}}},
{?eh,tc_start,{t21_SUITE,autoskip_tc}},
{?eh,tc_done,
- {t21_SUITE,autoskip_tc,{skipped,
+ {t21_SUITE,autoskip_tc,{auto_skipped,
{failed,
{t21_SUITE,init_per_testcase,
{kaboom,'_'}}}}}},
@@ -1589,7 +1589,7 @@ flat_spec2_events() ->
{?eh,test_stats,{1,2,{0,0}}},
{?eh,tc_start,{t12_SUITE,autoskip_tc}},
{?eh,tc_done,
- {t12_SUITE,autoskip_tc,{skipped,
+ {t12_SUITE,autoskip_tc,{auto_skipped,
{failed,
{t12_SUITE,init_per_testcase,
{kaboom,'_'}}}}}},
@@ -1612,7 +1612,7 @@ flat_spec2_events() ->
{?eh,test_stats,{2,4,{1,1}}},
{?eh,tc_start,{t12_SUITE,autoskip_tc}},
{?eh,tc_done,
- {t12_SUITE,autoskip_tc,{skipped,
+ {t12_SUITE,autoskip_tc,{auto_skipped,
{failed,
{t12_SUITE,init_per_testcase,
{kaboom,'_'}}}}}},
@@ -1635,7 +1635,7 @@ flat_spec2_events() ->
{?eh,test_stats,{3,6,{2,2}}},
{?eh,tc_start,{t22_SUITE,autoskip_tc}},
{?eh,tc_done,
- {t22_SUITE,autoskip_tc,{skipped,
+ {t22_SUITE,autoskip_tc,{auto_skipped,
{failed,
{t22_SUITE,init_per_testcase,
{kaboom,'_'}}}}}},