aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--lib/common_test/src/ct_framework.erl5
-rw-r--r--lib/common_test/test/ct_skip_SUITE.erl131
-rw-r--r--lib/test_server/src/test_server_ctrl.erl46
3 files changed, 169 insertions, 13 deletions
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index 15809637fd..670c073b29 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -1283,7 +1283,10 @@ report(What,Data) ->
node=node(),
data=Data}),
case Data of
- {_,Func,_} when Func /= end_per_suite, Func /= end_per_group ->
+ {_,Func,_} when Func /= init_per_suite,
+ Func /= init_per_group,
+ Func /= end_per_suite,
+ Func /= end_per_group ->
ct_hooks:on_tc_skip(What, Data),
add_to_stats(user_skipped);
_ ->
diff --git a/lib/common_test/test/ct_skip_SUITE.erl b/lib/common_test/test/ct_skip_SUITE.erl
index 10e834ff2a..5f88ac4825 100644
--- a/lib/common_test/test/ct_skip_SUITE.erl
+++ b/lib/common_test/test/ct_skip_SUITE.erl
@@ -146,9 +146,17 @@ testspec_skip(Config) when is_list(Config) ->
TestSpec3 = [{suites, TestDir, user_skip_7_SUITE},
{skip_groups, TestDir, user_skip_7_SUITE, psub1, "SKIPPED"}],
+ TestSpec4 = [{suites, TestDir, user_skip_7_SUITE},
+ {skip_suites, TestDir, user_skip_7_SUITE, "SKIPPED"}],
+
+ TestSpec5 = [{groups, TestDir, user_skip_6_SUITE, ptop1},
+ {skip_groups, TestDir, user_skip_6_SUITE, psub1, "SKIPPED"}],
+
{Opts,ERPid} = setup_testspec([{ts1,TestSpec1},
{ts2,TestSpec2},
- {ts3,TestSpec3}], Config),
+ {ts3,TestSpec3},
+ {ts4,TestSpec4},
+ {ts5,TestSpec5}], Config),
ok = ct_test_support:run(Opts, Config),
@@ -721,4 +729,123 @@ test_events(user_skip) ->
];
test_events(testspec_skip) ->
- [].
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,4}},
+ {?eh,tc_start,{ct_framework,init_per_suite}},
+ {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+ {parallel,
+ [{?eh,tc_start,
+ {user_skip_7_SUITE,{init_per_group,ptop1,[parallel]}}},
+ {?eh,tc_done,
+ {user_skip_7_SUITE,{init_per_group,ptop1,[parallel]},ok}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc1,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{1,0}}},
+ {parallel,
+ [{?eh,tc_start,
+ {user_skip_7_SUITE,{init_per_group,psub1,[parallel]}}},
+ {?eh,tc_done,
+ {user_skip_7_SUITE,{init_per_group,psub1,[parallel]},ok}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc3,"SKIPPED"}},
+ {?eh,tc_start,{user_skip_7_SUITE,tc4}},
+ {?eh,tc_done,{user_skip_7_SUITE,tc4,ok}},
+ {?eh,test_stats,{1,0,{2,0}}},
+ {?eh,tc_start,
+ {user_skip_7_SUITE,{end_per_group,psub1,[parallel]}}},
+ {?eh,tc_done,
+ {user_skip_7_SUITE,{end_per_group,psub1,[parallel]},ok}}]},
+ {?eh,tc_start,{user_skip_7_SUITE,tc2}},
+ {?eh,tc_done,{user_skip_7_SUITE,tc2,ok}},
+ {?eh,test_stats,{2,0,{2,0}}},
+ {?eh,tc_start,
+ {user_skip_7_SUITE,{end_per_group,ptop1,[parallel]}}},
+ {?eh,tc_done,
+ {user_skip_7_SUITE,{end_per_group,ptop1,[parallel]},ok}}]},
+ {?eh,tc_start,{ct_framework,end_per_suite}},
+ {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,4}},
+ {?eh,tc_start,{ct_framework,init_per_suite}},
+ {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,init_per_group,"SKIPPED"}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc1,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{1,0}}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc3,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{2,0}}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc4,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{3,0}}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc2,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{4,0}}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,end_per_group,"SKIPPED"}},
+ {?eh,tc_start,{ct_framework,end_per_suite}},
+ {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,4}},
+ {?eh,tc_start,{ct_framework,init_per_suite}},
+ {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+ {parallel,
+ [{?eh,tc_start,
+ {user_skip_7_SUITE,{init_per_group,ptop1,[parallel]}}},
+ {?eh,tc_done,
+ {user_skip_7_SUITE,{init_per_group,ptop1,[parallel]},ok}},
+ {?eh,tc_user_skip,
+ {user_skip_7_SUITE,init_per_group,"SKIPPED"}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc3,"SKIPPED"}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,tc4,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{2,0}}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,end_per_group,"SKIPPED"}},
+ {?eh,tc_start,{user_skip_7_SUITE,tc1}},
+ {?eh,tc_done,{user_skip_7_SUITE,tc1,ok}},
+ {?eh,tc_start,{user_skip_7_SUITE,tc2}},
+ {?eh,tc_done,{user_skip_7_SUITE,tc2,ok}},
+ {?eh,test_stats,{2,0,{2,0}}},
+ {?eh,tc_start,{user_skip_7_SUITE,{end_per_group,ptop1,[parallel]}}},
+ {?eh,tc_done,{user_skip_7_SUITE,{end_per_group,ptop1,[parallel]},ok}}]},
+ {?eh,tc_start,{ct_framework,end_per_suite}},
+ {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,0}},
+ {?eh,tc_user_skip,{user_skip_7_SUITE,all,"SKIPPED"}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,4}},
+ {?eh,tc_start,{ct_framework,init_per_suite}},
+ {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+ {parallel,
+ [{?eh,tc_start,{user_skip_6_SUITE,{init_per_group,ptop1,[parallel]}}},
+ {?eh,tc_done,{user_skip_6_SUITE,
+ {init_per_group,ptop1,[parallel]},
+ {skipped,"Top group skipped"}}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc1,"Top group skipped"}},
+ {?eh,test_stats,{0,0,{1,0}}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc3,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{2,0}}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc4,"SKIPPED"}},
+ {?eh,test_stats,{0,0,{3,0}}},
+ {?eh,tc_user_skip,{user_skip_6_SUITE,tc2,"Top group skipped"}},
+ {?eh,test_stats,{0,0,{4,0}}},
+ {?eh,tc_user_skip,
+ {user_skip_6_SUITE,end_per_group,"Top group skipped"}}]},
+ {?eh,tc_start,{ct_framework,end_per_suite}},
+ {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}
+ ].
+
+
diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl
index 2041ee294e..810591b16f 100644
--- a/lib/test_server/src/test_server_ctrl.erl
+++ b/lib/test_server/src/test_server_ctrl.erl
@@ -1402,10 +1402,16 @@ remove_conf([{conf, _Ref, Props, _MF}|Cases], NoConf, Repeats) ->
end;
remove_conf([{make,_Ref,_MF}|Cases], NoConf, Repeats) ->
remove_conf(Cases, NoConf, Repeats);
+remove_conf([{skip_case,{{_M,all},_Cmt}}|Cases], NoConf, Repeats) ->
+ remove_conf(Cases, NoConf, Repeats);
remove_conf([{skip_case,{Type,_Ref,_MF,_Cmt}}|Cases],
NoConf, Repeats) when Type==conf;
Type==make ->
remove_conf(Cases, NoConf, Repeats);
+remove_conf([{skip_case,{Type,_Ref,_MF,_Cmt},_Mode}|Cases],
+ NoConf, Repeats) when Type==conf;
+ Type==make ->
+ remove_conf(Cases, NoConf, Repeats);
remove_conf([C|Cases], NoConf, Repeats) ->
remove_conf(Cases, [C|NoConf], Repeats);
remove_conf([], NoConf, true) ->
@@ -1413,6 +1419,11 @@ remove_conf([], NoConf, true) ->
remove_conf([], NoConf, false) ->
lists:reverse(NoConf).
+get_suites([{skip_case,{{Mod,_Func},_Cmt}}|Tests], Mods) when is_atom(Mod) ->
+ case add_mod(Mod, Mods) of
+ true -> get_suites(Tests, [Mod|Mods]);
+ false -> get_suites(Tests, Mods)
+ end;
get_suites([{Mod,_Case}|Tests], Mods) when is_atom(Mod) ->
case add_mod(Mod, Mods) of
true -> get_suites(Tests, [Mod|Mods]);
@@ -1478,13 +1489,10 @@ do_test_cases(TopCases, SkipCases,
end,
put(test_server_cases, N),
put(test_server_case_num, 0),
+
TestSpec =
add_init_and_end_per_suite(TestSpec0, undefined, undefined, FwMod),
-
- %%! --- Fri Nov 8 14:50:39 2013 --- peppe was here!
- io:format(user, ">>> HERE'S THE TS: ~p~n~n", [TestSpec]),
-
TI = get_target_info(),
print(1, "Starting test~ts",
[print_if_known(N, {", ~w test cases",[N]},
@@ -1934,8 +1942,6 @@ add_init_and_end_per_suite([{skip_case,{conf,_,{Mod,_},_}}=Case|Cases], LastMod,
{PreCases, NextMod, NextRef} =
do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod),
PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)];
-add_init_and_end_per_suite([{skip_case,_}=Case|Cases], LastMod, LastRef, FwMod) ->
- [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)];
add_init_and_end_per_suite([{conf,Ref,Props,{FwMod,Func}}=Case|Cases], LastMod,
LastRef, FwMod) ->
%% if Mod == FwMod, this conf test is (probably) a test case group where
@@ -1957,6 +1963,9 @@ add_init_and_end_per_suite([{conf,_,_,{Mod,_}}=Case|Cases], LastMod,
{PreCases, NextMod, NextRef} =
do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod),
PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)];
+add_init_and_end_per_suite([SkipCase|Cases], LastMod, LastRef, FwMod)
+ when element(1,SkipCase) == skip_case ->
+ [SkipCase|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)];
add_init_and_end_per_suite([{conf,_,_,_}=Case|Cases], LastMod, LastRef, FwMod) ->
[Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)];
add_init_and_end_per_suite([{Mod,_}=Case|Cases], LastMod, LastRef, FwMod)
@@ -2238,6 +2247,13 @@ run_test_cases(TestSpec, Config, TimetrapData) ->
%% group1_end | --->
%%
+run_test_cases_loop([{SkipTag,CaseData={Type,_Ref,_Case,_Comment}}|Cases],
+ Config, TimetrapData, Mode, Status) when
+ ((SkipTag==auto_skip_case) or (SkipTag==skip_case)) and
+ ((Type==conf) or (Type==make)) ->
+ run_test_cases_loop([{SkipTag,CaseData,Mode}|Cases],
+ Config, TimetrapData, Mode, Status);
+
run_test_cases_loop([{SkipTag,{Type,Ref,Case,Comment},SkipMode}|Cases],
Config, TimetrapData, Mode, Status) when
((SkipTag==auto_skip_case) or (SkipTag==skip_case)) and
@@ -2373,6 +2389,13 @@ run_test_cases_loop([{auto_skip_case,{Case,Comment},SkipMode}|Cases],
run_test_cases_loop(Cases, Config, TimetrapData, Mode,
update_status(skipped, Mod, Func, Status));
+run_test_cases_loop([{skip_case,{{Mod,all}=Case,Comment}}|Cases],
+ Config, TimetrapData, Mode, Status) ->
+ skip_case(user, undefined, 0, Case, Comment, false, Mode),
+ test_server_sup:framework_call(report, [tc_user_skip,
+ {Mod,all,Comment}]),
+ run_test_cases_loop(Cases, Config, TimetrapData, Mode, Status);
+
run_test_cases_loop([{skip_case,{Case,Comment}}|Cases],
Config, TimetrapData, Mode, Status) ->
{Mod,Func} = skip_case(user, undefined, get(test_server_case_num)+1,
@@ -3227,6 +3250,7 @@ modify_cases_upto(Ref, ModOp, Cases, Orig, Alt) ->
case lists:any(fun({_,R,_,_}) when R == Ref -> true;
({_,R,_}) when R == Ref -> true;
({skip_case,{_,R,_,_},_}) when R == Ref -> true;
+ ({skip_case,{_,R,_,_}}) when R == Ref -> true;
(_) -> false
end, Cases) of
true ->
@@ -3261,6 +3285,12 @@ modify_cases_upto1(Ref, {copy,NewRef}, [{make,Ref,MF}=M|T], Orig, Alt) ->
modify_cases_upto1(Ref, {skip,Reason,_,Mode,SkipType},
[{skip_case,{Type,Ref,MF,_Cmt},_}|T], Orig, Alt) ->
{Orig,[{SkipType,{Type,Ref,MF,Reason},Mode}|Alt],T};
+modify_cases_upto1(Ref, {skip,Reason,_,Mode,SkipType},
+ [{skip_case,{Type,Ref,MF,_Cmt}}|T], Orig, Alt) ->
+ {Orig,[{SkipType,{Type,Ref,MF,Reason},Mode}|Alt],T};
+modify_cases_upto1(Ref, {copy,NewRef},
+ [{skip_case,{Type,Ref,MF,Cmt},Mode}=C|T], Orig, Alt) ->
+ {[C|Orig],[{skip_case,{Type,NewRef,MF,Cmt},Mode}|Alt],T};
modify_cases_upto1(Ref, {copy,NewRef},
[{skip_case,{Type,Ref,MF,Cmt}}=C|T], Orig, Alt) ->
{[C|Orig],[{skip_case,{Type,NewRef,MF,Cmt}}|Alt],T};
@@ -4504,10 +4534,6 @@ collect_cases({conf,Props,InitMF,CaseList,FinMF} = Conf, St) ->
FinMF}],
Skipped = skip_cases_upto(Ref, Cases2Skip, Comment,
conf, Mode, skip_case),
-
- %%! --- Fri Nov 8 16:47:50 2013 --- peppe was here!
- io:format(user, "!!! FLAT2 = ~p~n", [Skipped]),
-
{ok,[{skip_case,{conf,Ref,InitMF,Comment},Mode} |
Skipped],St1};
{error,_Reason} = Error ->