aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test/src/test_server_ctrl.erl
diff options
context:
space:
mode:
authorSiri Hansen <[email protected]>2019-03-20 16:48:41 +0100
committerSiri Hansen <[email protected]>2019-04-15 11:22:56 +0200
commit3d087911ef3ecd769333b822c337bbbb76877871 (patch)
treea618ec69afe9e37c7e15f2ba5d6a555e6eef8c33 /lib/common_test/src/test_server_ctrl.erl
parentacd87425acf7705328b01aa8f0a804207ffe4790 (diff)
downloadotp-3d087911ef3ecd769333b822c337bbbb76877871.tar.gz
otp-3d087911ef3ecd769333b822c337bbbb76877871.tar.bz2
otp-3d087911ef3ecd769333b822c337bbbb76877871.zip
[ct] Add {testcase,TC,RepeatProps} syntax for repeating test cases
Diffstat (limited to 'lib/common_test/src/test_server_ctrl.erl')
-rw-r--r--lib/common_test/src/test_server_ctrl.erl131
1 files changed, 118 insertions, 13 deletions
diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl
index 8ef28b3343..bf51bae5f5 100644
--- a/lib/common_test/src/test_server_ctrl.erl
+++ b/lib/common_test/src/test_server_ctrl.erl
@@ -1443,6 +1443,8 @@ remove_conf([C={Mod,error_in_suite,_}|Cases], NoConf, Repeats) ->
true ->
remove_conf(Cases, [C|NoConf], Repeats)
end;
+remove_conf([C={repeat,_,_}|Cases], NoConf, _Repeats) ->
+ remove_conf(Cases, [C|NoConf], true);
remove_conf([C|Cases], NoConf, Repeats) ->
remove_conf(Cases, [C|NoConf], Repeats);
remove_conf([], NoConf, true) ->
@@ -2060,6 +2062,14 @@ add_init_and_end_per_suite([SkipCase|Cases], LastMod, LastRef, FwMod)
[SkipCase|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)];
add_init_and_end_per_suite([{conf,_,_,_}=Case|Cases], LastMod, LastRef, FwMod) ->
[Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)];
+add_init_and_end_per_suite([{repeat,{Mod,_},_}=Case|Cases], LastMod, LastRef, FwMod)
+ when Mod =/= LastMod, Mod =/= FwMod ->
+ {PreCases, NextMod, NextRef} =
+ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod),
+ PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod,
+ NextRef, FwMod)];
+add_init_and_end_per_suite([{repeat,_,_}=Case|Cases], LastMod, LastRef, FwMod) ->
+ [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)];
add_init_and_end_per_suite([{Mod,_}=Case|Cases], LastMod, LastRef, FwMod)
when Mod =/= LastMod, Mod =/= FwMod ->
{PreCases, NextMod, NextRef} =
@@ -2925,6 +2935,29 @@ run_test_cases_loop([{conf,_Ref,_Props,_X}=Conf|_Cases0],
Config, _TimetrapData, _Mode, _Status) ->
erlang:error(badarg, [Conf,Config]);
+run_test_cases_loop([{repeat,Case,{RepeatType,N}}|Cases0], Config,
+ TimeTrapData, Mode, Status) ->
+ Ref = make_ref(),
+ Parallel = check_prop(parallel, Mode) =/= false,
+ Sequence = check_prop(sequence, Mode) =/= false,
+ RepeatStop = RepeatType=:=repeat_until_fail
+ orelse RepeatType=:=repeat_until_ok,
+
+ if Parallel andalso RepeatStop ->
+ %% Cannot check results of test case during parallal
+ %% execution, so only RepeatType=:=repeat is allowed in
+ %% combination with parallel groups.
+ erlang:error({illegal_combination,{parallel,RepeatType}});
+ Sequence andalso RepeatStop ->
+ %% Sequence is stop on fail + skip rest, so only
+ %% RepeatType=:=repeat makes sense inside a sequence.
+ erlang:error({illegal_combination,{sequence,RepeatType}});
+ true ->
+ Mode1 = [{Ref,[{repeat,{RepeatType,1,N}}],?now}|Mode],
+ run_test_cases_loop([Case | Cases0], Config, TimeTrapData,
+ Mode1, Status)
+ end;
+
run_test_cases_loop([{Mod,Case}|Cases], Config, TimetrapData, Mode, Status) ->
ActualCfg =
case get(test_server_create_priv_dir) of
@@ -2937,7 +2970,7 @@ run_test_cases_loop([{Mod,Case}|Cases], Config, TimetrapData, Mode, Status) ->
run_test_cases_loop([{Mod,Case,[ActualCfg]}|Cases], Config,
TimetrapData, Mode, Status);
-run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status) ->
+run_test_cases_loop([{Mod,Func,Args}=Case|Cases], Config, TimetrapData, Mode0, Status) ->
{Num,RunInit} =
case FwMod = get_fw_mod(?MODULE) of
Mod when Func == error_in_suite ->
@@ -2947,6 +2980,14 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status)
run_init}
end,
+ Mode =
+ case Mode0 of
+ [{_,[{repeat,{_,_,_}}],_}|RestMode] ->
+ RestMode;
+ _ ->
+ Mode0
+ end,
+
%% check the current execution mode and save info about the case if
%% detected that printouts to common log files is handled later
@@ -2974,36 +3015,42 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status)
if is_tuple(RetVal) -> element(1,RetVal);
true -> undefined
end,
- {Failed,Status1} =
+ {Result,Failed,Status1} =
case RetTag of
Skip when Skip==skip; Skip==skipped ->
- {false,update_status(skipped, Mod, Func, Status)};
+ {skipped,false,update_status(skipped, Mod, Func, Status)};
Fail when Fail=='EXIT'; Fail==failed ->
- {true,update_status(failed, Mod, Func, Status)};
+ {failed,true,update_status(failed, Mod, Func, Status)};
_ when Time==died, RetVal=/=ok ->
- {true,update_status(failed, Mod, Func, Status)};
+ {failed,true,update_status(failed, Mod, Func, Status)};
_ ->
- {false,update_status(ok, Mod, Func, Status)}
+ {ok,false,update_status(ok, Mod, Func, Status)}
end,
case check_prop(sequence, Mode) of
false ->
+ {Cases1,Mode1} =
+ check_repeat_testcase(Case,Result,Cases,Mode0),
stop_minor_log_file(),
- run_test_cases_loop(Cases, Config, TimetrapData, Mode, Status1);
+ run_test_cases_loop(Cases1, Config, TimetrapData, Mode1, Status1);
Ref ->
%% the case is in a sequence; we must check the result and
%% determine if the following cases should run or be skipped
if not Failed -> % proceed with next case
+ {Cases1,Mode1} =
+ check_repeat_testcase(Case,Result,Cases,Mode0),
stop_minor_log_file(),
- run_test_cases_loop(Cases, Config, TimetrapData, Mode, Status1);
+ run_test_cases_loop(Cases1, Config, TimetrapData, Mode1, Status1);
true -> % skip rest of cases in sequence
print(minor, "~n*** ~tw failed.~n"
" Skipping all other cases in sequence.",
[Func]),
+ {Cases1,Mode1} =
+ check_repeat_testcase(Case,Result,Cases,Mode0),
Reason = {failed,{Mod,Func}},
- Cases2 = skip_cases_upto(Ref, Cases, Reason, tc,
+ Cases2 = skip_cases_upto(Ref, Cases1, Reason, tc,
Mode, auto_skip_case),
stop_minor_log_file(),
- run_test_cases_loop(Cases2, Config, TimetrapData, Mode, Status1)
+ run_test_cases_loop(Cases2, Config, TimetrapData, Mode1, Status1)
end
end;
%% the test case is being executed in parallel with the main process (and
@@ -3012,7 +3059,8 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status)
%% io from Pid will be buffered by the test_server_io process and
%% handled later, so we have to save info about the case
queue_test_case_io(undefined, Pid, Num+1, Mod, Func),
- run_test_cases_loop(Cases, Config, TimetrapData, Mode, Status)
+ {Cases1,Mode1} = check_repeat_testcase(Case,ok,Cases,Mode0),
+ run_test_cases_loop(Cases1, Config, TimetrapData, Mode1, Status)
end;
%% TestSpec processing finished
@@ -3450,9 +3498,19 @@ modify_cases_upto1(Ref, {skip,Reason,FType,Mode,SkipType},
T, Orig, Alt)
end;
-%% next is some other case, ignore or copy
-modify_cases_upto1(Ref, {skip,_,_,_,_}=Op, [_Other|T], Orig, Alt) ->
+%% next is a repeated test case
+modify_cases_upto1(Ref, {skip,Reason,_,Mode,SkipType}=Op,
+ [{repeat,{_M,_F}=MF,_Repeat}|T], Orig, Alt) ->
+ modify_cases_upto1(Ref, Op, T, Orig, [{SkipType,{MF,Reason},Mode}|Alt]);
+
+%% next is an already skipped case, ignore or copy
+modify_cases_upto1(Ref, {skip,_,_,_,_}=Op, [{SkipType,_,_}|T], Orig, Alt)
+ when SkipType=:=skip_case; SkipType=:=auto_skip_case ->
modify_cases_upto1(Ref, Op, T, Orig, Alt);
+
+%% next is some other case, mark as skipped or copy
+modify_cases_upto1(Ref, {skip,Reason,_,Mode,SkipType}=Op, [Other|T], Orig, Alt) ->
+ modify_cases_upto1(Ref, Op, T, Orig, [{SkipType,{Other,Reason},Mode}|Alt]);
modify_cases_upto1(Ref, CopyOp, [C|T], Orig, Alt) ->
modify_cases_upto1(Ref, CopyOp, T, [C|Orig], [C|Alt]).
@@ -4795,6 +4853,14 @@ collect_cases({make,InitMFA,CaseList,FinMFA}, St0, Mode) ->
{error,_Reason} = Error -> Error
end;
+collect_cases({repeat,{Module, Case}, Repeat}, St, Mode) ->
+ case catch collect_case([Case], St#cc{mod=Module}, [], Mode) of
+ {ok, [{Module,Case}], _} ->
+ {ok, [{repeat,{Module, Case}, Repeat}], St};
+ Other ->
+ {error,Other}
+ end;
+
collect_cases({Module, Cases}, St, Mode) when is_list(Cases) ->
case (catch collect_case(Cases, St#cc{mod=Module}, [], Mode)) of
Result = {ok,_,_} ->
@@ -5758,3 +5824,42 @@ encoding(File) ->
E ->
E
end.
+
+check_repeat_testcase(Case,Result,Cases,
+ [{Ref,[{repeat,RepeatData0}],StartTime}|Mode0]) ->
+ case do_update_repeat_data(Result,RepeatData0) of
+ false ->
+ {Cases,Mode0};
+ RepeatData ->
+ {[Case|Cases],[{Ref,[{repeat,RepeatData}],StartTime}|Mode0]}
+ end;
+check_repeat_testcase(_,_,Cases,Mode) ->
+ {Cases,Mode}.
+
+do_update_repeat_data(_,{RT,N,N}) when is_integer(N) ->
+ report_repeat_testcase(N,N),
+ report_stop_repeat_testcase(done,{RT,N}),
+ false;
+do_update_repeat_data(ok,{repeat_until_ok=RT,M,N}) ->
+ report_repeat_testcase(M,N),
+ report_stop_repeat_testcase(RT,{RT,N}),
+ false;
+do_update_repeat_data(failed,{repeat_until_fail=RT,M,N}) ->
+ report_repeat_testcase(M,N),
+ report_stop_repeat_testcase(RT,{RT,N}),
+ false;
+do_update_repeat_data(_,{RT,M,N}) when is_integer(M) ->
+ report_repeat_testcase(M,N),
+ {RT,M+1,N};
+do_update_repeat_data(_,{_,M,N}=RepeatData) ->
+ report_repeat_testcase(M,N),
+ RepeatData.
+
+report_stop_repeat_testcase(Reason,RepVal) ->
+ print(minor, "~n*** Stopping test case repeat operation: ~w", [Reason]),
+ print(1, "Stopping test case repeat operation: ~w", [RepVal]).
+
+report_repeat_testcase(M,forever) ->
+ print(minor, "~n=== Repeated test case: ~w of infinity", [M]);
+report_repeat_testcase(M,N) ->
+ print(minor, "~n=== Repeated test case: ~w of ~w", [M,N]).