aboutsummaryrefslogtreecommitdiffstats
path: root/lib/test_server/src
diff options
context:
space:
mode:
authorLukas Larsson <[email protected]>2010-09-10 14:20:36 +0200
committerLukas Larsson <[email protected]>2010-09-10 14:20:36 +0200
commit3bb93b0491414fa571e284144bdc6de3e998273a (patch)
tree7a335caa1a4f39d073add45572cc3b5b0330dc38 /lib/test_server/src
parente5bcc889833a47d03fd427573ee0551977353f24 (diff)
parent2162c7f0f1e9abab7126a0606172ae97deaba44d (diff)
downloadotp-3bb93b0491414fa571e284144bdc6de3e998273a.tar.gz
otp-3bb93b0491414fa571e284144bdc6de3e998273a.tar.bz2
otp-3bb93b0491414fa571e284144bdc6de3e998273a.zip
Merge branch 'peppe/common_test/dev/OTP-8753' into dev
* peppe/common_test/dev/OTP-8753: Fix problem with prepared_tests tuple not being recognized Add verification terms to test suite Fix various test suite, makefile, and doc related problems Fix error in the repeat_until_* group property handling Add test suite for groups with repeat property Add verification events to sequence group test suite Fix error in handling of sequence groups Fix error in test suite Fix problems with sequences and sub-groups Add test cases to sequence suite Fix error with group term in ct:run_test/1 Add test suite to examplify problem with nested groups in a sequence
Diffstat (limited to 'lib/test_server/src')
-rw-r--r--lib/test_server/src/test_server_ctrl.erl93
1 files changed, 72 insertions, 21 deletions
diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl
index 72f274d63a..1dc5646184 100644
--- a/lib/test_server/src/test_server_ctrl.erl
+++ b/lib/test_server/src/test_server_ctrl.erl
@@ -2269,13 +2269,14 @@ maybe_get_privdir() ->
run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases],
Config, TimetrapData, Mode, Status) when Type==conf;
Type==make ->
-
file:set_cwd(filename:dirname(get(test_server_dir))),
CurrIOHandler = get(test_server_common_io_handler),
+ ParentMode = tl(Mode),
+
%% check and update the mode for test case execution and io msg handling
case {curr_ref(Mode),check_props(parallel, Mode)} of
{Ref,Ref} ->
- case check_props(parallel, tl(Mode)) of
+ case check_props(parallel, ParentMode) of
false ->
%% this is a skipped end conf for a top level parallel group,
%% buffered io can be flushed
@@ -2283,7 +2284,7 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases],
set_io_buffering(undefined),
{Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, false, SkipMode),
test_server_sup:framework_call(report, [tc_auto_skip,{?pl2a(Mod),Func,Comment}]),
- run_test_cases_loop(Cases, Config, TimetrapData, tl(Mode),
+ run_test_cases_loop(Cases, Config, TimetrapData, ParentMode,
delete_status(Ref, Status));
_ ->
%% this is a skipped end conf for a parallel group nested under a
@@ -2300,7 +2301,7 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases],
_ ->
ok
end,
- run_test_cases_loop(Cases, Config, TimetrapData, tl(Mode),
+ run_test_cases_loop(Cases, Config, TimetrapData, ParentMode,
delete_status(Ref, Status))
end;
{Ref,false} ->
@@ -2308,7 +2309,31 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases],
%% nested under a parallel group
{Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, false, SkipMode),
test_server_sup:framework_call(report, [tc_auto_skip,{?pl2a(Mod),Func,Comment}]),
- run_test_cases_loop(Cases, Config, TimetrapData, tl(Mode),
+
+ %% Check if this group is auto skipped because of error in the init conf.
+ %% If so, check if the parent group is a sequence, and if it is, skip
+ %% all proceeding tests in that group.
+ GrName = get_name(Mode),
+ Cases1 =
+ case get_tc_results(Status) of
+ {_,_,Fails} when length(Fails) > 0 ->
+ case lists:member({group_result,GrName}, Fails) of
+ true ->
+ case check_prop(sequence, ParentMode) of
+ false ->
+ Cases;
+ ParentRef ->
+ Reason = {group_result,GrName,failed},
+ skip_cases_upto(ParentRef, Cases,
+ Reason, tc, Mode)
+ end;
+ false ->
+ Cases
+ end;
+ _ ->
+ Cases
+ end,
+ run_test_cases_loop(Cases1, Config, TimetrapData, ParentMode,
delete_status(Ref, Status));
{Ref,_} ->
%% this is a skipped end conf for a non-parallel group nested under
@@ -2381,7 +2406,6 @@ run_test_cases_loop([{skip_case,{Case,Comment}}|Cases],
%% a start *or* end conf case, wrapping test cases or other conf cases
run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0,
Config, TimetrapData, Mode0, Status) ->
-
CurrIOHandler = get(test_server_common_io_handler),
%% check and update the mode for test case execution and io msg handling
{StartConf,Mode,IOHandler,ConfTime,Status1} =
@@ -2543,7 +2567,7 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0,
{true,EndStatus,RestCs,Fun};
{repeat_until_any_ok,_} ->
{RestCs,Fun} = case get_tc_results(Status1) of
- {Ok,_,_} when length(Ok) > 0 ->
+ {Ok,_,_Fails} when length(Ok) > 0 ->
{Cases1,ReportStop};
_ ->
{CopiedCases++Cases1,?void_fun}
@@ -2627,22 +2651,24 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0,
{_,Fail,_} when element(1,Fail) == 'EXIT';
element(1,Fail) == timetrap_timeout;
element(1,Fail) == failed ->
- {Cases2,Config1} =
+ {Cases2,Config1,Status3} =
if StartConf ->
ReportAbortRepeat(failed),
print(minor, "~n*** ~p failed.~n"
" Skipping all cases.", [Func]),
Reason = {failed,{Mod,Func,Fail}},
- {skip_cases_upto(Ref, Cases, Reason, conf, CurrMode),Config};
+ {skip_cases_upto(Ref, Cases, Reason, conf, CurrMode),
+ Config,
+ update_status(failed, group_result, get_name(Mode),
+ delete_status(Ref, Status2))};
not StartConf ->
ReportRepeatStop(),
print_conf_time(ConfTime),
- {Cases,tl(Config)}
+ {Cases,tl(Config),delete_status(Ref, Status2)}
end,
set_io_buffering(IOHandler),
stop_minor_log_file(),
- run_test_cases_loop(Cases2, Config1, TimetrapData, Mode,
- delete_status(Ref, Status2));
+ run_test_cases_loop(Cases2, Config1, TimetrapData, Mode, Status3);
{died,Why,_} when Func == init_per_suite ->
print(minor, "~n*** Unexpected exit during init_per_suite.~n", []),
Reason = {failed,{Mod,init_per_suite,Why}},
@@ -2686,22 +2712,38 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0,
Mode, Status2);
{_,_EndConfRetVal,Opts} ->
- %% check if return_group_result is set (ok, skipped or failed) and
- %% if so return the value to the group "above" so that result may be
- %% used for evaluating repeat_until_*
- Status3 =
+ %% Check if return_group_result is set (ok, skipped or failed) and
+ %% if so:
+ %% 1) *If* the parent group is a sequence, skip all proceeding tests
+ %% in that group.
+ %% 2) Return the value to the group "above" so that result may be
+ %% used for evaluating a 'repeat_until_*' property.
+ GrName = get_name(Mode0, Func),
+ {Cases2,Status3} =
case lists:keysearch(return_group_result, 1, Opts) of
+ {value,{_,failed}} ->
+ case {curr_ref(Mode),check_prop(sequence, Mode)} of
+ {ParentRef,ParentRef} ->
+ Reason = {group_result,GrName,failed},
+ {skip_cases_upto(ParentRef, Cases, Reason, tc, Mode),
+ update_status(failed, group_result, GrName,
+ delete_status(Ref, Status2))};
+ _ ->
+ {Cases,update_status(failed, group_result, GrName,
+ delete_status(Ref, Status2))}
+ end;
{value,{_,GroupResult}} ->
- update_status(GroupResult, group_result, Func,
- delete_status(Ref, Status2));
+ {Cases,update_status(GroupResult, group_result, GrName,
+ delete_status(Ref, Status2))};
false ->
- delete_status(Ref, Status2)
+ {Cases,update_status(ok, group_result, GrName,
+ delete_status(Ref, Status2))}
end,
print_conf_time(ConfTime),
ReportRepeatStop(),
set_io_buffering(IOHandler),
stop_minor_log_file(),
- run_test_cases_loop(Cases, tl(Config), TimetrapData, Mode, Status3)
+ run_test_cases_loop(Cases2, tl(Config), TimetrapData, Mode, Status3)
end;
run_test_cases_loop([{make,Ref,{Mod,Func,Args}}|Cases0], Config, TimetrapData, Mode, Status) ->
@@ -2835,7 +2877,10 @@ get_copied_cases([{_,{_,Cases}} | _Status]) ->
Cases.
get_tc_results([{_,{OkSkipFail,_}} | _Status]) ->
- OkSkipFail.
+ OkSkipFail;
+get_tc_results([]) -> % in case init_per_suite crashed
+ {[],[],[]}.
+
conf(Ref, Props) ->
{Ref,Props,?now}.
@@ -2870,6 +2915,12 @@ check_props(Attrib, Mode) ->
[Ref|_] -> Ref
end.
+get_name(Mode, Def) ->
+ case get_name(Mode) of
+ undefined -> Def;
+ Name -> Name
+ end.
+
get_name([{_Ref,Props,_}|_]) ->
proplists:get_value(name, Props);
get_name([]) ->