From a2e6001d96b347b4b87fc36873c5b0443cd855dd Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Fri, 6 Aug 2010 18:03:37 +0200 Subject: Fix problems with sequences and sub-groups Getting sequences to work properly when a sub-group fails in init_per_group has been unsuccessful (see tests in ct_sequence_1_SUITE). Skipping parent group tests from the auto_skip conf clause fails because of problems with where/how the group status is stored. Work in progress... --- lib/test_server/src/test_server_ctrl.erl | 93 ++++++++++++++++++++++++++------ 1 file changed, 76 insertions(+), 17 deletions(-) (limited to 'lib/test_server') diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index 72f274d63a..89ce03232d 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -2270,12 +2270,17 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], Config, TimetrapData, Mode, Status) when Type==conf; Type==make -> + %%! --- Fri Aug 6 13:26:19 2010 --- peppe was here! + io:format(user, "## {auto_skip,~p,~p,~p}: ~n~p~n~p~n", [get_name(Mode),Ref,Case,Mode,Status]), + file:set_cwd(filename:dirname(get(test_server_dir))), CurrIOHandler = get(test_server_common_io_handler), + ParentMode = tl(Mode), + %% check and update the mode for test case execution and io msg handling case {curr_ref(Mode),check_props(parallel, Mode)} of {Ref,Ref} -> - case check_props(parallel, tl(Mode)) of + case check_props(parallel, ParentMode) of false -> %% this is a skipped end conf for a top level parallel group, %% buffered io can be flushed @@ -2283,7 +2288,7 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], set_io_buffering(undefined), {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, false, SkipMode), test_server_sup:framework_call(report, [tc_auto_skip,{?pl2a(Mod),Func,Comment}]), - run_test_cases_loop(Cases, Config, TimetrapData, tl(Mode), + run_test_cases_loop(Cases, Config, TimetrapData, ParentMode, delete_status(Ref, Status)); _ -> %% this is a skipped end conf for a parallel group nested under a @@ -2300,7 +2305,7 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], _ -> ok end, - run_test_cases_loop(Cases, Config, TimetrapData, tl(Mode), + run_test_cases_loop(Cases, Config, TimetrapData, ParentMode, delete_status(Ref, Status)) end; {Ref,false} -> @@ -2308,7 +2313,31 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], %% nested under a parallel group {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, false, SkipMode), test_server_sup:framework_call(report, [tc_auto_skip,{?pl2a(Mod),Func,Comment}]), - run_test_cases_loop(Cases, Config, TimetrapData, tl(Mode), + + %% Check if this group is auto skipped because of error in the init conf. + %% If so, check if the parent group is a sequence, and if it is, skip + %% all proceeding tests in that group. + GrName = get_name(Mode), + Cases1 = + case get_tc_results(Status) of + {_,_,Fails} when length(Fails) > 0 -> + case lists:member({group_result,GrName}, Fails) of + true -> + case check_prop(sequence, ParentMode) of + false -> + Cases; + ParentRef -> + Reason = {group_result,GrName,failed}, + skip_cases_upto(ParentRef, Cases, + Reason, tc, Mode) + end; + false -> + Cases + end; + _ -> + Cases + end, + run_test_cases_loop(Cases1, Config, TimetrapData, ParentMode, delete_status(Ref, Status)); {Ref,_} -> %% this is a skipped end conf for a non-parallel group nested under @@ -2382,6 +2411,9 @@ run_test_cases_loop([{skip_case,{Case,Comment}}|Cases], run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, Config, TimetrapData, Mode0, Status) -> +%%! --- Fri Aug 6 13:26:19 2010 --- peppe was here! +io:format(user, "## {conf,~p,~p,~p}: ~p~n", [proplists:get_value(name,Props),Ref,Func,Status]), + CurrIOHandler = get(test_server_common_io_handler), %% check and update the mode for test case execution and io msg handling {StartConf,Mode,IOHandler,ConfTime,Status1} = @@ -2627,22 +2659,28 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, {_,Fail,_} when element(1,Fail) == 'EXIT'; element(1,Fail) == timetrap_timeout; element(1,Fail) == failed -> - {Cases2,Config1} = + {Cases2,Config1,Status3} = if StartConf -> ReportAbortRepeat(failed), print(minor, "~n*** ~p failed.~n" " Skipping all cases.", [Func]), Reason = {failed,{Mod,Func,Fail}}, - {skip_cases_upto(Ref, Cases, Reason, conf, CurrMode),Config}; + {skip_cases_upto(Ref, Cases, Reason, conf, CurrMode), + Config, + update_status(failed, group_result, get_name(Mode0), + delete_status(Ref, Status2))}; not StartConf -> ReportRepeatStop(), print_conf_time(ConfTime), - {Cases,tl(Config)} + {Cases,tl(Config),delete_status(Ref, Status2)} end, set_io_buffering(IOHandler), stop_minor_log_file(), - run_test_cases_loop(Cases2, Config1, TimetrapData, Mode, - delete_status(Ref, Status2)); + + %%! --- Fri Aug 6 13:25:35 2010 --- peppe was here! + io:format(user, "## {~p,~p,~p}: Status = ~p~n", [get_name(Mode0),Ref,Func,Status3]), + + run_test_cases_loop(Cases2, Config1, TimetrapData, Mode, Status3); {died,Why,_} when Func == init_per_suite -> print(minor, "~n*** Unexpected exit during init_per_suite.~n", []), Reason = {failed,{Mod,init_per_suite,Why}}, @@ -2686,22 +2724,37 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, Mode, Status2); {_,_EndConfRetVal,Opts} -> - %% check if return_group_result is set (ok, skipped or failed) and - %% if so return the value to the group "above" so that result may be - %% used for evaluating repeat_until_* - Status3 = + %% Check if return_group_result is set (ok, skipped or failed) and + %% if so: + %% 1) *If* the parent group is a sequence, skip all proceeding tests + %% in that group. + %% 2) Return the value to the group "above" so that result may be + %% used for evaluating a 'repeat_until_*' property. + GrName = get_name(Mode0, Func), + {Cases2,Status3} = case lists:keysearch(return_group_result, 1, Opts) of + {value,{_,failed}} -> + case {curr_ref(Mode),check_prop(sequence, Mode)} of + {ParentRef,ParentRef} -> + Reason = {group_result,GrName,failed}, + {skip_cases_upto(ParentRef, Cases, Reason, tc, Mode), + update_status(failed, group_result, GrName, + delete_status(Ref, Status2))}; + _ -> + {Cases,update_status(failed, group_result, GrName, + delete_status(Ref, Status2))} + end; {value,{_,GroupResult}} -> - update_status(GroupResult, group_result, Func, - delete_status(Ref, Status2)); + {Cases,update_status(GroupResult, group_result, GrName, + delete_status(Ref, Status2))}; false -> - delete_status(Ref, Status2) + {Cases,delete_status(Ref, Status2)} end, print_conf_time(ConfTime), ReportRepeatStop(), set_io_buffering(IOHandler), stop_minor_log_file(), - run_test_cases_loop(Cases, tl(Config), TimetrapData, Mode, Status3) + run_test_cases_loop(Cases2, tl(Config), TimetrapData, Mode, Status3) end; run_test_cases_loop([{make,Ref,{Mod,Func,Args}}|Cases0], Config, TimetrapData, Mode, Status) -> @@ -2870,6 +2923,12 @@ check_props(Attrib, Mode) -> [Ref|_] -> Ref end. +get_name(Mode, Def) -> + case get_name(Mode) of + undefined -> Def; + Name -> Name + end. + get_name([{_Ref,Props,_}|_]) -> proplists:get_value(name, Props); get_name([]) -> -- cgit v1.2.3 From 346a1fc0f0ecbaeafe2570102211a3145e66656b Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Wed, 11 Aug 2010 12:07:03 +0200 Subject: Fix error in handling of sequence groups --- lib/test_server/src/test_server_ctrl.erl | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) (limited to 'lib/test_server') diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index 89ce03232d..9ab526a26b 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -2269,10 +2269,6 @@ maybe_get_privdir() -> run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], Config, TimetrapData, Mode, Status) when Type==conf; Type==make -> - - %%! --- Fri Aug 6 13:26:19 2010 --- peppe was here! - io:format(user, "## {auto_skip,~p,~p,~p}: ~n~p~n~p~n", [get_name(Mode),Ref,Case,Mode,Status]), - file:set_cwd(filename:dirname(get(test_server_dir))), CurrIOHandler = get(test_server_common_io_handler), ParentMode = tl(Mode), @@ -2410,10 +2406,6 @@ run_test_cases_loop([{skip_case,{Case,Comment}}|Cases], %% a start *or* end conf case, wrapping test cases or other conf cases run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, Config, TimetrapData, Mode0, Status) -> - -%%! --- Fri Aug 6 13:26:19 2010 --- peppe was here! -io:format(user, "## {conf,~p,~p,~p}: ~p~n", [proplists:get_value(name,Props),Ref,Func,Status]), - CurrIOHandler = get(test_server_common_io_handler), %% check and update the mode for test case execution and io msg handling {StartConf,Mode,IOHandler,ConfTime,Status1} = @@ -2667,7 +2659,7 @@ io:format(user, "## {conf,~p,~p,~p}: ~p~n", [proplists:get_value(name,Props),Ref Reason = {failed,{Mod,Func,Fail}}, {skip_cases_upto(Ref, Cases, Reason, conf, CurrMode), Config, - update_status(failed, group_result, get_name(Mode0), + update_status(failed, group_result, get_name(Mode), delete_status(Ref, Status2))}; not StartConf -> ReportRepeatStop(), @@ -2676,10 +2668,6 @@ io:format(user, "## {conf,~p,~p,~p}: ~p~n", [proplists:get_value(name,Props),Ref end, set_io_buffering(IOHandler), stop_minor_log_file(), - - %%! --- Fri Aug 6 13:25:35 2010 --- peppe was here! - io:format(user, "## {~p,~p,~p}: Status = ~p~n", [get_name(Mode0),Ref,Func,Status3]), - run_test_cases_loop(Cases2, Config1, TimetrapData, Mode, Status3); {died,Why,_} when Func == init_per_suite -> print(minor, "~n*** Unexpected exit during init_per_suite.~n", []), -- cgit v1.2.3 From ac713b2123e0d5b43ea65b259aeb2e228d1ece9d Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Fri, 20 Aug 2010 16:26:13 +0200 Subject: Fix error in the repeat_until_* group property handling --- lib/test_server/src/test_server_ctrl.erl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'lib/test_server') diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index 9ab526a26b..1dc5646184 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -2567,7 +2567,7 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, {true,EndStatus,RestCs,Fun}; {repeat_until_any_ok,_} -> {RestCs,Fun} = case get_tc_results(Status1) of - {Ok,_,_} when length(Ok) > 0 -> + {Ok,_,_Fails} when length(Ok) > 0 -> {Cases1,ReportStop}; _ -> {CopiedCases++Cases1,?void_fun} @@ -2734,9 +2734,10 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, end; {value,{_,GroupResult}} -> {Cases,update_status(GroupResult, group_result, GrName, - delete_status(Ref, Status2))}; + delete_status(Ref, Status2))}; false -> - {Cases,delete_status(Ref, Status2)} + {Cases,update_status(ok, group_result, GrName, + delete_status(Ref, Status2))} end, print_conf_time(ConfTime), ReportRepeatStop(), @@ -2876,7 +2877,10 @@ get_copied_cases([{_,{_,Cases}} | _Status]) -> Cases. get_tc_results([{_,{OkSkipFail,_}} | _Status]) -> - OkSkipFail. + OkSkipFail; +get_tc_results([]) -> % in case init_per_suite crashed + {[],[],[]}. + conf(Ref, Props) -> {Ref,Props,?now}. -- cgit v1.2.3