From a811257b11a5c7833047e00c676aca00860161fe Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Mon, 14 Oct 2013 16:34:19 +0200 Subject: Change skip vs auto_skip behaviour When a configuration function returns {skip,Reason}, all affected test cases will now get status skipped, not auto_skipped. Furthermore, auto skipped cases will be reported as "auto_skipped" in the textual log file (not "skipped", as in previous versions). OTP-11305 --- lib/test_server/src/test_server_ctrl.erl | 178 ++++++++++++++++++++----------- 1 file changed, 113 insertions(+), 65 deletions(-) (limited to 'lib/test_server/src/test_server_ctrl.erl') diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index d0f31af198..9ab51df028 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -2221,20 +2221,22 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], {Ref,Ref} -> case check_props(parallel, ParentMode) of false -> - %% this is a skipped end conf for a top level parallel group, - %% buffered io can be flushed + %% this is a skipped end conf for a top level parallel + %% group, buffered io can be flushed handle_test_case_io_and_status(), set_io_buffering(undefined), - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, false, SkipMode), + {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, + false, SkipMode), test_server_sup:framework_call(report, [tc_auto_skip, {Mod,Func,Comment}]), run_test_cases_loop(Cases, Config, TimetrapData, ParentMode, delete_status(Ref, Status)); _ -> - %% this is a skipped end conf for a parallel group nested under a - %% parallel group (io buffering is active) + %% this is a skipped end conf for a parallel group nested + %% under a parallel group (io buffering is active) wait_for_cases(Ref), - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, true, SkipMode), + {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, + true, SkipMode), test_server_sup:framework_call(report, [tc_auto_skip, {Mod,Func,Comment}]), case CurrIOHandler of @@ -2246,18 +2248,21 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], _ -> ok end, - run_test_cases_loop(Cases, Config, TimetrapData, ParentMode, + run_test_cases_loop(Cases, Config, + TimetrapData, ParentMode, delete_status(Ref, Status)) end; {Ref,false} -> %% this is a skipped end conf for a non-parallel group that's not %% nested under a parallel group - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, false, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip,{Mod,Func,Comment}]), - - %% Check if this group is auto skipped because of error in the init conf. - %% If so, check if the parent group is a sequence, and if it is, skip - %% all proceeding tests in that group. + {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, + false, SkipMode), + test_server_sup:framework_call(report, [tc_auto_skip, + {Mod,Func,Comment}]), + + %% Check if this group is auto skipped because of error in the + %% init conf. If so, check if the parent group is a sequence, + %% and if it is, skip all proceeding tests in that group. GrName = get_name(Mode), Cases1 = case get_tc_results(Status) of @@ -2270,7 +2275,8 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], ParentRef -> Reason = {group_result,GrName,failed}, skip_cases_upto(ParentRef, Cases, - Reason, tc, Mode) + Reason, tc, Mode, + auto_skip_case) end; false -> Cases @@ -2283,8 +2289,10 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], {Ref,_} -> %% this is a skipped end conf for a non-parallel group nested under %% a parallel group (io buffering is active) - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, true, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip,{Mod,Func,Comment}]), + {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, + true, SkipMode), + test_server_sup:framework_call(report, [tc_auto_skip, + {Mod,Func,Comment}]), case CurrIOHandler of {Ref,_} -> %% current_io_handler was set by start conf of this @@ -2299,20 +2307,27 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], {_,false} -> %% this is a skipped start conf for a group which is not nested %% under a parallel group - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, false, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip,{Mod,Func,Comment}]), - run_test_cases_loop(Cases, Config, TimetrapData, [conf(Ref,[])|Mode], Status); + {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, + false, SkipMode), + test_server_sup:framework_call(report, [tc_auto_skip, + {Mod,Func,Comment}]), + run_test_cases_loop(Cases, Config, TimetrapData, + [conf(Ref,[])|Mode], Status); {_,Ref0} when is_reference(Ref0) -> - %% this is a skipped start conf for a group nested under a parallel group - %% and if this is the first nested group, io buffering must be activated + %% this is a skipped start conf for a group nested under a parallel + %% group and if this is the first nested group, io buffering must + %% be activated if CurrIOHandler == undefined -> set_io_buffering({Ref,self()}); true -> ok end, - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, true, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip,{Mod,Func,Comment}]), - run_test_cases_loop(Cases, Config, TimetrapData, [conf(Ref,[])|Mode], Status) + {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, + true, SkipMode), + test_server_sup:framework_call(report, [tc_auto_skip, + {Mod,Func,Comment}]), + run_test_cases_loop(Cases, Config, TimetrapData, + [conf(Ref,[])|Mode], Status) end; run_test_cases_loop([{auto_skip_case,{Case,Comment},SkipMode}|Cases], @@ -2333,7 +2348,8 @@ run_test_cases_loop([{skip_case,{conf,Ref,Case,Comment}}|Cases0], {Cases0,tl(Config)}; _ -> %% skipped start conf - {skip_cases_upto(Ref, Cases0, Comment, conf, Mode),Config} + {skip_cases_upto(Ref, Cases0, Comment, conf, Mode, skip_case), + Config} end, test_server_sup:framework_call(report, [tc_user_skip,{Mod,Func,Comment}]), run_test_cases_loop(Cases, Config1, TimetrapData, Mode, @@ -2597,7 +2613,8 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, "~n*** ~w returned bad elements in Config: ~p.~n", [Func,Bad]), Reason = {failed,{Mod,init_per_suite,bad_return}}, - Cases2 = skip_cases_upto(Ref, Cases, Reason, conf, CurrMode), + Cases2 = skip_cases_upto(Ref, Cases, Reason, conf, CurrMode, + auto_skip_case), set_io_buffering(IOHandler), stop_minor_log_file(), run_test_cases_loop(Cases2, Config, TimetrapData, Mode, @@ -2623,7 +2640,8 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, print(minor, "~n*** ~w failed.~n" " Skipping all cases.", [Func]), Reason = {failed,{Mod,Func,Fail}}, - {skip_cases_upto(Ref, Cases, Reason, conf, CurrMode), + {skip_cases_upto(Ref, Cases, Reason, conf, CurrMode, + auto_skip_case), Config, update_status(failed, group_result, get_name(Mode), delete_status(Ref, Status2))}; @@ -2635,13 +2653,15 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, set_io_buffering(IOHandler), stop_minor_log_file(), run_test_cases_loop(Cases2, Config1, TimetrapData, Mode, Status3); + {_,{Skip,Reason},_} when StartConf and ((Skip==skip) or (Skip==skipped)) -> ReportAbortRepeat(skipped), print(minor, "~n*** ~w skipped.~n" " Skipping all cases.", [Func]), set_io_buffering(IOHandler), stop_minor_log_file(), - run_test_cases_loop(skip_cases_upto(Ref, Cases, Reason, conf, CurrMode), + run_test_cases_loop(skip_cases_upto(Ref, Cases, Reason, conf, + CurrMode, skip_case), Config, TimetrapData, Mode, delete_status(Ref, Status2)); {_,{skip_and_save,Reason,_SavedConfig},_} when StartConf -> @@ -2650,13 +2670,15 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, " Skipping all cases.", [Func]), set_io_buffering(IOHandler), stop_minor_log_file(), - run_test_cases_loop(skip_cases_upto(Ref, Cases, Reason, conf, CurrMode), + run_test_cases_loop(skip_cases_upto(Ref, Cases, Reason, conf, + CurrMode, skip_case), Config, TimetrapData, Mode, delete_status(Ref, Status2)); {_,_Other,_} when Func == init_per_suite -> print(minor, "~n*** init_per_suite failed to return a Config list.~n", []), Reason = {failed,{Mod,init_per_suite,bad_return}}, - Cases2 = skip_cases_upto(Ref, Cases, Reason, conf, CurrMode), + Cases2 = skip_cases_upto(Ref, Cases, Reason, conf, CurrMode, + auto_skip_case), set_io_buffering(IOHandler), stop_minor_log_file(), run_test_cases_loop(Cases2, Config, TimetrapData, Mode, @@ -2668,7 +2690,6 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, stop_minor_log_file(), run_test_cases_loop(Cases, [hd(Config)|Config], TimetrapData, Mode, Status2); - {_,_EndConfRetVal,Opts} -> %% Check if return_group_result is set (ok, skipped or failed) and %% if so: @@ -2683,7 +2704,8 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, case {curr_ref(Mode),check_prop(sequence, Mode)} of {ParentRef,ParentRef} -> Reason = {group_result,GrName,failed}, - {skip_cases_upto(ParentRef, Cases, Reason, tc, Mode), + {skip_cases_upto(ParentRef, Cases, Reason, tc, + Mode, auto_skip_case), update_status(failed, group_result, GrName, delete_status(Ref, Status2))}; _ -> @@ -2701,16 +2723,19 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, ReportRepeatStop(), set_io_buffering(IOHandler), stop_minor_log_file(), - run_test_cases_loop(Cases2, tl(Config), TimetrapData, Mode, Status3) + run_test_cases_loop(Cases2, tl(Config), TimetrapData, + Mode, Status3) end; -run_test_cases_loop([{make,Ref,{Mod,Func,Args}}|Cases0], Config, TimetrapData, Mode, Status) -> +run_test_cases_loop([{make,Ref,{Mod,Func,Args}}|Cases0], Config, TimetrapData, + Mode, Status) -> case run_test_case(Ref, 0, Mod, Func, Args, skip_init, TimetrapData) of {_,Why={'EXIT',_},_} -> print(minor, "~n*** ~w failed.~n" " Skipping all cases.", [Func]), Reason = {failed,{Mod,Func,Why}}, - Cases = skip_cases_upto(Ref, Cases0, Reason, conf, Mode), + Cases = skip_cases_upto(Ref, Cases0, Reason, conf, Mode, + auto_skip_case), stop_minor_log_file(), run_test_cases_loop(Cases, Config, TimetrapData, Mode, Status); {_,_Whatever,_} -> @@ -2791,7 +2816,8 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status) " Skipping all other cases in sequence.", [Func]), Reason = {failed,{Mod,Func}}, - Cases2 = skip_cases_upto(Ref, Cases, Reason, tc, Mode), + Cases2 = skip_cases_upto(Ref, Cases, Reason, tc, + Mode, auto_skip_case), stop_minor_log_file(), run_test_cases_loop(Cases2, Config, TimetrapData, Mode, Status1) end @@ -3079,9 +3105,17 @@ skip_case1(Type, CaseNum, Mod, Func, Comment, Mode) -> Comment1 = reason_to_string(Comment), print(major, "~n=case ~w:~w", [Mod,Func]), - print(major, "=started ~s", [lists:flatten(timestamp_get(""))]), - print(major, "=result skipped: ~ts", [Comment1]), - print(2,"*** Skipping test case #~w ~w ***", [CaseNum,{Mod,Func}]), + print(major, "=started ~s", [lists:flatten(timestamp_get(""))]), + if Type == auto -> + print(major, "=result auto_skipped: ~ts", [Comment1]); + Type == user -> + print(major, "=result skipped: ~ts", [Comment1]) + end, + if CaseNum == 0 -> + print(2,"*** Skipping ~w ***", [{Mod,Func}]); + true -> + print(2,"*** Skipping test case #~w ~w ***", [CaseNum,{Mod,Func}]) + end, TR = xhtml("", [""]), GroupName = case get_name(Mode) of undefined -> ""; @@ -3110,12 +3144,14 @@ skip_case1(Type, CaseNum, Mod, Func, Comment, Mode) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% skip_cases_upto(Ref, Cases, Reason, Origin, Mode) -> Cases1 +%% skip_cases_upto(Ref, Cases, Reason, Origin, Mode, SkipType) -> Cases1 %% +%% SkipType = skip_case | auto_skip_case %% Mark all cases tagged with Ref as skipped. -skip_cases_upto(Ref, Cases, Reason, Origin, Mode) -> - {_,Modified,Rest} = modify_cases_upto(Ref, {skip,Reason,Origin,Mode}, Cases), +skip_cases_upto(Ref, Cases, Reason, Origin, Mode, SkipType) -> + {_,Modified,Rest} = + modify_cases_upto(Ref, {skip,Reason,Origin,Mode,SkipType}, Cases), Modified++Rest. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -3161,25 +3197,33 @@ modify_cases_upto(Ref, ModOp, Cases, Orig, Alt) -> end. %% next case is a conf with same ref, must be end conf = we're done -modify_cases_upto1(Ref, {skip,Reason,conf,Mode}, [{conf,Ref,_Props,MF}|T], Orig, Alt) -> +modify_cases_upto1(Ref, {skip,Reason,conf,_,skip_case}, + [{conf,Ref,_Props,MF}|T], Orig, Alt) -> + {Orig,[{skip_case,{conf,Ref,MF,Reason}}|Alt],T}; +modify_cases_upto1(Ref, {skip,Reason,conf,Mode,auto_skip_case}, + [{conf,Ref,_Props,MF}|T], Orig, Alt) -> {Orig,[{auto_skip_case,{conf,Ref,MF,Reason},Mode}|Alt],T}; modify_cases_upto1(Ref, {copy,NewRef}, [{conf,Ref,Props,MF}=C|T], Orig, Alt) -> {[C|Orig],[{conf,NewRef,update_repeat(Props),MF}|Alt],T}; %% we've skipped all remaining cases in a sequence -modify_cases_upto1(Ref, {skip,_,tc,_}, [{conf,Ref,_Props,_MF}|_]=Cs, Orig, Alt) -> +modify_cases_upto1(Ref, {skip,_,tc,_,_}, + [{conf,Ref,_Props,_MF}|_]=Cs, Orig, Alt) -> {Orig,Alt,Cs}; %% next is a make case -modify_cases_upto1(Ref, {skip,Reason,_,Mode}, [{make,Ref,MF}|T], Orig, Alt) -> - {Orig,[{auto_skip_case,{make,Ref,MF,Reason},Mode}|Alt],T}; +modify_cases_upto1(Ref, {skip,Reason,_,Mode,SkipType}, + [{make,Ref,MF}|T], Orig, Alt) -> + {Orig,[{SkipType,{make,Ref,MF,Reason},Mode}|Alt],T}; modify_cases_upto1(Ref, {copy,NewRef}, [{make,Ref,MF}=M|T], Orig, Alt) -> {[M|Orig],[{make,NewRef,MF}|Alt],T}; %% next case is a user skipped end conf with the same ref = we're done -modify_cases_upto1(Ref, {skip,Reason,_,Mode}, [{skip_case,{Type,Ref,MF,_Cmt}}|T], Orig, Alt) -> - {Orig,[{auto_skip_case,{Type,Ref,MF,Reason},Mode}|Alt],T}; -modify_cases_upto1(Ref, {copy,NewRef}, [{skip_case,{Type,Ref,MF,Cmt}}=C|T], Orig, Alt) -> +modify_cases_upto1(Ref, {skip,Reason,_,Mode,SkipType}, + [{skip_case,{Type,Ref,MF,_Cmt}}|T], Orig, Alt) -> + {Orig,[{SkipType,{Type,Ref,MF,Reason},Mode}|Alt],T}; +modify_cases_upto1(Ref, {copy,NewRef}, + [{skip_case,{Type,Ref,MF,Cmt}}=C|T], Orig, Alt) -> {[C|Orig],[{skip_case,{Type,NewRef,MF,Cmt}}|Alt],T}; %% next is a skip_case, could be one test case or 'all' in suite, we must proceed @@ -3187,13 +3231,17 @@ modify_cases_upto1(Ref, ModOp, [{skip_case,{_F,_Cmt}}=MF|T], Orig, Alt) -> modify_cases_upto1(Ref, ModOp, T, [MF|Orig], [MF|Alt]); %% next is a normal case (possibly in a sequence), mark as skipped, or copy, and proceed -modify_cases_upto1(Ref, {skip,Reason,_,Mode}=Op, [{_M,_F}=MF|T], Orig, Alt) -> +modify_cases_upto1(Ref, {skip,Reason,_,_,skip_case}=Op, + [{_M,_F}=MF|T], Orig, Alt) -> + modify_cases_upto1(Ref, Op, T, Orig, [{skip_case,{MF,Reason}}|Alt]); +modify_cases_upto1(Ref, {skip,Reason,_,Mode,auto_skip_case}=Op, + [{_M,_F}=MF|T], Orig, Alt) -> modify_cases_upto1(Ref, Op, T, Orig, [{auto_skip_case,{MF,Reason},Mode}|Alt]); modify_cases_upto1(Ref, CopyOp, [{_M,_F}=MF|T], Orig, Alt) -> modify_cases_upto1(Ref, CopyOp, T, [MF|Orig], [MF|Alt]); %% next is some other case, ignore or copy -modify_cases_upto1(Ref, {skip,_,_,_}=Op, [_|T], Orig, Alt) -> +modify_cases_upto1(Ref, {skip,_,_,_,_}=Op, [_|T], Orig, Alt) -> modify_cases_upto1(Ref, Op, T, Orig, Alt); modify_cases_upto1(Ref, CopyOp, [C|T], Orig, Alt) -> modify_cases_upto1(Ref, CopyOp, T, [C|Orig], [C|Alt]). @@ -3704,8 +3752,8 @@ progress(skip, CaseNum, Mod, Func, Loc, Reason, Time, fun() -> {?auto_skip_color,auto_skip} end, fun() -> {?user_skip_color,skip} end), print(major, "=result skipped", []), - print(1, "*** SKIPPED *** ~ts", - [get_info_str(Func, CaseNum, get(test_server_cases))]), + print(1, "*** SKIPPED ~ts ***", + [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), test_server_sup:framework_call(report, [tc_done,{Mod,Func, {skipped,Reason1}}]), ReasonStr = reason_to_string(Reason1), @@ -3734,8 +3782,8 @@ progress(skip, CaseNum, Mod, Func, Loc, Reason, Time, progress(failed, CaseNum, Mod, Func, Loc, timetrap_timeout, T, Comment0, {St0,St1}) -> print(major, "=result failed: timeout, ~p", [Loc]), - print(1, "*** FAILED *** ~ts", - [get_info_str(Func, CaseNum, get(test_server_cases))]), + print(1, "*** FAILED ~ts ***", + [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), test_server_sup:framework_call(report, [tc_done,{Mod,Func, {failed,timetrap_timeout}}]), @@ -3760,8 +3808,8 @@ progress(failed, CaseNum, Mod, Func, Loc, timetrap_timeout, T, progress(failed, CaseNum, Mod, Func, Loc, {testcase_aborted,Reason}, _T, Comment0, {St0,St1}) -> print(major, "=result failed: testcase_aborted, ~p", [Loc]), - print(1, "*** FAILED *** ~ts", - [get_info_str(Func, CaseNum, get(test_server_cases))]), + print(1, "*** FAILED ~ts ***", + [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), test_server_sup:framework_call(report, [tc_done,{Mod,Func, {failed,testcase_aborted}}]), @@ -3786,8 +3834,8 @@ progress(failed, CaseNum, Mod, Func, Loc, {testcase_aborted,Reason}, _T, progress(failed, CaseNum, Mod, Func, unknown, Reason, Time, Comment0, {St0,St1}) -> print(major, "=result failed: ~p, ~w", [Reason,unknown]), - print(1, "*** FAILED *** ~ts", - [get_info_str(Func, CaseNum, get(test_server_cases))]), + print(1, "*** FAILED ~ts ***", + [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), test_server_sup:framework_call(report, [tc_done,{Mod,Func, {failed,Reason}}]), TimeStr = io_lib:format(if is_float(Time) -> "~.3fs"; @@ -3822,8 +3870,8 @@ progress(failed, CaseNum, Mod, Func, unknown, Reason, Time, progress(failed, CaseNum, Mod, Func, Loc, Reason, Time, Comment0, {St0,St1}) -> print(major, "=result failed: ~p, ~p", [Reason,Loc]), - print(1, "*** FAILED *** ~ts", - [get_info_str(Func, CaseNum, get(test_server_cases))]), + print(1, "*** FAILED ~ts ***", + [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), test_server_sup:framework_call(report, [tc_done,{Mod,Func, {failed,Reason}}]), TimeStr = io_lib:format(if is_float(Time) -> "~.3fs"; @@ -3933,11 +3981,11 @@ update_skip_counters(RetVal, {US,AS}) -> {_,Result} = if_auto_skip(RetVal, fun() -> {US,AS+1} end, fun() -> {US+1,AS} end), Result. -get_info_str(Func, 0, _Cases) -> - atom_to_list(Func); -get_info_str(_Func, CaseNum, unknown) -> +get_info_str(Mod,Func, 0, _Cases) -> + io_lib:format("~w", [{Mod,Func}]); +get_info_str(_Mod,_Func, CaseNum, unknown) -> "test case " ++ integer_to_list(CaseNum); -get_info_str(_Func, CaseNum, Cases) -> +get_info_str(_Mod,_Func, CaseNum, Cases) -> "test case " ++ integer_to_list(CaseNum) ++ " of " ++ integer_to_list(Cases). -- cgit v1.2.3 From 759a1f6240117cf64eecac26c21f82de0e877332 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Thu, 31 Oct 2013 16:49:59 +0100 Subject: Fix problem with handling Config and FW reports correctly --- lib/test_server/src/test_server_ctrl.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/test_server/src/test_server_ctrl.erl') diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index 9ab51df028..7b7cd24bff 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -2662,7 +2662,7 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, stop_minor_log_file(), run_test_cases_loop(skip_cases_upto(Ref, Cases, Reason, conf, CurrMode, skip_case), - Config, TimetrapData, Mode, + [hd(Config)|Config], TimetrapData, Mode, delete_status(Ref, Status2)); {_,{skip_and_save,Reason,_SavedConfig},_} when StartConf -> ReportAbortRepeat(skipped), @@ -2672,7 +2672,7 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, stop_minor_log_file(), run_test_cases_loop(skip_cases_upto(Ref, Cases, Reason, conf, CurrMode, skip_case), - Config, TimetrapData, Mode, + [hd(Config)|Config], TimetrapData, Mode, delete_status(Ref, Status2)); {_,_Other,_} when Func == init_per_suite -> print(minor, "~n*** init_per_suite failed to return a Config list.~n", []), -- cgit v1.2.3 From 9b5bd44fe00a7d1de3d7042950966133b26e830c Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Fri, 1 Nov 2013 01:44:20 +0100 Subject: Change status from skip to auto_skip for config func that fails due to require --- lib/test_server/src/test_server_ctrl.erl | 4 ---- 1 file changed, 4 deletions(-) (limited to 'lib/test_server/src/test_server_ctrl.erl') diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index 7b7cd24bff..ac09e25579 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -3970,10 +3970,6 @@ if_auto_skip(Reason={failed,{_,init_per_testcase,_}}, True, _False) -> {Reason,True()}; if_auto_skip({_T,{skip,Reason={failed,{_,init_per_testcase,_}}},_Opts}, True, _False) -> {Reason,True()}; -if_auto_skip({fw_auto_skip,Reason}, True, _False) -> - {Reason,True()}; -if_auto_skip({_T,{skip,{fw_auto_skip,Reason}},_Opts}, True, _False) -> - {Reason,True()}; if_auto_skip(Reason, _True, False) -> {Reason,False()}. -- cgit v1.2.3 From 02a0023b265bba5518b60b524d64fa8056fee911 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Wed, 6 Nov 2013 16:06:35 +0100 Subject: Correct various bugs related to auto_skip and groups --- lib/test_server/src/test_server_ctrl.erl | 55 +++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 12 deletions(-) (limited to 'lib/test_server/src/test_server_ctrl.erl') diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index ac09e25579..bef967ff12 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -1918,7 +1918,8 @@ add_init_and_end_per_suite([{conf,Ref,Props,{FwMod,Func}}=Case|Cases], LastMod, Suite when Suite =/= undefined, Suite =/= LastMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Suite, FwMod), - Case1 = {conf,Ref,proplists:delete(suite,Props),{FwMod,Func}}, + Case1 = {conf,Ref,[{suite,NextMod}|proplists:delete(suite,Props)], + {FwMod,Func}}, PreCases ++ [Case1|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; _ -> @@ -2654,6 +2655,27 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, stop_minor_log_file(), run_test_cases_loop(Cases2, Config1, TimetrapData, Mode, Status3); + {_,{auto_skip,SkipReason},_} -> + %% this case can only happen if the framework (not the user) + %% decides to skip execution of a conf function + {Cases2,Config1,Status3} = + if StartConf -> + ReportAbortRepeat(auto_skipped), + print(minor, "~n*** ~w auto skipped.~n" + " Skipping all cases.", [Func]), + {skip_cases_upto(Ref, Cases, SkipReason, conf, CurrMode, + auto_skip_case), + Config, + delete_status(Ref, Status2)}; + not StartConf -> + ReportRepeatStop(), + print_conf_time(ConfTime), + {Cases,tl(Config),delete_status(Ref, Status2)} + end, + set_io_buffering(IOHandler), + stop_minor_log_file(), + run_test_cases_loop(Cases2, Config1, TimetrapData, Mode, Status3); + {_,{Skip,Reason},_} when StartConf and ((Skip==skip) or (Skip==skipped)) -> ReportAbortRepeat(skipped), print(minor, "~n*** ~w skipped.~n" @@ -3615,7 +3637,8 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, {died,Reason} -> progress(failed, Num, Mod, Func, Loc, Reason, Time, Comment, Style); - {_,{'EXIT',{Skip,Reason}}} when Skip==skip; Skip==skipped -> + {_,{'EXIT',{Skip,Reason}}} when Skip==skip; Skip==skipped; + Skip==auto_skip -> progress(skip, Num, Mod, Func, Loc, Reason, Time, Comment, Style); {_,{'EXIT',_Pid,{Skip,Reason}}} when Skip==skip; Skip==skipped -> @@ -3627,10 +3650,13 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, {_,{'EXIT',Reason}} -> progress(failed, Num, Mod, Func, Loc, Reason, Time, Comment, Style); - {_, {Fail, Reason}} when Fail =:= fail; Fail =:= failed -> + {_,{Fail,Reason}} when Fail =:= fail; Fail =:= failed -> progress(failed, Num, Mod, Func, Loc, Reason, Time, Comment, Style); - {_, {Skip, Reason}} when Skip==skip; Skip==skipped -> + {_,Reason={auto_skip,_Why}} -> + progress(skip, Num, Mod, Func, Loc, Reason, + Time, Comment, Style); + {_,{Skip,Reason}} when Skip==skip; Skip==skipped -> progress(skip, Num, Mod, Func, Loc, Reason, Time, Comment, Style); {Time,RetVal} -> @@ -3747,15 +3773,15 @@ num2str(N) -> integer_to_list(N). progress(skip, CaseNum, Mod, Func, Loc, Reason, Time, Comment, {St0,St1}) -> - {Reason1,{Color,Ret}} = + {Reason1,{Color,Ret,ReportTag}} = if_auto_skip(Reason, - fun() -> {?auto_skip_color,auto_skip} end, - fun() -> {?user_skip_color,skip} end), - print(major, "=result skipped", []), + fun() -> {?auto_skip_color,auto_skip,auto_skipped} end, + fun() -> {?user_skip_color,skip,skipped} end), + print(major, "=result ~w: ~p", [ReportTag,Reason1]), print(1, "*** SKIPPED ~ts ***", [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), test_server_sup:framework_call(report, [tc_done,{Mod,Func, - {skipped,Reason1}}]), + {ReportTag,Reason1}}]), ReasonStr = reason_to_string(Reason1), ReasonStr1 = lists:flatten([string:strip(S,left) || S <- string:tokens(ReasonStr,[$\n])]), @@ -3968,13 +3994,18 @@ fw_name(Mod) -> if_auto_skip(Reason={failed,{_,init_per_testcase,_}}, True, _False) -> {Reason,True()}; -if_auto_skip({_T,{skip,Reason={failed,{_,init_per_testcase,_}}},_Opts}, True, _False) -> +if_auto_skip({skip,Reason={failed,{_,init_per_testcase,_}}}, True, _False) -> + {Reason,True()}; +if_auto_skip({auto_skip,Reason}, True, _False) -> {Reason,True()}; if_auto_skip(Reason, _True, False) -> {Reason,False()}. -update_skip_counters(RetVal, {US,AS}) -> - {_,Result} = if_auto_skip(RetVal, fun() -> {US,AS+1} end, fun() -> {US+1,AS} end), +update_skip_counters({_T,Pat,_Opts}, {US,AS}) -> + {_,Result} = if_auto_skip(Pat, fun() -> {US,AS+1} end, fun() -> {US+1,AS} end), + Result; +update_skip_counters(Pat, {US,AS}) -> + {_,Result} = if_auto_skip(Pat, fun() -> {US,AS+1} end, fun() -> {US+1,AS} end), Result. get_info_str(Mod,Func, 0, _Cases) -> -- cgit v1.2.3 From f34f567125c86a9f12bff473e7a7e2fd4b9a0b3f Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Thu, 7 Nov 2013 00:50:37 +0100 Subject: Correct tests cases that fail because of modified events --- lib/test_server/src/test_server_ctrl.erl | 55 +++++++++++++------------------- 1 file changed, 22 insertions(+), 33 deletions(-) (limited to 'lib/test_server/src/test_server_ctrl.erl') diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index bef967ff12..353559df03 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -2210,13 +2210,19 @@ run_test_cases(TestSpec, Config, TimetrapData) -> %% group1_end | ---> %% -run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], - Config, TimetrapData, Mode, Status) when Type==conf; - Type==make -> +run_test_cases_loop([{SkipTag,{Type,Ref,Case,Comment},SkipMode}|Cases], + Config, TimetrapData, Mode, Status) when + ((SkipTag==auto_skip_case) or (SkipTag==skip_case)) and + ((Type==conf) or (Type==make)) -> file:set_cwd(filename:dirname(get(test_server_dir))), CurrIOHandler = get(test_server_common_io_handler), ParentMode = tl(Mode), + {AutoOrUser,ReportTag} = + if SkipTag == auto_skip_case -> {auto,tc_auto_skip}; + SkipTag == skip_case -> {user,tc_user_skip} + end, + %% check and update the mode for test case execution and io msg handling case {curr_ref(Mode),check_props(parallel, Mode)} of {Ref,Ref} -> @@ -2226,9 +2232,9 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], %% group, buffered io can be flushed handle_test_case_io_and_status(), set_io_buffering(undefined), - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, + {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, false, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip, + test_server_sup:framework_call(report, [ReportTag, {Mod,Func,Comment}]), run_test_cases_loop(Cases, Config, TimetrapData, ParentMode, delete_status(Ref, Status)); @@ -2236,9 +2242,9 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], %% this is a skipped end conf for a parallel group nested %% under a parallel group (io buffering is active) wait_for_cases(Ref), - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, + {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, true, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip, + test_server_sup:framework_call(report, [ReportTag, {Mod,Func,Comment}]), case CurrIOHandler of {Ref,_} -> @@ -2256,9 +2262,9 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], {Ref,false} -> %% this is a skipped end conf for a non-parallel group that's not %% nested under a parallel group - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, + {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, false, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip, + test_server_sup:framework_call(report, [ReportTag, {Mod,Func,Comment}]), %% Check if this group is auto skipped because of error in the @@ -2277,7 +2283,7 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], Reason = {group_result,GrName,failed}, skip_cases_upto(ParentRef, Cases, Reason, tc, Mode, - auto_skip_case) + SkipTag) end; false -> Cases @@ -2290,9 +2296,9 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], {Ref,_} -> %% this is a skipped end conf for a non-parallel group nested under %% a parallel group (io buffering is active) - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, + {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, true, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip, + test_server_sup:framework_call(report, [ReportTag, {Mod,Func,Comment}]), case CurrIOHandler of {Ref,_} -> @@ -2308,9 +2314,9 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], {_,false} -> %% this is a skipped start conf for a group which is not nested %% under a parallel group - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, + {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, false, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip, + test_server_sup:framework_call(report, [ReportTag, {Mod,Func,Comment}]), run_test_cases_loop(Cases, Config, TimetrapData, [conf(Ref,[])|Mode], Status); @@ -2323,9 +2329,9 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], true -> ok end, - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, + {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, true, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip, + test_server_sup:framework_call(report, [ReportTag, {Mod,Func,Comment}]), run_test_cases_loop(Cases, Config, TimetrapData, [conf(Ref,[])|Mode], Status) @@ -2339,23 +2345,6 @@ run_test_cases_loop([{auto_skip_case,{Case,Comment},SkipMode}|Cases], run_test_cases_loop(Cases, Config, TimetrapData, Mode, update_status(skipped, Mod, Func, Status)); -run_test_cases_loop([{skip_case,{conf,Ref,Case,Comment}}|Cases0], - Config, TimetrapData, Mode, Status) -> - {Mod,Func} = skip_case(user, Ref, 0, Case, Comment, is_io_buffered()), - {Cases,Config1} = - case curr_ref(Mode) of - Ref -> - %% skipped end conf - {Cases0,tl(Config)}; - _ -> - %% skipped start conf - {skip_cases_upto(Ref, Cases0, Comment, conf, Mode, skip_case), - Config} - end, - test_server_sup:framework_call(report, [tc_user_skip,{Mod,Func,Comment}]), - run_test_cases_loop(Cases, Config1, TimetrapData, Mode, - update_status(skipped, Mod, Func, Status)); - run_test_cases_loop([{skip_case,{Case,Comment}}|Cases], Config, TimetrapData, Mode, Status) -> {Mod,Func} = skip_case(user, undefined, get(test_server_case_num)+1, -- cgit v1.2.3 From 16f45bc71e6a9cec351ca562a7a1e77569d4cdcf Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Thu, 7 Nov 2013 22:13:54 +0100 Subject: Change report tag for failed init_per_testcase from skipped to auto_skipped --- lib/test_server/src/test_server_ctrl.erl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'lib/test_server/src/test_server_ctrl.erl') diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index 353559df03..da8deccb45 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -3049,13 +3049,13 @@ cases_to_shuffle(Ref, Cases) -> cases_to_shuffle(Ref, [{conf,Ref,_,_} | _]=Cs, N, Ix) -> % end {N-1,Ix,Cs}; -cases_to_shuffle(Ref, [{skip_case,{_,Ref,_,_}} | _]=Cs, N, Ix) -> % end +cases_to_shuffle(Ref, [{skip_case,{_,Ref,_,_},_} | _]=Cs, N, Ix) -> % end {N-1,Ix,Cs}; cases_to_shuffle(Ref, [{conf,Ref1,_,_}=C | Cs], N, Ix) -> % nested group {Cs1,Rest} = get_subcases(Ref1, Cs, []), cases_to_shuffle(Ref, Rest, N+1, [{N,[C|Cs1]} | Ix]); -cases_to_shuffle(Ref, [{skip_case,{_,Ref1,_,_}}=C | Cs], N, Ix) -> % nested group +cases_to_shuffle(Ref, [{skip_case,{_,Ref1,_,_},_}=C | Cs], N, Ix) -> % nested group {Cs1,Rest} = get_subcases(Ref1, Cs, []), cases_to_shuffle(Ref, Rest, N+1, [{N,[C|Cs1]} | Ix]); @@ -3064,7 +3064,7 @@ cases_to_shuffle(Ref, [C | Cs], N, Ix) -> get_subcases(SubRef, [{conf,SubRef,_,_}=C | Cs], SubCs) -> {lists:reverse([C|SubCs]),Cs}; -get_subcases(SubRef, [{skip_case,{_,SubRef,_,_}}=C | Cs], SubCs) -> +get_subcases(SubRef, [{skip_case,{_,SubRef,_,_},_}=C | Cs], SubCs) -> {lists:reverse([C|SubCs]),Cs}; get_subcases(SubRef, [C|Cs], SubCs) -> get_subcases(SubRef, Cs, [C|SubCs]). @@ -3198,7 +3198,7 @@ modify_cases_upto(Ref, ModOp, Cases, Orig, Alt) -> %% same ref in the list, if not, this *is* an end conf case case lists:any(fun({_,R,_,_}) when R == Ref -> true; ({_,R,_}) when R == Ref -> true; - ({skip_case,{_,R,_,_}}) when R == Ref -> true; + ({skip_case,{_,R,_,_},_}) when R == Ref -> true; (_) -> false end, Cases) of true -> @@ -3208,9 +3208,9 @@ modify_cases_upto(Ref, ModOp, Cases, Orig, Alt) -> end. %% next case is a conf with same ref, must be end conf = we're done -modify_cases_upto1(Ref, {skip,Reason,conf,_,skip_case}, +modify_cases_upto1(Ref, {skip,Reason,conf,Mode,skip_case}, [{conf,Ref,_Props,MF}|T], Orig, Alt) -> - {Orig,[{skip_case,{conf,Ref,MF,Reason}}|Alt],T}; + {Orig,[{skip_case,{conf,Ref,MF,Reason},Mode}|Alt],T}; modify_cases_upto1(Ref, {skip,Reason,conf,Mode,auto_skip_case}, [{conf,Ref,_Props,MF}|T], Orig, Alt) -> {Orig,[{auto_skip_case,{conf,Ref,MF,Reason},Mode}|Alt],T}; @@ -3231,7 +3231,7 @@ modify_cases_upto1(Ref, {copy,NewRef}, [{make,Ref,MF}=M|T], Orig, Alt) -> %% next case is a user skipped end conf with the same ref = we're done modify_cases_upto1(Ref, {skip,Reason,_,Mode,SkipType}, - [{skip_case,{Type,Ref,MF,_Cmt}}|T], Orig, Alt) -> + [{skip_case,{Type,Ref,MF,_Cmt},_}|T], Orig, Alt) -> {Orig,[{SkipType,{Type,Ref,MF,Reason},Mode}|Alt],T}; modify_cases_upto1(Ref, {copy,NewRef}, [{skip_case,{Type,Ref,MF,Cmt}}=C|T], Orig, Alt) -> -- cgit v1.2.3 From a522754dde5071ca131f3463a57f5ca4ecfedd6b Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Fri, 8 Nov 2013 16:55:39 +0100 Subject: Fix problem with suites and groups skipped from test specification --- lib/test_server/src/test_server_ctrl.erl | 79 +++++++++++++++++++++++++------- 1 file changed, 63 insertions(+), 16 deletions(-) (limited to 'lib/test_server/src/test_server_ctrl.erl') diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index da8deccb45..2041ee294e 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -1480,6 +1480,11 @@ do_test_cases(TopCases, SkipCases, put(test_server_case_num, 0), TestSpec = add_init_and_end_per_suite(TestSpec0, undefined, undefined, FwMod), + + + %%! --- Fri Nov 8 14:50:39 2013 --- peppe was here! + io:format(user, ">>> HERE'S THE TS: ~p~n~n", [TestSpec]), + TI = get_target_info(), print(1, "Starting test~ts", [print_if_known(N, {", ~w test cases",[N]}, @@ -1795,23 +1800,40 @@ downcase([], Result) -> %% %% Errors are silently ignored. -html_convert_modules(TestSpec, _Config) -> - Mods = html_isolate_modules(TestSpec), +html_convert_modules(TestSpec, _Config, FwMod) -> + Mods = html_isolate_modules(TestSpec, FwMod), html_convert_modules(Mods), copy_html_files(get(test_server_dir), get(test_server_log_dir_base)). %% Retrieve a list of modules out of the test spec. -html_isolate_modules(List) -> html_isolate_modules(List, sets:new()). - -html_isolate_modules([], Set) -> sets:to_list(Set); -html_isolate_modules([{skip_case,_}|Cases], Set) -> - html_isolate_modules(Cases, Set); -html_isolate_modules([{conf,_Ref,_Props,{Mod,_Func}}|Cases], Set) -> - html_isolate_modules(Cases, sets:add_element(Mod, Set)); -html_isolate_modules([{Mod,_Case}|Cases], Set) -> - html_isolate_modules(Cases, sets:add_element(Mod, Set)); -html_isolate_modules([{Mod,_Case,_Args}|Cases], Set) -> - html_isolate_modules(Cases, sets:add_element(Mod, Set)). +html_isolate_modules(List, FwMod) -> + html_isolate_modules(List, sets:new(), FwMod). + +html_isolate_modules([], Set, _) -> sets:to_list(Set); +html_isolate_modules([{skip_case,_}|Cases], Set, FwMod) -> + html_isolate_modules(Cases, Set, FwMod); +html_isolate_modules([{conf,_Ref,Props,{FwMod,_Func}}|Cases], Set, FwMod) -> + Set1 = case proplists:get_value(suite, Props) of + undefined -> Set; + Mod -> sets:add_element(Mod, Set) + end, + html_isolate_modules(Cases, Set1, FwMod); +html_isolate_modules([{conf,_Ref,_Props,{Mod,_Func}}|Cases], Set, FwMod) -> + html_isolate_modules(Cases, sets:add_element(Mod, Set), FwMod); +html_isolate_modules([{skip_case,{conf,_Ref,{FwMod,_Func},_Cmt},Mode}|Cases], + Set, FwMod) -> + Set1 = case proplists:get_value(suite, get_props(Mode)) of + undefined -> Set; + Mod -> sets:add_element(Mod, Set) + end, + html_isolate_modules(Cases, Set1, FwMod); +html_isolate_modules([{skip_case,{conf,_Ref,{Mod,_Func},_Cmt},_Props}|Cases], + Set, FwMod) -> + html_isolate_modules(Cases, sets:add_element(Mod, Set), FwMod); +html_isolate_modules([{Mod,_Case}|Cases], Set, FwMod) -> + html_isolate_modules(Cases, sets:add_element(Mod, Set), FwMod); +html_isolate_modules([{Mod,_Case,_Args}|Cases], Set, FwMod) -> + html_isolate_modules(Cases, sets:add_element(Mod, Set), FwMod). %% Given a list of modules, convert each module's source code to HTML. html_convert_modules([Mod|Mods]) -> @@ -1902,6 +1924,11 @@ add_init_and_end_per_suite([{skip_case,{{Mod,_},_}}=Case|Cases], LastMod, {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod), PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; +add_init_and_end_per_suite([{skip_case,{conf,_,{Mod,_},_},_}=Case|Cases], LastMod, + LastRef, FwMod) when Mod =/= LastMod -> + {PreCases, NextMod, NextRef} = + do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod), + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; add_init_and_end_per_suite([{skip_case,{conf,_,{Mod,_},_}}=Case|Cases], LastMod, LastRef, FwMod) when Mod =/= LastMod -> {PreCases, NextMod, NextRef} = @@ -2044,7 +2071,8 @@ run_test_cases(TestSpec, Config, TimetrapData) -> true -> ok; false -> - html_convert_modules(TestSpec, Config) + FwMod = get_fw_mod(?MODULE), + html_convert_modules(TestSpec, Config, FwMod) end, run_test_cases_loop(TestSpec, [Config], TimetrapData, [], []), @@ -4460,12 +4488,31 @@ collect_cases({conf,Props,InitMF,CaseList,FinMF} = Conf, St) -> Props1 -> Ref = make_ref(), Skips = St#cc.skip, + Props2 = [{suite,St#cc.mod} | lists:delete(suite,Props1)], + Mode = [{Ref,Props2,undefined}], case in_skip_list({St#cc.mod,Conf}, Skips) of {true,Comment} -> % conf init skipped - {ok,[{skip_case,{conf,Ref,InitMF,Comment}} | + {ok,[{skip_case,{conf,Ref,InitMF,Comment},Mode} | [] ++ [{conf,Ref,[],FinMF}]],St}; {true,Name,Comment} when is_atom(Name) -> % all cases skipped - {ok,[{skip_case,{{St#cc.mod,{group,Name}},Comment}}],St}; + case collect_cases(CaseList, St) of + {ok,[],_St} = Empty -> + Empty; + {ok,FlatCases,St1} -> + Cases2Skip = FlatCases ++ [{conf,Ref, + keep_name(Props1), + FinMF}], + Skipped = skip_cases_upto(Ref, Cases2Skip, Comment, + conf, Mode, skip_case), + + %%! --- Fri Nov 8 16:47:50 2013 --- peppe was here! + io:format(user, "!!! FLAT2 = ~p~n", [Skipped]), + + {ok,[{skip_case,{conf,Ref,InitMF,Comment},Mode} | + Skipped],St1}; + {error,_Reason} = Error -> + Error + end; {true,ToSkip,_} when is_list(ToSkip) -> % some cases skipped case collect_cases(CaseList, St#cc{skip=ToSkip++Skips}) of -- cgit v1.2.3 From af1891a1415d9aedb7de866639cf997c31b98e35 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Tue, 12 Nov 2013 00:48:24 +0100 Subject: Add test cases for new and modified functionality --- lib/test_server/src/test_server_ctrl.erl | 46 +++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 10 deletions(-) (limited to 'lib/test_server/src/test_server_ctrl.erl') diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index 2041ee294e..810591b16f 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -1402,10 +1402,16 @@ remove_conf([{conf, _Ref, Props, _MF}|Cases], NoConf, Repeats) -> end; remove_conf([{make,_Ref,_MF}|Cases], NoConf, Repeats) -> remove_conf(Cases, NoConf, Repeats); +remove_conf([{skip_case,{{_M,all},_Cmt}}|Cases], NoConf, Repeats) -> + remove_conf(Cases, NoConf, Repeats); remove_conf([{skip_case,{Type,_Ref,_MF,_Cmt}}|Cases], NoConf, Repeats) when Type==conf; Type==make -> remove_conf(Cases, NoConf, Repeats); +remove_conf([{skip_case,{Type,_Ref,_MF,_Cmt},_Mode}|Cases], + NoConf, Repeats) when Type==conf; + Type==make -> + remove_conf(Cases, NoConf, Repeats); remove_conf([C|Cases], NoConf, Repeats) -> remove_conf(Cases, [C|NoConf], Repeats); remove_conf([], NoConf, true) -> @@ -1413,6 +1419,11 @@ remove_conf([], NoConf, true) -> remove_conf([], NoConf, false) -> lists:reverse(NoConf). +get_suites([{skip_case,{{Mod,_Func},_Cmt}}|Tests], Mods) when is_atom(Mod) -> + case add_mod(Mod, Mods) of + true -> get_suites(Tests, [Mod|Mods]); + false -> get_suites(Tests, Mods) + end; get_suites([{Mod,_Case}|Tests], Mods) when is_atom(Mod) -> case add_mod(Mod, Mods) of true -> get_suites(Tests, [Mod|Mods]); @@ -1478,13 +1489,10 @@ do_test_cases(TopCases, SkipCases, end, put(test_server_cases, N), put(test_server_case_num, 0), + TestSpec = add_init_and_end_per_suite(TestSpec0, undefined, undefined, FwMod), - - %%! --- Fri Nov 8 14:50:39 2013 --- peppe was here! - io:format(user, ">>> HERE'S THE TS: ~p~n~n", [TestSpec]), - TI = get_target_info(), print(1, "Starting test~ts", [print_if_known(N, {", ~w test cases",[N]}, @@ -1934,8 +1942,6 @@ add_init_and_end_per_suite([{skip_case,{conf,_,{Mod,_},_}}=Case|Cases], LastMod, {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod), PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; -add_init_and_end_per_suite([{skip_case,_}=Case|Cases], LastMod, LastRef, FwMod) -> - [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; add_init_and_end_per_suite([{conf,Ref,Props,{FwMod,Func}}=Case|Cases], LastMod, LastRef, FwMod) -> %% if Mod == FwMod, this conf test is (probably) a test case group where @@ -1957,6 +1963,9 @@ add_init_and_end_per_suite([{conf,_,_,{Mod,_}}=Case|Cases], LastMod, {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod), PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; +add_init_and_end_per_suite([SkipCase|Cases], LastMod, LastRef, FwMod) + when element(1,SkipCase) == skip_case -> + [SkipCase|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; add_init_and_end_per_suite([{conf,_,_,_}=Case|Cases], LastMod, LastRef, FwMod) -> [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; add_init_and_end_per_suite([{Mod,_}=Case|Cases], LastMod, LastRef, FwMod) @@ -2238,6 +2247,13 @@ run_test_cases(TestSpec, Config, TimetrapData) -> %% group1_end | ---> %% +run_test_cases_loop([{SkipTag,CaseData={Type,_Ref,_Case,_Comment}}|Cases], + Config, TimetrapData, Mode, Status) when + ((SkipTag==auto_skip_case) or (SkipTag==skip_case)) and + ((Type==conf) or (Type==make)) -> + run_test_cases_loop([{SkipTag,CaseData,Mode}|Cases], + Config, TimetrapData, Mode, Status); + run_test_cases_loop([{SkipTag,{Type,Ref,Case,Comment},SkipMode}|Cases], Config, TimetrapData, Mode, Status) when ((SkipTag==auto_skip_case) or (SkipTag==skip_case)) and @@ -2373,6 +2389,13 @@ run_test_cases_loop([{auto_skip_case,{Case,Comment},SkipMode}|Cases], run_test_cases_loop(Cases, Config, TimetrapData, Mode, update_status(skipped, Mod, Func, Status)); +run_test_cases_loop([{skip_case,{{Mod,all}=Case,Comment}}|Cases], + Config, TimetrapData, Mode, Status) -> + skip_case(user, undefined, 0, Case, Comment, false, Mode), + test_server_sup:framework_call(report, [tc_user_skip, + {Mod,all,Comment}]), + run_test_cases_loop(Cases, Config, TimetrapData, Mode, Status); + run_test_cases_loop([{skip_case,{Case,Comment}}|Cases], Config, TimetrapData, Mode, Status) -> {Mod,Func} = skip_case(user, undefined, get(test_server_case_num)+1, @@ -3227,6 +3250,7 @@ modify_cases_upto(Ref, ModOp, Cases, Orig, Alt) -> case lists:any(fun({_,R,_,_}) when R == Ref -> true; ({_,R,_}) when R == Ref -> true; ({skip_case,{_,R,_,_},_}) when R == Ref -> true; + ({skip_case,{_,R,_,_}}) when R == Ref -> true; (_) -> false end, Cases) of true -> @@ -3261,6 +3285,12 @@ modify_cases_upto1(Ref, {copy,NewRef}, [{make,Ref,MF}=M|T], Orig, Alt) -> modify_cases_upto1(Ref, {skip,Reason,_,Mode,SkipType}, [{skip_case,{Type,Ref,MF,_Cmt},_}|T], Orig, Alt) -> {Orig,[{SkipType,{Type,Ref,MF,Reason},Mode}|Alt],T}; +modify_cases_upto1(Ref, {skip,Reason,_,Mode,SkipType}, + [{skip_case,{Type,Ref,MF,_Cmt}}|T], Orig, Alt) -> + {Orig,[{SkipType,{Type,Ref,MF,Reason},Mode}|Alt],T}; +modify_cases_upto1(Ref, {copy,NewRef}, + [{skip_case,{Type,Ref,MF,Cmt},Mode}=C|T], Orig, Alt) -> + {[C|Orig],[{skip_case,{Type,NewRef,MF,Cmt},Mode}|Alt],T}; modify_cases_upto1(Ref, {copy,NewRef}, [{skip_case,{Type,Ref,MF,Cmt}}=C|T], Orig, Alt) -> {[C|Orig],[{skip_case,{Type,NewRef,MF,Cmt}}|Alt],T}; @@ -4504,10 +4534,6 @@ collect_cases({conf,Props,InitMF,CaseList,FinMF} = Conf, St) -> FinMF}], Skipped = skip_cases_upto(Ref, Cases2Skip, Comment, conf, Mode, skip_case), - - %%! --- Fri Nov 8 16:47:50 2013 --- peppe was here! - io:format(user, "!!! FLAT2 = ~p~n", [Skipped]), - {ok,[{skip_case,{conf,Ref,InitMF,Comment},Mode} | Skipped],St1}; {error,_Reason} = Error -> -- cgit v1.2.3 From 98c0e6608100da393df24722afea159a1f5dcc22 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Thu, 14 Nov 2013 00:19:03 +0100 Subject: Fix problems with info functions and add more tests --- lib/test_server/src/test_server_ctrl.erl | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) (limited to 'lib/test_server/src/test_server_ctrl.erl') diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index 810591b16f..97e6a171db 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -1161,8 +1161,11 @@ init_tester(Mod, Func, Args, Dir, Name, {_,_,MinLev}=Levels, end, {SkippedN,SkipStr} = case get(test_server_skipped) of - {0,_} -> {0,""}; - {Skipped,_} -> {Skipped,io_lib:format(", ~w Skipped", [Skipped])} + {0,0} -> + {0,""}; + {USkipped,ASkipped} -> + Skipped = USkipped+ASkipped, + {Skipped,io_lib:format(", ~w Skipped", [Skipped])} end, OkN = get(test_server_ok), FailedN = get(test_server_failed), @@ -1412,6 +1415,13 @@ remove_conf([{skip_case,{Type,_Ref,_MF,_Cmt},_Mode}|Cases], NoConf, Repeats) when Type==conf; Type==make -> remove_conf(Cases, NoConf, Repeats); +remove_conf([C={Mod,error_in_suite,_}|Cases], NoConf, Repeats) -> + FwMod = get_fw_mod(?MODULE), + if Mod == FwMod -> + remove_conf(Cases, NoConf, Repeats); + true -> + remove_conf(Cases, [C|NoConf], Repeats) + end; remove_conf([C|Cases], NoConf, Repeats) -> remove_conf(Cases, [C|NoConf], Repeats); remove_conf([], NoConf, true) -> @@ -2822,7 +2832,14 @@ run_test_cases_loop([{Mod,Case}|Cases], Config, TimetrapData, Mode, Status) -> TimetrapData, Mode, Status); run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status) -> - Num = put(test_server_case_num, get(test_server_case_num)+1), + {Num,RunInit} = + case FwMod = get_fw_mod(?MODULE) of + Mod when Func == error_in_suite -> + {-1,skip_init}; + _ -> + {put(test_server_case_num, get(test_server_case_num)+1), + run_init} + end, %% check the current execution mode and save info about the case if %% detected that printouts to common log files is handled later @@ -2837,7 +2854,7 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status) end, case run_test_case(undefined, Num+1, Mod, Func, Args, - run_init, TimetrapData, Mode) of + RunInit, TimetrapData, Mode) of %% callback to framework module failed, exit immediately {_,{framework_error,{FwMod,FwFunc},Reason},_} -> print(minor, "~n*** ~w failed in ~w. Reason: ~p~n", @@ -3193,6 +3210,7 @@ skip_case1(Type, CaseNum, Mod, Func, Comment, Mode) -> "SKIPPED" "~ts\n", [num2str(CaseNum),fw_name(Mod),GroupName,Func,ResultCol,Comment1]), + if CaseNum > 0 -> {US,AS} = get(test_server_skipped), case Type of -- cgit v1.2.3 From 093890dc793e85a09b40f1eca878f410c73cf625 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Thu, 14 Nov 2013 17:39:23 +0100 Subject: Modify the auto_skip report for group config funcs to include group name Also correct failing test cases and find and fix remaining bugs. --- lib/test_server/src/test_server_ctrl.erl | 37 +++++++++++++++++++------------- 1 file changed, 22 insertions(+), 15 deletions(-) (limited to 'lib/test_server/src/test_server_ctrl.erl') diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index 97e6a171db..352e58f91c 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -2288,8 +2288,9 @@ run_test_cases_loop([{SkipTag,{Type,Ref,Case,Comment},SkipMode}|Cases], set_io_buffering(undefined), {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, false, SkipMode), - test_server_sup:framework_call(report, [ReportTag, - {Mod,Func,Comment}]), + ConfData = {Mod,{Func,get_name(SkipMode)},Comment}, + test_server_sup:framework_call(report, + [ReportTag,ConfData]), run_test_cases_loop(Cases, Config, TimetrapData, ParentMode, delete_status(Ref, Status)); _ -> @@ -2298,8 +2299,8 @@ run_test_cases_loop([{SkipTag,{Type,Ref,Case,Comment},SkipMode}|Cases], wait_for_cases(Ref), {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, true, SkipMode), - test_server_sup:framework_call(report, [ReportTag, - {Mod,Func,Comment}]), + ConfData = {Mod,{Func,get_name(SkipMode)},Comment}, + test_server_sup:framework_call(report, [ReportTag,ConfData]), case CurrIOHandler of {Ref,_} -> %% current_io_handler was set by start conf of this @@ -2318,8 +2319,8 @@ run_test_cases_loop([{SkipTag,{Type,Ref,Case,Comment},SkipMode}|Cases], %% nested under a parallel group {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, false, SkipMode), - test_server_sup:framework_call(report, [ReportTag, - {Mod,Func,Comment}]), + ConfData = {Mod,{Func,get_name(SkipMode)},Comment}, + test_server_sup:framework_call(report, [ReportTag,ConfData]), %% Check if this group is auto skipped because of error in the %% init conf. If so, check if the parent group is a sequence, @@ -2352,8 +2353,8 @@ run_test_cases_loop([{SkipTag,{Type,Ref,Case,Comment},SkipMode}|Cases], %% a parallel group (io buffering is active) {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, true, SkipMode), - test_server_sup:framework_call(report, [ReportTag, - {Mod,Func,Comment}]), + ConfData = {Mod,{Func,get_name(SkipMode)},Comment}, + test_server_sup:framework_call(report, [ReportTag,ConfData]), case CurrIOHandler of {Ref,_} -> %% current_io_handler was set by start conf of this @@ -2370,8 +2371,8 @@ run_test_cases_loop([{SkipTag,{Type,Ref,Case,Comment},SkipMode}|Cases], %% under a parallel group {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, false, SkipMode), - test_server_sup:framework_call(report, [ReportTag, - {Mod,Func,Comment}]), + ConfData = {Mod,{Func,get_name(SkipMode)},Comment}, + test_server_sup:framework_call(report, [ReportTag,ConfData]), run_test_cases_loop(Cases, Config, TimetrapData, [conf(Ref,[])|Mode], Status); {_,Ref0} when is_reference(Ref0) -> @@ -2385,8 +2386,8 @@ run_test_cases_loop([{SkipTag,{Type,Ref,Case,Comment},SkipMode}|Cases], end, {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, true, SkipMode), - test_server_sup:framework_call(report, [ReportTag, - {Mod,Func,Comment}]), + ConfData = {Mod,{Func,get_name(SkipMode)},Comment}, + test_server_sup:framework_call(report, [ReportTag,ConfData]), run_test_cases_loop(Cases, Config, TimetrapData, [conf(Ref,[])|Mode], Status) end; @@ -3180,11 +3181,17 @@ skip_case1(Type, CaseNum, Mod, Func, Comment, Mode) -> ResultCol = if Type == auto -> ?auto_skip_color; Type == user -> ?user_skip_color end, - - Comment1 = reason_to_string(Comment), - print(major, "~n=case ~w:~w", [Mod,Func]), + GroupName = case get_name(Mode) of + undefined -> + ""; + GrName -> + GrName1 = cast_to_list(GrName), + print(major, "=group_props ~p", [[{name,GrName1}]]), + GrName1 + end, print(major, "=started ~s", [lists:flatten(timestamp_get(""))]), + Comment1 = reason_to_string(Comment), if Type == auto -> print(major, "=result auto_skipped: ~ts", [Comment1]); Type == user -> -- cgit v1.2.3