diff options
-rw-r--r-- | lib/common_test/src/ct_framework.erl | 91 | ||||
-rw-r--r-- | lib/common_test/src/ct_hooks.erl | 8 | ||||
-rw-r--r-- | lib/common_test/test/ct_error_SUITE.erl | 26 | ||||
-rw-r--r-- | lib/common_test/test/ct_error_SUITE_data/error/test/lib_error_1_SUITE.erl | 21 | ||||
-rw-r--r-- | lib/common_test/test/ct_hooks_SUITE.erl | 2 | ||||
-rw-r--r-- | lib/test_server/src/test_server.erl | 111 |
6 files changed, 167 insertions, 92 deletions
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index ae3f8a69e3..4fcd7bfa5a 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -52,15 +52,23 @@ %%% %%% @doc Test server framework callback, called by the test_server %%% when a new test case is started. -init_tc(Mod,Func0,Config) -> - {TCCfgFunc,Func} = case Func0 of - {init_per_testcase,_} -> Func0; - {end_per_testcase,_} -> Func0; - _ -> {undefined,Func0} - end, - +init_tc(Mod,EPTC={end_per_testcase,_},[Config]) -> %% in case Mod == ct_framework, lookup the suite name Suite = get_suite_name(Mod, Config), + case ct_hooks:init_tc(Suite,EPTC,Config) of + NewConfig when is_list(NewConfig) -> + {ok,[NewConfig]}; + Other-> + Other + end; + +init_tc(Mod,Func0,Args) -> + %% in case Mod == ct_framework, lookup the suite name + Suite = get_suite_name(Mod, Args), + {Func,HookFunc} = case Func0 of + {init_per_testcase,F} -> {F,Func0}; + _ -> {Func0,Func0} + end, %% check if previous testcase was interpreted and has left %% a "dead" trace window behind - if so, kill it @@ -92,7 +100,7 @@ init_tc(Mod,Func0,Config) -> end, [create]), case ct_util:read_suite_data({seq,Suite,Func}) of undefined -> - init_tc1(Mod,Suite,Func,TCCfgFunc,Config); + init_tc1(Mod,Suite,Func,HookFunc,Args); Seq when is_atom(Seq) -> case ct_util:read_suite_data({seq,Suite,Seq}) of [Func|TCs] -> % this is the 1st case in Seq @@ -108,19 +116,19 @@ init_tc(Mod,Func0,Config) -> _ -> ok end, - init_tc1(Mod,Suite,Func,TCCfgFunc,Config); + init_tc1(Mod,Suite,Func,HookFunc,Args); {failed,Seq,BadFunc} -> {auto_skip,{sequence_failed,Seq,BadFunc}} end end - end. + end. init_tc1(?MODULE,_,error_in_suite,_,[Config0]) when is_list(Config0) -> ct_logs:init_tc(false), ct_event:notify(#event{name=tc_start, node=node(), data={?MODULE,error_in_suite}}), - ct_suite_init(?MODULE,error_in_suite,undefined,[], Config0), + ct_suite_init(?MODULE,error_in_suite,[],Config0), case ?val(error,Config0) of undefined -> {fail,"unknown_error_in_suite"}; @@ -128,7 +136,7 @@ init_tc1(?MODULE,_,error_in_suite,_,[Config0]) when is_list(Config0) -> {fail,Reason} end; -init_tc1(Mod,Suite,Func,TCCfgFunc,[Config0]) when is_list(Config0) -> +init_tc1(Mod,Suite,Func,HookFunc,[Config0]) when is_list(Config0) -> Config1 = case ct_util:read_suite_data(last_saved_config) of {{Suite,LastFunc},SavedConfig} -> % last testcase @@ -162,11 +170,13 @@ init_tc1(Mod,Suite,Func,TCCfgFunc,[Config0]) when is_list(Config0) -> %% testcase info function (these should only survive the %% testcase, not the whole suite) FuncSpec = group_or_func(Func,Config0), - if is_tuple(FuncSpec) -> % group - ok; - true -> - ct_config:delete_default_config(testcase) - end, + HookFunc1 = + if is_tuple(FuncSpec) -> % group + FuncSpec; + true -> + ct_config:delete_default_config(testcase), + HookFunc + end, Initialize = fun() -> ct_logs:init_tc(false), ct_event:notify(#event{name=tc_start, @@ -190,15 +200,15 @@ init_tc1(Mod,Suite,Func,TCCfgFunc,[Config0]) when is_list(Config0) -> Initialize(), {fail,Reason}; _ -> - init_tc2(Mod,Suite,Func,TCCfgFunc,SuiteInfo, - MergeResult,Config) + init_tc2(Mod,Suite,Func,HookFunc1, + SuiteInfo,MergeResult,Config) end end; -init_tc1(_Mod,_Suite,_Func,_TCCfgFunc,Args) -> +init_tc1(_Mod,_Suite,_Func,_HookFunc,Args) -> {ok,Args}. -init_tc2(Mod,Suite,Func,TCCfgFunc,SuiteInfo,MergeResult,Config) -> +init_tc2(Mod,Suite,Func,HookFunc,SuiteInfo,MergeResult,Config) -> %% timetrap must be handled before require MergedInfo = timetrap_first(MergeResult, [], []), %% tell logger to use specified style sheet @@ -245,8 +255,7 @@ init_tc2(Mod,Suite,Func,TCCfgFunc,SuiteInfo,MergeResult,Config) -> {ok,PostInitHook,Config1} -> case get('$test_server_framework_test') of undefined -> - ct_suite_init(Suite,FuncSpec,TCCfgFunc, - PostInitHook,Config1); + ct_suite_init(Suite,HookFunc,PostInitHook,Config1); Fun -> PostInitHookResult = do_post_init_hook(PostInitHook, Config1), @@ -259,11 +268,7 @@ init_tc2(Mod,Suite,Func,TCCfgFunc,SuiteInfo,MergeResult,Config) -> end end. -ct_suite_init(Suite,FuncSpec,TCCfgFunc, - PostInitHook,Config) when is_list(Config) -> - HookFunc = if TCCfgFunc /= undefined -> {TCCfgFunc,FuncSpec}; - true -> FuncSpec - end, +ct_suite_init(Suite,HookFunc,PostInitHook,Config) when is_list(Config) -> case ct_hooks:init_tc(Suite,HookFunc,Config) of NewConfig when is_list(NewConfig) -> PostInitHookResult = do_post_init_hook(PostInitHook,NewConfig), @@ -669,14 +674,23 @@ end_tc(Mod,Func,{TCPid,Result,[Args]}, Return) when is_pid(TCPid) -> end_tc(Mod,Func,{Result,[Args]}, Return) -> end_tc(Mod,Func,self(),Result,Args,Return). +end_tc(Mod,IPTC={init_per_testcase,_Func},_TCPid,Result,Args,Return) -> + %% in case Mod == ct_framework, lookup the suite name + Suite = get_suite_name(Mod, Args), + case ct_hooks:end_tc(Suite,IPTC,Args,Result,Return) of + '$ct_no_change' -> + ok; + HookResult -> + HookResult + end; + end_tc(Mod,Func0,TCPid,Result,Args,Return) -> - {TCCfgFunc,Func} = case Func0 of - {init_per_testcase,_} -> Func0; - {end_per_testcase,_} -> Func0; - _ -> {undefined,Func0} - end, %% in case Mod == ct_framework, lookup the suite name Suite = get_suite_name(Mod, Args), + {EPTC,Func} = case Func0 of + {end_per_testcase,F} -> {true,F}; + _ -> {false,Func0} + end, test_server:timetrap_cancel(), @@ -703,10 +717,13 @@ end_tc(Mod,Func0,TCPid,Result,Args,Return) -> end, ct_util:delete_suite_data(last_saved_config), - FuncSpec = group_or_func(Func,Args), - HookFunc = if TCCfgFunc /= undefined -> Func0; - true -> FuncSpec - end, + {FuncSpec,HookFunc} = + if not EPTC -> + FS = group_or_func(Func,Args), + {FS,FS}; + true -> + {Func,Func0} + end, {Result1,FinalNotify} = case ct_hooks:end_tc(Suite,HookFunc,Args,Result,Return) of '$ct_no_change' -> diff --git a/lib/common_test/src/ct_hooks.erl b/lib/common_test/src/ct_hooks.erl index 4008ea998a..b604074d12 100644 --- a/lib/common_test/src/ct_hooks.erl +++ b/lib/common_test/src/ct_hooks.erl @@ -96,7 +96,9 @@ init_tc(_Mod, {end_per_group, GroupName, _}, Config) -> init_tc(_Mod, {init_per_testcase,TC}, Config) -> call(fun call_generic/3, Config, [pre_init_per_testcase, TC]); init_tc(_Mod, {end_per_testcase,TC}, Config) -> - call(fun call_generic/3, Config, [pre_end_per_testcase, TC]). + call(fun call_generic/3, Config, [pre_end_per_testcase, TC]); +init_tc(_Mod, TC = error_in_suite, Config) -> + call(fun call_generic/3, Config, [pre_init_per_testcase, TC]). %% @doc Called as each test case is completed. This includes all configuration %% tests. @@ -133,8 +135,12 @@ end_tc(_Mod, {init_per_testcase,TC}, Config, Result, _Return) -> '$ct_no_change'); end_tc(_Mod, {end_per_testcase,TC}, Config, Result, _Return) -> call(fun call_generic/3, Result, [post_end_per_testcase, TC, Config], + '$ct_no_change'); +end_tc(_Mod, TC = error_in_suite, Config, Result, _Return) -> + call(fun call_generic/3, Result, [post_end_per_testcase, TC, Config], '$ct_no_change'). + %% Case = TestCase | {TestCase,GroupName} on_tc_skip(How, {Suite, Case, Reason}) -> call(fun call_cleanup/3, {How, Reason}, [on_tc_skip, Suite, Case]). diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index 8326705575..53cfbd7118 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -728,23 +728,25 @@ test_events(lib_error) -> {lib_error_1_SUITE,no_lines_throw,{failed,{error,{thrown,catch_me_if_u_can}}}}}, {?eh,test_stats,{0,8,{0,0}}}, {?eh,tc_start,{lib_error_1_SUITE,init_tc_error}}, - {?eh,tc_done,{lib_error_1_SUITE,init_tc_error,ok}}, - {?eh,test_stats,{1,8,{0,0}}}, + {?eh,tc_done,{ct_framework,init_tc, + {framework_error,{{badmatch,[1,2]},'_'}}}}, + {?eh,test_stats,{0,9,{0,0}}}, {?eh,tc_start,{lib_error_1_SUITE,init_tc_exit}}, - {?eh,tc_done,{lib_error_1_SUITE,init_tc_exit,ok}}, - {?eh,test_stats,{2,8,{0,0}}}, + {?eh,tc_done,{ct_framework,init_tc,{framework_error,byebye}}}, + {?eh,test_stats,{0,10,{0,0}}}, {?eh,tc_start,{lib_error_1_SUITE,init_tc_throw}}, - {?eh,tc_done,{lib_error_1_SUITE,init_tc_throw,ok}}, - {?eh,test_stats,{3,8,{0,0}}}, + {?eh,tc_done,{ct_framework,init_tc,{framework_error,catch_me_if_u_can}}}, + {?eh,test_stats,{0,11,{0,0}}}, {?eh,tc_start,{lib_error_1_SUITE,end_tc_error}}, - {?eh,tc_done,{lib_error_1_SUITE,end_tc_error,ok}}, - {?eh,test_stats,{3,9,{0,0}}}, + {?eh,tc_done,{ct_framework,end_tc, + {framework_error,{{badmatch,[1,2]},'_'}}}}, + {?eh,test_stats,{0,12,{0,0}}}, {?eh,tc_start,{lib_error_1_SUITE,end_tc_exit}}, - {?eh,tc_done,{lib_error_1_SUITE,end_tc_exit,ok}}, - {?eh,test_stats,{3,10,{0,0}}}, + {?eh,tc_done,{ct_framework,end_tc,{framework_error,byebye}}}, + {?eh,test_stats,{0,13,{0,0}}}, {?eh,tc_start,{lib_error_1_SUITE,end_tc_throw}}, - {?eh,tc_done,{lib_error_1_SUITE,end_tc_throw,ok}}, - {?eh,test_stats,{3,11,{0,0}}}, + {?eh,tc_done,{ct_framework,end_tc,{framework_error,catch_me_if_u_can}}}, + {?eh,test_stats,{0,14,{0,0}}}, {?eh,tc_start,{lib_error_1_SUITE,end_per_suite}}, {?eh,tc_done,{lib_error_1_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/lib_error_1_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/lib_error_1_SUITE.erl index 658c7e8232..6f1b391ae6 100644 --- a/lib/common_test/test/ct_error_SUITE_data/error/test/lib_error_1_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/lib_error_1_SUITE.erl @@ -152,23 +152,32 @@ no_lines_throw(_) -> lib_no_lines:do_throw(), ok. -init_tc_error(_) -> +init_tc_error() -> put('$test_server_framework_test', fun(init_tc, _Default) -> lib_no_lines:do_error(), ok; (_, Default) -> Default - end), ok. + end), []. -init_tc_exit(_) -> +init_tc_error(_) -> + ok. + +init_tc_exit() -> put('$test_server_framework_test', fun(init_tc, _Default) -> lib_no_lines:do_exit(), ok; (_, Default) -> Default - end), ok. + end), []. -init_tc_throw(_) -> +init_tc_exit(_) -> + ok. + +init_tc_throw() -> put('$test_server_framework_test', fun(init_tc, _Default) -> lib_no_lines:do_throw(), ok; (_, Default) -> Default - end), ok. + end), []. + +init_tc_throw(_) -> + ok. end_tc_error(_) -> put('$test_server_framework_test', diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl index f00b2bbb43..8353f5936c 100644 --- a/lib/common_test/test/ct_hooks_SUITE.erl +++ b/lib/common_test/test/ct_hooks_SUITE.erl @@ -324,6 +324,8 @@ test_events(one_empty_cth) -> {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, {?eh,cth,{empty_cth,pre_init_per_testcase,[test_case,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_testcase,[test_case,'$proplist','_',[]]}}, + {?eh,cth,{empty_cth,pre_end_per_testcase,[test_case,'$proplist',[]]}}, {?eh,cth,{empty_cth,post_end_per_testcase,[test_case,'$proplist','_',[]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, diff --git a/lib/test_server/src/test_server.erl b/lib/test_server/src/test_server.erl index 9bab85bcc9..5d5e996bb2 100644 --- a/lib/test_server/src/test_server.erl +++ b/lib/test_server/src/test_server.erl @@ -716,10 +716,10 @@ call_end_conf(Mod,Func,TCPid,TCExitReason,Loc,Conf,TVal) -> Starter ! {self(),{call_end_conf,Data,ok}} end); true -> - do_call_end_conf(Starter,Mod,Func,Data,Conf,TVal) + do_call_end_conf(Starter,Mod,Func,Data,TCExitReason,Conf,TVal) end. -do_call_end_conf(Starter,Mod,Func,Data,Conf,TVal) -> +do_call_end_conf(Starter,Mod,Func,Data,TCExitReason,Conf,TVal) -> EndConfProc = fun() -> process_flag(trap_exit,true), % to catch timetraps @@ -727,16 +727,29 @@ do_call_end_conf(Starter,Mod,Func,Data,Conf,TVal) -> EndConfApply = fun() -> timetrap(TVal), - try apply(Mod,end_per_testcase,[Func,Conf]) of + %% We can't handle fails or skips here + %% (neither input nor output). The error can + %% be read from Conf though (tc_status). + EndConf = + case do_init_tc_call(Mod,{end_per_testcase,Func}, + [Conf], + {TCExitReason,[Conf]}) of + {_,[EPTCInit]} when is_list(EPTCInit) -> + EPTCInit; + _ -> + Conf + end, + try apply(Mod,end_per_testcase,[Func,EndConf]) of _ -> ok catch _:Why -> timer:sleep(1), - group_leader() ! {printout,12, - "WARNING! " - "~w:end_per_testcase(~w, ~p)" - " crashed!\n\tReason: ~p\n", - [Mod,Func,Conf,Why]} + GLMsg = {printout,12, + "WARNING! " + "~w:end_per_testcase(~w, ~p)" + " crashed!\n\tReason: ~p\n", + [Mod,Func,Conf,Why]}, + group_leader() ! GLMsg end, Supervisor ! {self(),end_conf} end, @@ -757,28 +770,38 @@ do_call_end_conf(Starter,Mod,Func,Data,Conf,TVal) -> end, spawn_link(EndConfProc). -spawn_fw_call(Mod,{init_per_testcase,Func},CurrConf,Pid, - {timetrap_timeout,TVal}=Why, - Loc,SendTo) -> +spawn_fw_call(Mod,IPTC={init_per_testcase,Func},CurrConf,Pid, + Why,Loc,SendTo) -> FwCall = fun() -> Skip = {skip,{failed,{Mod,init_per_testcase,Why}}}, %% if init_per_testcase fails, the test case %% should be skipped - try do_end_tc_call(Mod,Func, {Pid,Skip,[CurrConf]}, Why) of + try begin do_end_tc_call(Mod,IPTC, {Pid,Skip,[CurrConf]}, Why), + do_init_tc_call(Mod,{end_per_testcase,Func}, + [CurrConf],{ok,[CurrConf]}), + do_end_tc_call(Mod,{end_per_testcase,Func}, + {Pid,Skip,[CurrConf]}, Why) end of _ -> ok catch _:FwEndTCErr -> exit({fw_notify_done,end_tc,FwEndTCErr}) end, + Time = case Why of + {timetrap_timeout,TVal} -> TVal/1000; + _ -> died + end, + group_leader() ! {printout,12, + "ERROR! ~w:init_per_testcase(~w, ~p)" + " failed!\n\tReason: ~tp\n", + [Mod,Func,CurrConf,Why]}, %% finished, report back - SendTo ! {self(),fw_notify_done, - {TVal/1000,Skip,Loc,[],undefined}} + SendTo ! {self(),fw_notify_done,{Time,Skip,Loc,[],undefined}} end, spawn_link(FwCall); -spawn_fw_call(Mod,{end_per_testcase,Func},EndConf,Pid, - {timetrap_timeout,TVal}=Why,_Loc,SendTo) -> +spawn_fw_call(Mod,EPTC={end_per_testcase,Func},EndConf,Pid, + Why,_Loc,SendTo) -> FwCall = fun() -> {RetVal,Report} = @@ -792,24 +815,38 @@ spawn_fw_call(Mod,{end_per_testcase,Func},EndConf,Pid, E = {failed,{Mod,end_per_testcase,Why}}, {Result,E} end, - group_leader() ! {printout,12, - "WARNING! ~w:end_per_testcase(~w, ~p)" - " failed!\n\tReason: timetrap timeout" - " after ~w ms!\n", [Mod,Func,EndConf,TVal]}, - FailLoc = proplists:get_value(tc_fail_loc, EndConf), - try do_end_tc_call(Mod,Func, - {Pid,Report,[EndConf]}, Why) of + {Time,Warn} = + case Why of + {timetrap_timeout,TVal} -> + group_leader() ! + {printout,12, + "WARNING! ~w:end_per_testcase(~w, ~p)" + " failed!\n\tReason: timetrap timeout" + " after ~w ms!\n", [Mod,Func,EndConf,TVal]}, + W = "<font color=\"red\">" + "WARNING: end_per_testcase timed out!</font>", + {TVal/1000,W}; + _ -> + group_leader() ! + {printout,12, + "WARNING! ~w:end_per_testcase(~w, ~p)" + " failed!\n\tReason: ~tp\n", + [Mod,Func,EndConf,Why]}, + W = "<font color=\"red\">" + "WARNING: end_per_testcase failed!</font>", + {died,W} + end, + try do_end_tc_call(Mod,EPTC,{Pid,Report,[EndConf]}, Why) of _ -> ok catch _:FwEndTCErr -> exit({fw_notify_done,end_tc,FwEndTCErr}) end, - Warn = "<font color=\"red\">" - "WARNING: end_per_testcase timed out!</font>", + FailLoc = proplists:get_value(tc_fail_loc, EndConf), %% finished, report back (if end_per_testcase fails, a warning %% should be printed as part of the comment) SendTo ! {self(),fw_notify_done, - {TVal/1000,RetVal,FailLoc,[],Warn}} + {Time,RetVal,FailLoc,[],Warn}} end, spawn_link(FwCall); @@ -832,10 +869,12 @@ spawn_fw_call(FwMod,FwFunc,_,_Pid,{framework_error,FwError},_,SendTo) -> spawn_link(FwCall); spawn_fw_call(Mod,Func,CurrConf,Pid,Error,Loc,SendTo) -> - Func1 = case Func of - {_InitOrEndPerTC,F} -> F; - F -> F - end, + {Func1,EndTCFunc} = case Func of + CF when CF == init_per_suite; CF == end_per_suite; + CF == init_per_group; CF == end_per_group -> + {CF,CF}; + TC -> {TC,{end_per_testcase,TC}} + end, FwCall = fun() -> try fw_error_notify(Mod,Func1,[], @@ -847,8 +886,7 @@ spawn_fw_call(Mod,Func,CurrConf,Pid,Error,Loc,SendTo) -> FwErrorNotifyErr}) end, Conf = [{tc_status,{failed,Error}}|CurrConf], - try do_end_tc_call(Mod,Func1, - {Pid,Error,[Conf]},Error) of + try do_end_tc_call(Mod,EndTCFunc,{Pid,Error,[Conf]},Error) of _ -> ok catch _:FwEndTCErr -> @@ -942,7 +980,7 @@ run_test_case_eval(Mod, Func, Args0, Name, Ref, RunInit, {auto_skip,{failed,Error}}), {{0,NewResult},Where,[]}; {fail,Reason} -> - Conf = [{tc_status,{failed,Reason}} | hd(Args0)], + Conf = [{tc_status,{failed,Reason}} | hd(Args0)], fw_error_notify(Mod, Func, Conf, Reason), NewResult = do_end_tc_call(Mod,FWInitFunc, {{error,Reason},[Conf]}, @@ -1104,20 +1142,21 @@ do_end_tc_call(Mod, IPTC={init_per_testcase,Func}, Res, Return) -> case Return of {NOk,_} when NOk == auto_skip; NOk == fail; NOk == skip ; NOk == skipped -> + {_,Args} = Res, IPTCEndRes = case do_end_tc_call1(Mod, IPTC, Res, Return) of IPTCEndConfig when is_list(IPTCEndConfig) -> IPTCEndConfig; _ -> - Res + Args end, EPTCInitRes = case do_init_tc_call(Mod,{end_per_testcase,Func}, IPTCEndRes,Return) of {ok,EPTCInitConfig} when is_list(EPTCInitConfig) -> - EPTCInitConfig; + {Return,EPTCInitConfig}; _ -> - IPTCEndRes + Return end, do_end_tc_call1(Mod, {end_per_testcase,Func}, EPTCInitRes, Return); |