%% %% %CopyrightBegin% %% %% Copyright Ericsson AB 2004-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. %% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. %% %% %CopyrightEnd% %% %%% @doc Common Test Framework callback module. %%% %%%

This module exports framework callback functions which are %%% called from the test_server.

-module(ct_framework). -export([init_tc/3, end_tc/3, end_tc/4, get_suite/2, get_all_cases/1]). -export([report/2, warn/1, error_notification/4]). -export([get_logopts/0, format_comment/1, get_html_wrapper/3]). -export([error_in_suite/1, init_per_suite/1, end_per_suite/1, init_per_group/2, end_per_group/2]). -export([make_all_conf/3, make_conf/5]). -include("ct_event.hrl"). -include("ct_util.hrl"). -define(val(Key, List), proplists:get_value(Key, List)). -define(val(Key, List, Def), proplists:get_value(Key, List, Def)). -define(rev(L), lists:reverse(L)). %%%----------------------------------------------------------------- %%% @spec init_tc(Mod,Func,Args) -> {ok,NewArgs} | {error,Reason} | %%% {skip,Reason} | {auto_skip,Reason} %%% Mod = atom() %%% Func = atom() %%% Args = list() %%% NewArgs = list() %%% Reason = term() %%% %%% @doc Test server framework callback, called by the test_server %%% when a new test case is started. init_tc(Mod,Func,Config) -> %% in case Mod == ct_framework, lookup the suite name Suite = get_suite_name(Mod, Config), %% check if previous testcase was interpreted and has left %% a "dead" trace window behind - if so, kill it case ct_util:get_testdata(interpret) of {What,kill,{TCPid,AttPid}} -> ct_util:kill_attached(TCPid,AttPid), ct_util:set_testdata({interpret,{What,kill,{undefined,undefined}}}); _ -> ok end, %% check if we need to add defaults explicitly because %% there's no init_per_suite exported from Mod {InitFailed,DoInit} = case ct_util:get_testdata(curr_tc) of {Suite,{suite0_failed,_}=Failure} -> {Failure,false}; {?MODULE,_} when Func == init_per_suite -> {false,true}; % no init_per_suite in Mod {?MODULE,_} -> {false,false}; % should not really happen {Suite,_} -> {false,false}; % Func is not 1st case in suite _ when Func == init_per_suite -> {false,false}; % defaults will be added anyway _ -> {false,true} % first case in suite end, case InitFailed of false -> ct_util:set_testdata({curr_tc,{Suite,Func}}), case ct_util:read_suite_data({seq,Suite,Func}) of undefined -> init_tc1(Mod,Func,Config,DoInit); Seq when is_atom(Seq) -> case ct_util:read_suite_data({seq,Suite,Seq}) of [Func|TCs] -> % this is the 1st case in Seq %% make sure no cases in this seq are marked as failed %% from an earlier execution in the same suite lists:foreach( fun(TC) -> ct_util:save_suite_data({seq,Suite,TC},Seq) end, TCs); _ -> ok end, init_tc1(Mod,Func,Config,DoInit); {failed,Seq,BadFunc} -> {skip,{sequence_failed,Seq,BadFunc}} end; {_,{require,Reason}} -> {skip,{require_failed_in_suite0,Reason}}; _ -> {skip,InitFailed} end. init_tc1(?MODULE,error_in_suite,[Config0],_) when is_list(Config0) -> ct_logs:init_tc(false), ct_event:notify(#event{name=tc_start, node=node(), data={?MODULE,error_in_suite}}), case ?val(error, Config0) of undefined -> {skip,"unknown_error_in_suite"}; Reason -> {skip,Reason} end; init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) -> Config1 = case ct_util:read_suite_data(last_saved_config) of {{Mod,LastFunc},SavedConfig} -> % last testcase [{saved_config,{LastFunc,SavedConfig}} | lists:keydelete(saved_config,1,Config0)]; {{LastSuite,InitOrEnd}, SavedConfig} when InitOrEnd == init_per_suite ; InitOrEnd == end_per_suite -> %% last suite [{saved_config,{LastSuite,SavedConfig}} | lists:keydelete(saved_config,1,Config0)]; undefined -> lists:keydelete(saved_config,1,Config0) end, ct_util:delete_suite_data(last_saved_config), Config = lists:keydelete(watchdog,1,Config1), if Func /= init_per_suite, DoInit /= true -> ok; true -> %% delete all default values used in previous suite ct_config:delete_default_config(suite), %% release all name -> key bindings (once per suite) ct_config:release_allocated() end, GroupPath = ?val(tc_group_path, Config, []), AllGroups = [?val(tc_group_properties, Config, []) | GroupPath], %% clear all config data default values set by previous %% testcase info function (these should only survive the %% testcase, not the whole suite) FuncSpec = group_or_func(Func,Config0), if is_tuple(FuncSpec) -> % group ok; true -> ct_config:delete_default_config(testcase) end, %% in case Mod == ct_framework, lookup the suite name Suite = get_suite_name(Mod, Config), case add_defaults(Mod,Func,AllGroups,DoInit) of Error = {suite0_failed,_} -> ct_logs:init_tc(false), ct_event:notify(#event{name=tc_start, node=node(), data={Mod,FuncSpec}}), ct_util:set_testdata({curr_tc,{Suite,Error}}), {error,Error}; {SuiteInfo,MergeResult} -> case MergeResult of {error,Reason} when DoInit == false -> ct_logs:init_tc(false), ct_event:notify(#event{name=tc_start, node=node(), data={Mod,FuncSpec}}), {skip,Reason}; _ -> init_tc2(Mod,Func,SuiteInfo,MergeResult, Config,DoInit) end end; init_tc1(_Mod,_Func,Args,_DoInit) -> {ok,Args}. init_tc2(Mod,Func,SuiteInfo,MergeResult,Config,DoInit) -> %% if first testcase fails when there's no init_per_suite %% we must do suite/0 configurations before skipping test MergedInfo = case MergeResult of {error,_} when DoInit == true -> SuiteInfo; _ -> MergeResult end, %% timetrap must be handled before require MergedInfo1 = timetrap_first(MergedInfo, [], []), %% tell logger to use specified style sheet case lists:keysearch(stylesheet,1,MergedInfo++Config) of {value,{stylesheet,SSFile}} -> ct_logs:set_stylesheet(Func,add_data_dir(SSFile,Config)); _ -> case ct_util:get_testdata(stylesheet) of undefined -> ct_logs:clear_stylesheet(Func); SSFile -> ct_logs:set_stylesheet(Func,SSFile) end end, %% suppress output for connections (Conns is a %% list of {Type,Bool} tuples, e.g. {telnet,true}), case ct_util:get_overridden_silenced_connections() of undefined -> case lists:keysearch(silent_connections,1,MergedInfo++Config) of {value,{silent_connections,Conns}} -> ct_util:silence_connections(Conns); _ -> ok end; Conns -> ct_util:silence_connections(Conns) end, if Func /= init_per_suite, DoInit /= true -> ct_logs:init_tc(false); true -> ct_logs:init_tc(true) end, FuncSpec = group_or_func(Func,Config), ct_event:notify(#event{name=tc_start, node=node(), data={Mod,FuncSpec}}), case catch configure(MergedInfo1,MergedInfo1,SuiteInfo, {FuncSpec,DoInit},Config) of {suite0_failed,Reason} -> ct_util:set_testdata({curr_tc,{Mod,{suite0_failed,{require,Reason}}}}), {skip,{require_failed_in_suite0,Reason}}; {error,Reason} -> {auto_skip,{require_failed,Reason}}; {'EXIT',Reason} -> {auto_skip,Reason}; {ok,FinalConfig} -> case MergeResult of {error,Reason} -> %% suite0 configure finished now, report that %% first test case actually failed {skip,Reason}; _ -> case get('$test_server_framework_test') of undefined -> ct_suite_init(Mod, FuncSpec, FinalConfig); Fun -> case Fun(init_tc, FinalConfig) of NewConfig when is_list(NewConfig) -> {ok,NewConfig}; Else -> Else end end end end. ct_suite_init(Mod, Func, [Config]) when is_list(Config) -> %%! --- Thu Mar 15 15:16:17 2012 --- peppe was here! %%!io:format(user, "+++ ct_suite_init ~p:~p -- ~p~n", [Mod,Func,Config]), case ct_hooks:init_tc(Mod, Func, Config) of NewConfig when is_list(NewConfig) -> {ok, [NewConfig]}; Else -> Else end. add_defaults(Mod,Func, GroupPath, DoInit) -> Suite = get_suite_name(Mod, GroupPath), case (catch Suite:suite()) of {'EXIT',{undef,_}} -> SuiteInfo = merge_with_suite_defaults(Suite,[]), SuiteInfoNoCTH = [I || I <- SuiteInfo, element(1,I) =/= ct_hooks], case add_defaults1(Mod,Func, GroupPath, SuiteInfoNoCTH, DoInit) of Error = {error,_} -> {SuiteInfo,Error}; MergedInfo -> {SuiteInfo,MergedInfo} end; {'EXIT',Reason} -> ErrStr = io_lib:format("~n*** ERROR *** " "~w:suite/0 failed: ~p~n", [Suite,Reason]), io:format(ErrStr, []), io:format(user, ErrStr, []), {suite0_failed,{exited,Reason}}; SuiteInfo when is_list(SuiteInfo) -> case lists:all(fun(E) when is_tuple(E) -> true; (_) -> false end, SuiteInfo) of true -> SuiteInfo1 = merge_with_suite_defaults(Suite, SuiteInfo), SuiteInfoNoCTH = [I || I <- SuiteInfo1, element(1,I) =/= ct_hooks], case add_defaults1(Mod,Func, GroupPath, SuiteInfoNoCTH, DoInit) of Error = {error,_} -> {SuiteInfo1,Error}; MergedInfo -> {SuiteInfo1,MergedInfo} end; false -> ErrStr = io_lib:format("~n*** ERROR *** " "Invalid return value from " "~w:suite/0: ~p~n", [Suite,SuiteInfo]), io:format(ErrStr, []), io:format(user, ErrStr, []), {suite0_failed,bad_return_value} end; SuiteInfo -> ErrStr = io_lib:format("~n*** ERROR *** " "Invalid return value from " "~w:suite/0: ~p~n", [Suite,SuiteInfo]), io:format(ErrStr, []), io:format(user, ErrStr, []), {suite0_failed,bad_return_value} end. add_defaults1(Mod,Func, GroupPath, SuiteInfo, DoInit) -> Suite = get_suite_name(Mod, GroupPath), %% GroupPathInfo (for subgroup on level X) = %% [LevelXGroupInfo, LevelX-1GroupInfo, ..., TopLevelGroupInfo] GroupPathInfo = lists:map(fun(GroupProps) -> Name = ?val(name, GroupProps), case catch Suite:group(Name) of GrInfo when is_list(GrInfo) -> GrInfo; _ -> [] end end, GroupPath), Args = if Func == init_per_group ; Func == end_per_group -> [?val(name, hd(GroupPath))]; true -> [] end, TestCaseInfo = case catch apply(Mod,Func,Args) of TCInfo when is_list(TCInfo) -> TCInfo; _ -> [] end, %% let test case info (also for all config funcs) override group info, %% and lower level group info override higher level info TCAndGroupInfo = [TestCaseInfo | remove_info_in_prev(TestCaseInfo, GroupPathInfo)], %% find and save require terms found in suite info SuiteReqs = [SDDef || SDDef <- SuiteInfo, ((require == element(1,SDDef)) or (default_config == element(1,SDDef)))], case check_for_clashes(TestCaseInfo, GroupPathInfo, SuiteReqs) of [] -> add_defaults2(Mod,Func, TCAndGroupInfo,SuiteInfo,SuiteReqs, DoInit); Clashes -> {error,{config_name_already_in_use,Clashes}} end. get_suite_name(?MODULE, [Cfg|_]) when is_list(Cfg), Cfg /= [] -> get_suite_name(?MODULE, Cfg); get_suite_name(?MODULE, Cfg) when is_list(Cfg), Cfg /= [] -> case ?val(tc_group_properties, Cfg) of undefined -> case ?val(suite, Cfg) of undefined -> ?MODULE; Suite -> Suite end; GrProps -> case ?val(suite, GrProps) of undefined -> ?MODULE; Suite -> Suite end end; get_suite_name(Mod, _) -> Mod. %% Check that alias names are not already in use check_for_clashes(TCInfo, GrPathInfo, SuiteInfo) -> {CurrGrInfo,SearchIn} = case GrPathInfo of [] -> {[],[SuiteInfo]}; [Curr|Path] -> {Curr,[SuiteInfo|Path]} end, ReqNames = fun(Info) -> [element(2,R) || R <- Info, size(R) == 3, require == element(1,R)] end, ExistingNames = lists:flatten([ReqNames(L) || L <- SearchIn]), CurrGrReqNs = ReqNames(CurrGrInfo), GrClashes = [Name || Name <- CurrGrReqNs, true == lists:member(Name, ExistingNames)], AllReqNs = CurrGrReqNs ++ ExistingNames, TCClashes = [Name || Name <- ReqNames(TCInfo), true == lists:member(Name, AllReqNs)], TCClashes ++ GrClashes. %% Delete the info terms in Terms from all following info lists remove_info_in_prev(Terms, [[] | Rest]) -> [[] | remove_info_in_prev(Terms, Rest)]; remove_info_in_prev(Terms, [Info | Rest]) -> UniqueInInfo = [U || U <- Info, ((timetrap == element(1,U)) and (not lists:keymember(timetrap,1,Terms))) or ((require == element(1,U)) and (not lists:member(U,Terms))) or ((default_config == element(1,U)) and (not keysmember([default_config,1, element(2,U),2], Terms)))], OtherTermsInInfo = [T || T <- Info, timetrap /= element(1,T), require /= element(1,T), default_config /= element(1,T), false == lists:keymember(element(1,T),1, Terms)], KeptInfo = UniqueInInfo ++ OtherTermsInInfo, [KeptInfo | remove_info_in_prev(Terms ++ KeptInfo, Rest)]; remove_info_in_prev(_, []) -> []. keysmember([Key,Pos|Next], List) -> case [Elem || Elem <- List, Key == element(Pos,Elem)] of [] -> false; Found -> keysmember(Next, Found) end; keysmember([], _) -> true. add_defaults2(Mod,init_per_suite, IPSInfo, SuiteInfo,SuiteReqs, false) -> add_defaults2(Mod,init_per_suite, IPSInfo, SuiteInfo,SuiteReqs, true); add_defaults2(_Mod,IPG, IPGAndGroupInfo, SuiteInfo,SuiteReqs, DoInit) when IPG == init_per_group -> %% If DoInit == true, we have to process the suite() list, otherwise %% it has already been handled (see clause for init_per_suite) case DoInit of true -> %% note: we know for sure this is a top level group Info = lists:flatten([IPGAndGroupInfo, SuiteReqs]), Info ++ remove_info_in_prev(Info, [SuiteInfo]); false -> SuiteInfo1 = remove_info_in_prev(lists:flatten([IPGAndGroupInfo, SuiteReqs]), [SuiteInfo]), %% don't require terms in prev groups (already processed) case IPGAndGroupInfo of [IPGInfo] -> lists:flatten([IPGInfo,SuiteInfo1]); [IPGInfo | [CurrGroupInfo | PrevGroupInfo]] -> PrevGroupInfo1 = delete_require_terms(PrevGroupInfo), lists:flatten([IPGInfo,CurrGroupInfo,PrevGroupInfo1, SuiteInfo1]) end end; add_defaults2(_Mod,_Func, TCAndGroupInfo, SuiteInfo,SuiteReqs, false) -> %% Include require elements from test case info and current group, %% but not from previous groups or suite/0 (since we've already required %% those vars). Let test case info elements override group and suite %% info elements. SuiteInfo1 = remove_info_in_prev(lists:flatten([TCAndGroupInfo, SuiteReqs]), [SuiteInfo]), %% don't require terms in prev groups (already processed) case TCAndGroupInfo of [TCInfo] -> lists:flatten([TCInfo,SuiteInfo1]); [TCInfo | [CurrGroupInfo | PrevGroupInfo]] -> PrevGroupInfo1 = delete_require_terms(PrevGroupInfo), lists:flatten([TCInfo,CurrGroupInfo,PrevGroupInfo1, SuiteInfo1]) end; add_defaults2(_Mod,_Func, TCInfo, SuiteInfo,SuiteReqs, true) -> %% Here we have to process the suite info list also (no call to %% init_per_suite before this first test case). This TC can't belong %% to a group, or the clause for init_per_group would've caught this. Info = lists:flatten([TCInfo, SuiteReqs]), lists:flatten([Info,remove_info_in_prev(Info, [SuiteInfo])]). delete_require_terms([Info | Prev]) -> Info1 = [T || T <- Info, require /= element(1,T), default_config /= element(1,T)], [Info1 | delete_require_terms(Prev)]; delete_require_terms([]) -> []. merge_with_suite_defaults(Mod,SuiteInfo) -> case lists:keysearch(suite_defaults,1,Mod:module_info(attributes)) of {value,{suite_defaults,Defaults}} -> SDReqs = [SDDef || SDDef <- Defaults, require == element(1,SDDef), false == lists:keymember(element(2,SDDef),2, SuiteInfo)], SuiteInfo ++ SDReqs ++ [SDDef || SDDef <- Defaults, require /= element(1,SDDef), false == lists:keymember(element(1,SDDef),1, SuiteInfo)]; false -> SuiteInfo end. timetrap_first([Trap = {timetrap,_} | Rest],Info,Found) -> timetrap_first(Rest,Info,[Trap | Found]); timetrap_first([Other | Rest],Info,Found) -> timetrap_first(Rest,[Other | Info],Found); timetrap_first([],Info,[]) -> [{timetrap,{minutes,30}} | ?rev(Info)]; timetrap_first([],Info,Found) -> ?rev(Found) ++ ?rev(Info). configure([{require,Required}|Rest],Info,SuiteInfo,Scope,Config) -> case ct:require(Required) of ok -> configure(Rest,Info,SuiteInfo,Scope,Config); Error = {error,Reason} -> case required_default('_UNDEF',Required,Info, SuiteInfo,Scope) of ok -> configure(Rest,Info,SuiteInfo,Scope,Config); _ -> case lists:keymember(Required,2,SuiteInfo) of true -> {suite0_failed,Reason}; false -> Error end end end; configure([{require,Name,Required}|Rest],Info,SuiteInfo,Scope,Config) -> case ct:require(Name,Required) of ok -> configure(Rest,Info,SuiteInfo,Scope,Config); Error = {error,Reason} -> case required_default(Name,Required,Info,SuiteInfo,Scope) of ok -> configure(Rest,Info,SuiteInfo,Scope,Config); _ -> case lists:keymember(Name,2,SuiteInfo) of true -> {suite0_failed,Reason}; false -> Error end end end; configure([{timetrap,off}|Rest],Info,SuiteInfo,Scope,Config) -> configure(Rest,Info,SuiteInfo,Scope,Config); configure([{timetrap,Time}|Rest],Info,SuiteInfo,Scope,Config) -> Dog = test_server:timetrap(Time), configure(Rest,Info,SuiteInfo,Scope,[{watchdog,Dog}|Config]); configure([{ct_hooks, Hook} | Rest], Info, SuiteInfo, Scope, Config) -> configure(Rest, Info, SuiteInfo, Scope, [{ct_hooks, Hook} | Config]); configure([_|Rest],Info,SuiteInfo,Scope,Config) -> configure(Rest,Info,SuiteInfo,Scope,Config); configure([],_,_,_,Config) -> {ok,[Config]}. %% the require element in Info may come from suite/0 and %% should be scoped 'suite', or come from the group info %% function and be scoped 'group', or come from the testcase %% info function and then be scoped 'testcase' required_default(Name,Key,Info,SuiteInfo,{FuncSpec,true}) -> case try_set_default(Name,Key,SuiteInfo,suite) of ok -> ok; _ -> required_default(Name,Key,Info,[],{FuncSpec,false}) end; required_default(Name,Key,Info,_,{init_per_suite,_}) -> try_set_default(Name,Key,Info,suite); required_default(Name,Key,Info,_,{{init_per_group,GrName,_},_}) -> try_set_default(Name,Key,Info,{group,GrName}); required_default(Name,Key,Info,_,_FuncSpec) -> try_set_default(Name,Key,Info,testcase). try_set_default(Name,Key,Info,Where) -> CfgElems = case lists:keysearch(Name,1,Info) of {value,{Name,Val}} -> [Val]; false -> case catch [{Key,element(3,Elem)} || Elem <- Info, element(1,Elem)==default_config, element(2,Elem)==Key] of {'EXIT',_} -> []; Result -> Result end end, case {Name,CfgElems} of {_,[]} -> no_default; {'_UNDEF',_} -> [ct_config:set_default_config([CfgVal],Where) || CfgVal <- CfgElems], ok; _ -> [ct_config:set_default_config(Name,[CfgVal],Where) || CfgVal <- CfgElems], ok end. %%%----------------------------------------------------------------- %%% @spec end_tc(Mod,Func,Args) -> {ok,NewArgs}| {error,Reason} | %%% {skip,Reason} | {auto_skip,Reason} %%% Mod = atom() %%% Func = atom() %%% Args = list() %%% NewArgs = list() %%% Reason = term() %%% %%% @doc Test server framework callback, called by the test_server %%% when a test case is finished. end_tc(Mod, Fun, Args) -> %% Have to keep end_tc/3 for backwards compatibility issues end_tc(Mod, Fun, Args, '$end_tc_dummy'). end_tc(?MODULE,error_in_suite,_, _) -> % bad start! ok; end_tc(Mod,Func,{TCPid,Result,[Args]}, Return) when is_pid(TCPid) -> end_tc(Mod,Func,TCPid,Result,Args,Return); end_tc(Mod,Func,{Result,[Args]}, Return) -> end_tc(Mod,Func,self(),Result,Args,Return). end_tc(Mod,Func,TCPid,Result,Args,Return) -> case lists:keysearch(watchdog,1,Args) of {value,{watchdog,Dog}} -> test_server:timetrap_cancel(Dog); false -> ok end, %% save the testcase process pid so that it can be used %% to look up the attached trace window later case ct_util:get_testdata(interpret) of {What,kill,_} -> AttPid = ct_util:get_attached(self()), ct_util:set_testdata({interpret,{What,kill,{self(),AttPid}}}); _ -> ok end, ct_util:delete_testdata(comment), ct_util:delete_suite_data(last_saved_config), FuncSpec = case group_or_func(Func,Args) of {_,GroupName,_Props} = Group -> if Func == end_per_group -> ct_config:delete_default_config({group,GroupName}); true -> ok end, case lists:keysearch(save_config,1,Args) of {value,{save_config,SaveConfig}} -> ct_util:save_suite_data( last_saved_config, {Mod,{group,GroupName}}, SaveConfig), Group; false -> Group end; _ -> case lists:keysearch(save_config,1,Args) of {value,{save_config,SaveConfig}} -> ct_util:save_suite_data(last_saved_config, {Mod,Func},SaveConfig), Func; false -> Func end end, ct_util:reset_silent_connections(), case get('$test_server_framework_test') of undefined -> {FinalResult,FinalNotify} = case ct_hooks:end_tc( Mod, FuncSpec, Args, Result, Return) of '$ct_no_change' -> {ok,Result}; FinalResult1 -> {FinalResult1,FinalResult1} end, % send sync notification so that event handlers may print % in the log file before it gets closed ct_event:sync_notify(#event{name=tc_done, node=node(), data={Mod,FuncSpec, tag_cth(FinalNotify)}}); Fun -> % send sync notification so that event handlers may print % in the log file before it gets closed ct_event:sync_notify(#event{name=tc_done, node=node(), data={Mod,FuncSpec,tag(Result)}}), FinalResult = Fun(end_tc, Return) end, case FinalResult of {skip,{sequence_failed,_,_}} -> %% ct_logs:init_tc is never called for a skipped test case %% in a failing sequence, so neither should end_tc ok; _ -> case ct_logs:end_tc(TCPid) of {error,Reason} -> exit({error,{logger,Reason}}); _ -> ok end end, case Func of end_per_suite -> ct_util:match_delete_suite_data({seq,Mod,'_'}); _ -> ok end, FinalResult. %% {error,Reason} | {skip,Reason} | {timetrap_timeout,TVal} | %% {testcase_aborted,Reason} | testcase_aborted_or_killed | %% {'EXIT',Reason} | Other (ignored return value, e.g. 'ok') tag({STag,Reason}) when STag == skip; STag == skipped -> {skipped,Reason}; tag(E = {ETag,_}) when ETag == error; ETag == 'EXIT'; ETag == timetrap_timeout; ETag == testcase_aborted -> {failed,E}; tag(E = testcase_aborted_or_killed) -> {failed,E}; tag(Other) -> Other. tag_cth({STag,Reason}) when STag == skip; STag == skipped -> {skipped,Reason}; tag_cth({fail, Reason}) -> {failed, {error,Reason}}; tag_cth(E = {ETag,_}) when ETag == error; ETag == 'EXIT'; ETag == timetrap_timeout; ETag == testcase_aborted -> {failed,E}; tag_cth(E = testcase_aborted_or_killed) -> {failed,E}; tag_cth(List) when is_list(List) -> ok; tag_cth(Other) -> Other. %%%----------------------------------------------------------------- %%% @spec error_notification(Mod,Func,Args,Error) -> ok %%% Mod = atom() %%% Func = atom() %%% Args = list() %%% Error = term() %%% %%% @doc This function is called as the result of testcase %%% Func in suite Mod crashing. %%% Error specifies the reason for failing. error_notification(Mod,Func,_Args,{Error,Loc}) -> ErrSpec = case Error of {What={_E,_R},Trace} when is_list(Trace) -> What; What -> What end, ErrStr = case ErrSpec of {badmatch,Descr} -> Descr1 = lists:flatten(io_lib:format("~P",[Descr,10])), if length(Descr1) > 50 -> Descr2 = string:substr(Descr1,1,50), io_lib:format("{badmatch,~s...}",[Descr2]); true -> io_lib:format("{badmatch,~s}",[Descr1]) end; {test_case_failed,Reason} -> case (catch io_lib:format("{test_case_failed,~s}", [Reason])) of {'EXIT',_} -> io_lib:format("{test_case_failed,~p}", [Reason]); Result -> Result end; {Spec,_Reason} when is_atom(Spec) -> io_lib:format("~w", [Spec]); Other -> io_lib:format("~P", [Other,5]) end, ErrorHtml = "" ++ ErrStr ++ "", case {Mod,Error} of %% some notifications come from the main test_server process %% and for these cases the existing comment may not be modified {_,{timetrap_timeout,_TVal}} -> ok; {_,{testcase_aborted,_Info}} -> ok; {_,testcase_aborted_or_killed} -> ok; {undefined,_OtherError} -> ok; _ -> %% this notification comes from the test case process, so %% we can add error info to comment with test_server:comment/1 case ct_util:get_testdata(comment) of undefined -> test_server:comment(ErrorHtml); Comment -> CommentHtml = "" ++ "(" ++ "" ++ Comment ++ "" ++ ")" ++ "", Str = io_lib:format("~s ~s", [ErrorHtml,CommentHtml]), test_server:comment(Str) end end, PrintErr = fun(ErrFormat, ErrArgs) -> Div = "~n- - - - - - - - - - - - - - - - " "- - - - - - - - - -~n", io:format(user, lists:concat([Div,ErrFormat,Div,"~n"]), ErrArgs), ct_logs:tc_log(ct_error_notify, "CT Error Notification", ErrFormat, ErrArgs) end, case Loc of [{?MODULE,error_in_suite}] -> PrintErr("Error in suite detected: ~s", [ErrStr]); R when R == unknown; R == undefined -> PrintErr("Error detected: ~s", [ErrStr]); %% if a function specified by all/0 does not exist, we %% pick up undef here [{LastMod,LastFunc}|_] when ErrStr == "undef" -> PrintErr("~w:~w could not be executed~nReason: ~s", [LastMod,LastFunc,ErrStr]); [{LastMod,LastFunc}|_] -> PrintErr("~w:~w failed~nReason: ~s", [LastMod,LastFunc,ErrStr]); [{LastMod,LastFunc,LastLine}|_] -> %% print error to console, we are only %% interested in the last executed expression PrintErr("~w:~w failed on line ~w~nReason: ~s", [LastMod,LastFunc,LastLine,ErrStr]), case ct_util:read_suite_data({seq,Mod,Func}) of undefined -> ok; Seq -> SeqTCs = ct_util:read_suite_data({seq,Mod,Seq}), mark_as_failed(Seq,Mod,Func,SeqTCs) end end, ok. %% cases in seq that have already run mark_as_failed(Seq,Mod,Func,[Func|TCs]) -> mark_as_failed1(Seq,Mod,Func,TCs); mark_as_failed(Seq,Mod,Func,[_TC|TCs]) -> mark_as_failed(Seq,Mod,Func,TCs); mark_as_failed(_,_,_,[]) -> ok; mark_as_failed(_,_,_,undefined) -> ok. %% mark rest of cases in seq to be skipped mark_as_failed1(Seq,Mod,Func,[TC|TCs]) -> ct_util:save_suite_data({seq,Mod,TC},{failed,Seq,Func}), mark_as_failed1(Seq,Mod,Func,TCs); mark_as_failed1(_,_,_,[]) -> ok. group_or_func(Func, Config) when Func == init_per_group; Func == end_per_group -> case ?val(tc_group_properties, Config) of undefined -> {Func,unknown,[]}; GrProps -> GrName = ?val(name,GrProps), {Func,GrName,proplists:delete(name,GrProps)} end; group_or_func(Func, _Config) -> Func. %%%----------------------------------------------------------------- %%% @spec get_suite(Mod, Func) -> Tests %%% %%% @doc Called from test_server for every suite (Func==all) %%% and every test case. If the former, all test cases in the suite %%% should be returned. get_suite(Mod, all) -> case catch apply(Mod, groups, []) of {'EXIT',_} -> get_all(Mod, []); GroupDefs when is_list(GroupDefs) -> case catch find_groups(Mod, all, all, GroupDefs) of {error,_} = Error -> %% this makes test_server call error_in_suite as first %% (and only) test case so we can report Error properly [{?MODULE,error_in_suite,[[Error]]}]; ConfTests -> get_all(Mod, ConfTests) end; _ -> E = "Bad return value from "++atom_to_list(Mod)++":groups/0", [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}] end; %%!============================================================ %%! Note: The handling of sequences in get_suite/2 and get_all/2 %%! is deprecated and should be removed at some point... %%!============================================================ %% group get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) -> Name = ?val(name, Props), case catch apply(Mod, groups, []) of {'EXIT',_} -> [Group]; GroupDefs when is_list(GroupDefs) -> case catch find_groups(Mod, Name, TCs, GroupDefs) of {error,_} = Error -> %% this makes test_server call error_in_suite as first %% (and only) test case so we can report Error properly [{?MODULE,error_in_suite,[[Error]]}]; [] -> []; ConfTests -> case lists:member(skipped, Props) of true -> %% a *subgroup* specified *only* as skipped (and not %% as an explicit test) should not be returned, or %% init/end functions for top groups will be executed case catch ?val(name, element(2, hd(ConfTests))) of Name -> % top group delete_subs(ConfTests, ConfTests); _ -> [] end; false -> ConfTests1 = delete_subs(ConfTests, ConfTests), case ?val(override, Props) of undefined -> ConfTests1; [] -> ConfTests1; ORSpec -> ORSpec1 = if is_tuple(ORSpec) -> [ORSpec]; true -> ORSpec end, search_and_override(ConfTests1, ORSpec1, Mod) end end end; _ -> E = "Bad return value from "++atom_to_list(Mod)++":groups/0", [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}] end; %% testcase get_suite(Mod, Name) -> get_seq(Mod, Name). %%%----------------------------------------------------------------- get_all_cases(Suite) -> case get_suite(Suite, all) of [{?MODULE,error_in_suite,[[{error,_}=Error]]}] -> Error; [{?MODULE,error_in_suite,[[Error]]}] -> {error,Error}; Tests -> Cases = get_all_cases1(Suite, Tests), ?rev(lists:foldl(fun(TC, TCs) -> case lists:member(TC, TCs) of true -> TCs; false -> [TC | TCs] end end, [], Cases)) end. get_all_cases1(Suite, [{conf,_Props,_Init,GrTests,_End} | Tests]) -> get_all_cases1(Suite, GrTests) ++ get_all_cases1(Suite, Tests); get_all_cases1(Suite, [Test | Tests]) when is_atom(Test) -> [{Suite,Test} | get_all_cases1(Suite, Tests)]; get_all_cases1(Suite, [Test | Tests]) -> [Test | get_all_cases1(Suite, Tests)]; get_all_cases1(_, []) -> []. %%%----------------------------------------------------------------- find_groups(Mod, Name, TCs, GroupDefs) -> Found = find(Mod, Name, TCs, GroupDefs, [], GroupDefs, false), trim(Found). find(Mod, all, _TCs, [{Name,Props,Tests} | Gs], Known, Defs, _) when is_atom(Name), is_list(Props), is_list(Tests) -> cyclic_test(Mod, Name, Known), [make_conf(Mod, Name, Props, find(Mod, all, all, Tests, [Name | Known], Defs, true)) | find(Mod, all, all, Gs, [], Defs, true)]; find(Mod, Name, TCs, [{Name,Props,Tests} | _Gs], Known, Defs, false) when is_atom(Name), is_list(Props), is_list(Tests) -> cyclic_test(Mod, Name, Known), case TCs of all -> [make_conf(Mod, Name, Props, find(Mod, Name, TCs, Tests, [Name | Known], Defs, true))]; _ -> Tests1 = [TC || TC <- TCs, lists:member(TC, Tests) == true], [make_conf(Mod, Name, Props, Tests1)] end; find(Mod, Name, TCs, [{Name1,Props,Tests} | Gs], Known, Defs, false) when is_atom(Name1), is_list(Props), is_list(Tests) -> cyclic_test(Mod, Name1, Known), [make_conf(Mod,Name1,Props, find(Mod, Name, TCs, Tests, [Name1 | Known], Defs, false)) | find(Mod, Name, TCs, Gs, [], Defs, false)]; find(Mod, Name, _TCs, [{Name,_Props,_Tests} | _Gs], _Known, _Defs, true) when is_atom(Name) -> E = "Duplicate groups named "++atom_to_list(Name)++" in "++ atom_to_list(Mod)++":groups/0", throw({error,list_to_atom(E)}); find(Mod, Name, all, [{Name1,Props,Tests} | Gs], Known, Defs, true) when is_atom(Name1), is_list(Props), is_list(Tests) -> cyclic_test(Mod, Name1, Known), [make_conf(Mod, Name1, Props, find(Mod, Name, all, Tests, [Name1 | Known], Defs, true)) | find(Mod, Name, all, Gs, [], Defs, true)]; find(Mod, Name, TCs, [{group,Name1} | Gs], Known, Defs, Found) when is_atom(Name1) -> find(Mod, Name, TCs, [expand(Mod, Name1, Defs) | Gs], Known, Defs, Found); %% Undocumented remote group feature, use with caution find(Mod, Name, TCs, [{group, ExtMod, ExtGrp} | Gs], Known, Defs, true) when is_atom(ExtMod), is_atom(ExtGrp) -> ExternalDefs = ExtMod:groups(), ExternalTCs = find(ExtMod, ExtGrp, TCs, [{group, ExtGrp}], [], ExternalDefs, false), ExternalTCs ++ find(Mod, Name, TCs, Gs, Known, Defs, true); find(Mod, Name, TCs, [{Name1,Tests} | Gs], Known, Defs, Found) when is_atom(Name1), is_list(Tests) -> find(Mod, Name, TCs, [{Name1,[],Tests} | Gs], Known, Defs, Found); find(Mod, Name, TCs, [_TC | Gs], Known, Defs, false) -> find(Mod, Name, TCs, Gs, Known, Defs, false); find(Mod, Name, TCs, [TC | Gs], Known, Defs, true) when is_atom(TC) -> [{Mod, TC} | find(Mod, Name, TCs, Gs, Known, Defs, true)]; find(Mod, Name, TCs, [{ExternalTC, Case} = TC | Gs], Known, Defs, true) when is_atom(ExternalTC), is_atom(Case) -> [TC | find(Mod, Name, TCs, Gs, Known, Defs, true)]; find(Mod, _Name, _TCs, [BadTerm | _Gs], Known, _Defs, _Found) -> Where = if length(Known) == 0 -> atom_to_list(Mod)++":groups/0"; true -> "group "++atom_to_list(lists:last(Known))++ " in "++atom_to_list(Mod)++":groups/0" end, Term = io_lib:format("~p", [BadTerm]), E = "Bad term "++lists:flatten(Term)++" in "++Where, throw({error,list_to_atom(E)}); find(_Mod, _Name, _TCs, [], _Known, _Defs, false) -> ['$NOMATCH']; find(_Mod, _Name, _TCs, [], _Known, _Defs, _Found) -> []. delete_subs([{conf, _,_,_,_} = Conf | Confs], All) -> All1 = delete_conf(Conf, All), case is_sub(Conf, All1) of true -> delete_subs(Confs, All1); false -> delete_subs(Confs, All) end; delete_subs([_Else | Confs], All) -> delete_subs(Confs, All); delete_subs([], All) -> All. delete_conf({conf,Props,_,_,_}, Confs) -> Name = ?val(name, Props), [Conf || Conf = {conf,Props0,_,_,_} <- Confs, Name =/= ?val(name, Props0)]. is_sub({conf,Props,_,_,_}=Conf, [{conf,_,_,Tests,_} | Confs]) -> Name = ?val(name, Props), case lists:any(fun({conf,Props0,_,_,_}) -> case ?val(name, Props0) of N when N == Name -> true; _ -> false end; (_) -> false end, Tests) of true -> true; false -> is_sub(Conf, Tests) or is_sub(Conf, Confs) end; is_sub(Conf, [_TC | Tests]) -> is_sub(Conf, Tests); is_sub(_Conf, []) -> false. trim(['$NOMATCH' | Tests]) -> trim(Tests); trim([{conf,Props,Init,Tests,End} | Confs]) -> case trim(Tests) of [] -> trim(Confs); Trimmed -> [{conf,Props,Init,Trimmed,End} | trim(Confs)] end; trim([TC | Tests]) -> [TC | trim(Tests)]; trim([]) -> []. cyclic_test(Mod, Name, Names) -> case lists:member(Name, Names) of true -> E = "Cyclic reference to group "++atom_to_list(Name)++ " in "++atom_to_list(Mod)++":groups/0", throw({error,list_to_atom(E)}); false -> ok end. expand(Mod, Name, Defs) -> case lists:keysearch(Name, 1, Defs) of {value,Def} -> Def; false -> E = "Invalid group "++atom_to_list(Name)++ " in "++atom_to_list(Mod)++":groups/0", throw({error,list_to_atom(E)}) end. make_all_conf(Dir, Mod, _Props) -> case code:is_loaded(Mod) of false -> code:load_abs(filename:join(Dir,atom_to_list(Mod))); _ -> ok end, make_all_conf(Mod). make_all_conf(Mod) -> case catch apply(Mod, groups, []) of {'EXIT',_} -> {error,{invalid_group_definition,Mod}}; GroupDefs when is_list(GroupDefs) -> case catch find_groups(Mod, all, all, GroupDefs) of {error,_} = Error -> %% this makes test_server call error_in_suite as first %% (and only) test case so we can report Error properly [{?MODULE,error_in_suite,[[Error]]}]; [] -> {error,{invalid_group_spec,Mod}}; ConfTests -> [{conf,Props,Init,all,End} || {conf,Props,Init,_,End} <- delete_subs(ConfTests, ConfTests)] end end. make_conf(Dir, Mod, Name, Props, TestSpec) -> case code:is_loaded(Mod) of false -> code:load_abs(filename:join(Dir,atom_to_list(Mod))); _ -> ok end, make_conf(Mod, Name, Props, TestSpec). make_conf(Mod, Name, Props, TestSpec) -> case code:is_loaded(Mod) of false -> code:load_file(Mod); _ -> ok end, {InitConf,EndConf,ExtraProps} = case erlang:function_exported(Mod,init_per_group,2) of true -> {{Mod,init_per_group},{Mod,end_per_group},[]}; false -> ct_logs:log("TEST INFO", "init_per_group/2 and " "end_per_group/2 missing for group " "~p in ~p, using default.", [Name,Mod]), {{?MODULE,init_per_group}, {?MODULE,end_per_group}, [{suite,Mod}]} end, {conf,[{name,Name}|Props++ExtraProps],InitConf,TestSpec,EndConf}. %%%----------------------------------------------------------------- get_all(Mod, ConfTests) -> case catch apply(Mod, all, []) of {'EXIT',_} -> Reason = list_to_atom(atom_to_list(Mod)++":all/0 is missing"), %% this makes test_server call error_in_suite as first %% (and only) test case so we can report Reason properly [{?MODULE,error_in_suite,[[{error,Reason}]]}]; AllTCs when is_list(AllTCs) -> case catch save_seqs(Mod,AllTCs) of {error,What} -> [{?MODULE,error_in_suite,[[{error,What}]]}]; SeqsAndTCs -> %% expand group references in all() using ConfTests case catch expand_groups(SeqsAndTCs, ConfTests, Mod) of {error,_} = Error -> [{?MODULE,error_in_suite,[[Error]]}]; Tests -> delete_subs(Tests, Tests) end end; Skip = {skip,_Reason} -> Skip; _ -> Reason = list_to_atom("Bad return value from "++atom_to_list(Mod)++":all/0"), [{?MODULE,error_in_suite,[[{error,Reason}]]}] end. expand_groups([H | T], ConfTests, Mod) -> [expand_groups(H, ConfTests, Mod) | expand_groups(T, ConfTests, Mod)]; expand_groups([], _ConfTests, _Mod) -> []; expand_groups({group,Name}, ConfTests, Mod) -> expand_groups({group,Name,default,[]}, ConfTests, Mod); expand_groups({group,Name,default}, ConfTests, Mod) -> expand_groups({group,Name,default,[]}, ConfTests, Mod); expand_groups({group,Name,ORProps}, ConfTests, Mod) when is_list(ORProps) -> expand_groups({group,Name,ORProps,[]}, ConfTests, Mod); expand_groups({group,Name,ORProps,SubORSpec}, ConfTests, Mod) -> FindConf = fun(Conf = {conf,Props,Init,Ts,End}) -> case ?val(name, Props) of Name when ORProps == default -> [Conf]; Name -> [{conf,[{name,Name}|ORProps],Init,Ts,End}]; _ -> [] end end, case lists:flatmap(FindConf, ConfTests) of [] -> throw({error,invalid_ref_msg(Name, Mod)}); Matching when SubORSpec == [] -> Matching; Matching -> override_props(Matching, SubORSpec, Name,Mod) end; expand_groups(SeqOrTC, _ConfTests, _Mod) -> SeqOrTC. %% search deep for the matching conf test and modify it and any %% sub tests according to the override specification search_and_override([Conf = {conf,Props,Init,Tests,End}], ORSpec, Mod) -> Name = ?val(name, Props), case lists:keysearch(Name, 1, ORSpec) of {value,{Name,default}} -> [Conf]; {value,{Name,ORProps}} -> [{conf,[{name,Name}|ORProps],Init,Tests,End}]; {value,{Name,default,[]}} -> [Conf]; {value,{Name,default,SubORSpec}} -> override_props([Conf], SubORSpec, Name,Mod); {value,{Name,ORProps,SubORSpec}} -> override_props([{conf,[{name,Name}|ORProps], Init,Tests,End}], SubORSpec, Name,Mod); _ -> [{conf,Props,Init,search_and_override(Tests,ORSpec,Mod),End}] end. %% Modify the Tests element according to the override specification override_props([{conf,Props,Init,Tests,End} | Confs], SubORSpec, Name,Mod) -> {Subs,SubORSpec1} = override_sub_props(Tests, [], SubORSpec, Mod), [{conf,Props,Init,Subs,End} | override_props(Confs, SubORSpec1, Name,Mod)]; override_props([], [], _,_) -> []; override_props([], SubORSpec, Name,Mod) -> Es = [invalid_ref_msg(Name, element(1,Spec), Mod) || Spec <- SubORSpec], throw({error,Es}). override_sub_props([], New, ORSpec, _) -> {?rev(New),ORSpec}; override_sub_props([T = {conf,Props,Init,Tests,End} | Ts], New, ORSpec, Mod) -> Name = ?val(name, Props), case lists:keysearch(Name, 1, ORSpec) of {value,Spec} -> % group found in spec Props1 = case element(2, Spec) of default -> Props; ORProps -> [{name,Name} | ORProps] end, case catch element(3, Spec) of Undef when Undef == [] ; 'EXIT' == element(1, Undef) -> override_sub_props(Ts, [{conf,Props1,Init,Tests,End} | New], lists:keydelete(Name, 1, ORSpec), Mod); SubORSpec when is_list(SubORSpec) -> case override_sub_props(Tests, [], SubORSpec, Mod) of {Subs,[]} -> override_sub_props(Ts, [{conf,Props1,Init, Subs,End} | New], lists:keydelete(Name, 1, ORSpec), Mod); {_,NonEmptySpec} -> Es = [invalid_ref_msg(Name, element(1, GrRef), Mod) || GrRef <- NonEmptySpec], throw({error,Es}) end; BadGrSpec -> throw({error,{invalid_form,BadGrSpec}}) end; _ -> % not a group in spec override_sub_props(Ts, [T | New], ORSpec, Mod) end; override_sub_props([TC | Ts], New, ORSpec, Mod) -> override_sub_props(Ts, [TC | New], ORSpec, Mod). invalid_ref_msg(Name, Mod) -> E = "Invalid reference to group "++ atom_to_list(Name)++" in "++ atom_to_list(Mod)++":all/0", list_to_atom(E). invalid_ref_msg(Name0, Name1, Mod) -> E = "Invalid reference to group "++ atom_to_list(Name1)++" from "++atom_to_list(Name0)++ " in "++atom_to_list(Mod)++":all/0", list_to_atom(E). %%!============================================================ %%! The support for sequences by means of using sequences/0 %%! will be removed in OTP R15. The code below is only kept %%! for backwards compatibility. From OTP R13 groups with %%! sequence property should be used instead! %%!============================================================ %%!============================================================ %%! START OF DEPRECATED SUPPORT FOR SEQUENCES ---> get_seq(Mod, Func) -> case ct_util:read_suite_data({seq,Mod,Func}) of undefined -> case catch apply(Mod,sequences,[]) of {'EXIT',_} -> []; Seqs -> case lists:keysearch(Func,1,Seqs) of {value,{Func,SeqTCs}} -> case catch save_seq(Mod,Func,SeqTCs) of {error,What} -> [{?MODULE,error_in_suite,[[{error,What}]]}]; _ -> SeqTCs end; false -> [] end end; TCs when is_list(TCs) -> TCs; _ -> [] end. save_seqs(Mod,AllTCs) -> case lists:keymember(sequence,1,AllTCs) of true -> case catch apply(Mod,sequences,[]) of {'EXIT',_} -> Reason = list_to_atom(atom_to_list(Mod)++ ":sequences/0 is missing"), throw({error,Reason}); Seqs -> save_seqs(Mod,AllTCs,Seqs,AllTCs) end; false -> AllTCs end. save_seqs(Mod,[{sequence,Seq}|TCs],Seqs,All) -> case lists:keysearch(Seq,1,Seqs) of {value,{Seq,SeqTCs}} -> save_seq(Mod,Seq,SeqTCs,All), [Seq|save_seqs(Mod,TCs,Seqs,All)]; false -> Reason = list_to_atom( atom_to_list(Seq)++" is missing in "++ atom_to_list(Mod)), throw({error,Reason}) end; save_seqs(Mod,[TC|TCs],Seqs,All) -> [TC|save_seqs(Mod,TCs,Seqs,All)]; save_seqs(_,[],_,_) -> []. save_seq(Mod,Seq,SeqTCs) -> save_seq(Mod,Seq,SeqTCs,apply(Mod,all,[])). save_seq(Mod,Seq,SeqTCs,All) -> check_private(Seq,SeqTCs,All), check_multiple(Mod,Seq,SeqTCs), ct_util:save_suite_data({seq,Mod,Seq},SeqTCs), lists:foreach(fun(TC) -> ct_util:save_suite_data({seq,Mod,TC},Seq) end, SeqTCs). check_private(Seq,TCs,All) -> Bad = lists:filter(fun(TC) -> lists:member(TC,All) end, TCs), if Bad /= [] -> Reason = io_lib:format("regular test cases not allowed in sequence ~p: " "~p",[Seq,Bad]), throw({error,list_to_atom(lists:flatten(Reason))}); true -> ok end. check_multiple(Mod,Seq,TCs) -> Bad = lists:filter(fun(TC) -> case ct_util:read_suite_data({seq,Mod,TC}) of Seq1 when Seq1 /= undefined, Seq1 /= Seq -> true; _ -> false end end,TCs), if Bad /= [] -> Reason = io_lib:format("test cases found in multiple sequences: " "~p",[Bad]), throw({error,list_to_atom(lists:flatten(Reason))}); true -> ok end. %%! <--- END OF DEPRECATED SUPPORT FOR SEQUENCES %%!============================================================ %% let test_server call this function as a testcase only so that %% the user may see info about what's missing in the suite error_in_suite(Config) -> Reason = test_server:lookup_config(error,Config), exit(Reason). %% if init_per_suite and end_per_suite are missing in the suite, %% these will be called instead (without any trace of them in the %% log files), only so that it's possible to call hook functions %% for configuration init_per_suite(Config) -> Config. end_per_suite(_Config) -> ok. %% if the group config functions are missing in the suite, %% use these instead init_per_group(GroupName, Config) -> ct:comment(io_lib:format("start of ~p", [GroupName])), ct_logs:log("TEST INFO", "init_per_group/2 for ~w missing " "in suite, using default.", [GroupName]), Config. end_per_group(GroupName, _) -> ct:comment(io_lib:format("end of ~p", [GroupName])), ct_logs:log("TEST INFO", "end_per_group/2 for ~w missing " "in suite, using default.", [GroupName]), ok. %%%----------------------------------------------------------------- %%% @spec report(What,Data) -> ok report(What,Data) -> case What of loginfo -> %% logfiles and direcories have been created for a test and the %% top level test index page needs to be refreshed TestName = filename:basename(?val(topdir, Data), ".logs"), RunDir = ?val(rundir, Data), ct_logs:make_all_suites_index({TestName,RunDir}), ok; tests_start -> case ct_util:get_testdata(cover) of undefined -> ok; {_CovFile,_CovNodes,CovImport,CovExport,_CovAppData} -> %% Always import cover data from files specified by CovImport %% if no CovExport defined. If CovExport is defined, only %% import from CovImport files initially, then use CovExport %% to pass coverdata between proceeding tests (in the same run). Imps = case CovExport of [] -> % don't export data between tests CovImport; _ -> case filelib:is_file(CovExport) of true -> [CovExport]; false -> CovImport end end, lists:foreach( fun(Imp) -> case cover:import(Imp) of ok -> ok; {error,Reason} -> ct_logs:log("COVER INFO", "Importing cover data from: ~s fails! " "Reason: ~p", [Imp,Reason]) end end, Imps) end; tests_done -> ok; tc_start -> %% Data = {{Suite,Func},LogFileName} ct_event:sync_notify(#event{name=tc_logfile, node=node(), data=Data}), ok; tc_done -> {_Suite,Case,Result} = Data, case Result of {failed, _} -> ct_hooks:on_tc_fail(What, Data); {skipped,{failed,{_,init_per_testcase,_}}} -> ct_hooks:on_tc_skip(tc_auto_skip, Data); {skipped,{require_failed,_}} -> ct_hooks:on_tc_skip(tc_auto_skip, Data); {skipped,_} -> ct_hooks:on_tc_skip(tc_user_skip, Data); _Else -> ok end, case {Case,Result} of {init_per_suite,_} -> ok; {end_per_suite,_} -> ok; {init_per_group,_} -> ok; {end_per_group,_} -> ok; {_,ok} -> add_to_stats(ok); {_,{skipped,{failed,{_,init_per_testcase,_}}}} -> add_to_stats(auto_skipped); {_,{skipped,{require_failed,_}}} -> add_to_stats(auto_skipped); {_,{skipped,{timetrap_error,_}}} -> add_to_stats(auto_skipped); {_,{skipped,{invalid_time_format,_}}} -> add_to_stats(auto_skipped); {_,{skipped,_}} -> add_to_stats(user_skipped); {_,{SkipOrFail,_Reason}} -> add_to_stats(SkipOrFail) end; tc_user_skip -> %% test case specified as skipped in testspec %% Data = {Suite,Case,Comment} ct_event:sync_notify(#event{name=tc_user_skip, node=node(), data=Data}), ct_hooks:on_tc_skip(What, Data), add_to_stats(user_skipped); tc_auto_skip -> %% test case skipped because of error in init_per_suite %% Data = {Suite,Case,Comment} {_Suite,Case,_Result} = Data, %% this test case does not have a log, so printouts %% from event handlers should end up in the main log ct_event:sync_notify(#event{name=tc_auto_skip, node=node(), data=Data}), ct_hooks:on_tc_skip(What, Data), if Case /= end_per_suite, Case /= end_per_group -> add_to_stats(auto_skipped); true -> ok end; framework_error -> case Data of {{M,F},E} -> ct_event:sync_notify(#event{name=tc_done, node=node(), data={M,F,{framework_error,E}}}); _ -> ct_event:sync_notify(#event{name=tc_done, node=node(), data=Data}) end; _ -> ok end, catch vts:report(What,Data). add_to_stats(Result) -> Update = fun({Ok,Failed,Skipped={UserSkipped,AutoSkipped}}) -> Stats = case Result of ok -> {Ok+1,Failed,Skipped}; failed -> {Ok,Failed+1,Skipped}; skipped -> {Ok,Failed,{UserSkipped+1,AutoSkipped}}; user_skipped -> {Ok,Failed,{UserSkipped+1,AutoSkipped}}; auto_skipped -> {Ok,Failed,{UserSkipped,AutoSkipped+1}} end, ct_event:sync_notify(#event{name=test_stats, node=node(), data=Stats}), Stats end, ct_util:update_testdata(stats, Update). %%%----------------------------------------------------------------- %%% @spec warn(What) -> true | false warn(What) when What==nodes; What==processes -> false; warn(_What) -> true. %%%----------------------------------------------------------------- %%% @spec add_data_dir(File0) -> File1 add_data_dir(File,Config) when is_atom(File) -> add_data_dir(atom_to_list(File),Config); add_data_dir(File,Config) when is_list(File) -> case filename:split(File) of [File] -> %% no user path, add data dir case lists:keysearch(data_dir,1,Config) of {value,{data_dir,DataDir}} -> filename:join(DataDir,File); _ -> File end; _ -> File end. %%%----------------------------------------------------------------- %%% @spec get_logopts() -> [LogOpt] get_logopts() -> case ct_util:get_testdata(logopts) of undefined -> []; LogOpts -> LogOpts end. %%%----------------------------------------------------------------- %%% @spec format_comment(Comment) -> HtmlComment format_comment(Comment) -> "" ++ Comment ++ "". %%%----------------------------------------------------------------- %%% @spec get_html_wrapper(TestName, PrintLabel, Cwd) -> Header get_html_wrapper(TestName, PrintLabel, Cwd) -> ct_logs:get_ts_html_wrapper(TestName, PrintLabel, Cwd).