diff options
Diffstat (limited to 'lib/common_test/src/ct_framework.erl')
| -rw-r--r-- | lib/common_test/src/ct_framework.erl | 627 |
1 files changed, 478 insertions, 149 deletions
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 9e597edf38..c24a7c238b 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -24,10 +24,10 @@ -module(ct_framework). --export([init_tc/3, end_tc/3, end_tc/4, get_suite/2, report/2, warn/1]). --export([error_notification/4]). +-export([init_tc/3, end_tc/3, end_tc/4, get_suite/2, get_all_cases/1]). +-export([report/2, warn/1, error_notification/4]). --export([overview_html_header/1]). +-export([get_logopts/0, format_comment/1, get_html_wrapper/3]). -export([error_in_suite/1, ct_init_per_group/2, ct_end_per_group/2]). @@ -36,6 +36,10 @@ -include("ct_event.hrl"). -include("ct_util.hrl"). +-define(val(Key, List), proplists:get_value(Key, List)). +-define(val(Key, List, Def), proplists:get_value(Key, List, Def)). +-define(rev(L), lists:reverse(L)). + %%%----------------------------------------------------------------- %%% @spec init_tc(Mod,Func,Args) -> {ok,NewArgs} | {error,Reason} | %%% {skip,Reason} | {auto_skip,Reason} @@ -48,6 +52,8 @@ %%% @doc Test server framework callback, called by the test_server %%% when a new test case is started. init_tc(Mod,Func,Config) -> + %% in case Mod == ct_framework, lookup the suite name + Suite = get_suite_name(Mod, Config), %% check if previous testcase was interpreted and has left %% a "dead" trace window behind - if so, kill it case ct_util:get_testdata(interpret) of @@ -57,34 +63,36 @@ init_tc(Mod,Func,Config) -> _ -> ok end, - %% check if we need to add defaults explicitly because %% there's no init_per_suite exported from Mod {InitFailed,DoInit} = case ct_util:get_testdata(curr_tc) of - {Mod,{suite0_failed,_}=Failure} -> + {Suite,{suite0_failed,_}=Failure} -> {Failure,false}; - {Mod,_} -> + {?MODULE,_} -> % should not really happen {false,false}; - _ when Func == init_per_suite -> + {Suite,_} -> % Func is not 1st case in suite {false,false}; - _ -> + _ when Func == init_per_suite -> % defaults will be added anyway + {false,false}; + _ -> % first case in suite {false,true} end, case InitFailed of false -> - ct_util:set_testdata({curr_tc,{Mod,Func}}), - case ct_util:read_suite_data({seq,Mod,Func}) of + ct_util:set_testdata({curr_tc,{Suite,Func}}), + case ct_util:read_suite_data({seq,Suite,Func}) of undefined -> init_tc1(Mod,Func,Config,DoInit); Seq when is_atom(Seq) -> - case ct_util:read_suite_data({seq,Mod,Seq}) of + case ct_util:read_suite_data({seq,Suite,Seq}) of [Func|TCs] -> % this is the 1st case in Seq %% make sure no cases in this seq are marked as failed %% from an earlier execution in the same suite - lists:foreach(fun(TC) -> - ct_util:save_suite_data({seq,Mod,TC},Seq) - end, TCs); + lists:foreach( + fun(TC) -> + ct_util:save_suite_data({seq,Suite,TC},Seq) + end, TCs); _ -> ok end, @@ -98,6 +106,17 @@ init_tc(Mod,Func,Config) -> {skip,InitFailed} end. +init_tc1(?MODULE,error_in_suite,[Config0],_) when is_list(Config0) -> + ct_logs:init_tc(false), + ct_event:notify(#event{name=tc_start, + node=node(), + data={?MODULE,error_in_suite}}), + case ?val(error, Config0) of + undefined -> + {skip,"unknown_error_in_suite"}; + Reason -> + {skip,Reason} + end; init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) -> Config1 = case ct_util:read_suite_data(last_saved_config) of @@ -116,41 +135,46 @@ init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) -> Config = lists:keydelete(watchdog,1,Config1), if Func /= init_per_suite, DoInit /= true -> ok; - true -> + true -> %% delete all default values used in previous suite ct_config:delete_default_config(suite), %% release all name -> key bindings (once per suite) ct_config:release_allocated() end, - TestCaseInfo = - case catch apply(Mod,Func,[]) of - Result when is_list(Result) -> Result; - _ -> [] - end, + + GroupPath = ?val(tc_group_path, Config, []), + AllGroups = [?val(tc_group_properties, Config, []) | GroupPath], + %% clear all config data default values set by previous %% testcase info function (these should only survive the %% testcase, not the whole suite) - ct_config:delete_default_config(testcase), - case add_defaults(Mod,Func,TestCaseInfo,DoInit) of + FuncSpec = group_or_func(Func,Config0), + if is_tuple(FuncSpec) -> % group + ok; + true -> + ct_config:delete_default_config(testcase) + end, + %% in case Mod == ct_framework, lookup the suite name + Suite = get_suite_name(Mod, Config), + case add_defaults(Mod,Func,AllGroups,DoInit) of Error = {suite0_failed,_} -> - ct_logs:init_tc(), - FuncSpec = group_or_func(Func,Config0), + ct_logs:init_tc(false), ct_event:notify(#event{name=tc_start, node=node(), data={Mod,FuncSpec}}), - ct_util:set_testdata({curr_tc,{Mod,Error}}), + ct_util:set_testdata({curr_tc,{Suite,Error}}), {error,Error}; {SuiteInfo,MergeResult} -> case MergeResult of {error,Reason} when DoInit == false -> - ct_logs:init_tc(), - FuncSpec = group_or_func(Func,Config0), + ct_logs:init_tc(false), ct_event:notify(#event{name=tc_start, node=node(), data={Mod,FuncSpec}}), {skip,Reason}; _ -> - init_tc2(Mod,Func,SuiteInfo,MergeResult,Config,DoInit) + init_tc2(Mod,Func,SuiteInfo,MergeResult, + Config,DoInit) end end; init_tc1(_Mod,_Func,Args,_DoInit) -> @@ -194,20 +218,26 @@ init_tc2(Mod,Func,SuiteInfo,MergeResult,Config,DoInit) -> Conns -> ct_util:silence_connections(Conns) end, - - ct_logs:init_tc(), + if Func /= init_per_suite, DoInit /= true -> + ct_logs:init_tc(false); + true -> + ct_logs:init_tc(true) + end, FuncSpec = group_or_func(Func,Config), ct_event:notify(#event{name=tc_start, node=node(), data={Mod,FuncSpec}}), - - case configure(MergedInfo1,MergedInfo1,SuiteInfo,{Func,DoInit},Config) of + + case catch configure(MergedInfo1,MergedInfo1,SuiteInfo, + {FuncSpec,DoInit},Config) of {suite0_failed,Reason} -> ct_util:set_testdata({curr_tc,{Mod,{suite0_failed,{require,Reason}}}}), {skip,{require_failed_in_suite0,Reason}}; {error,Reason} -> {auto_skip,{require_failed,Reason}}; - {ok, FinalConfig} -> + {'EXIT',Reason} -> + {auto_skip,Reason}; + {ok,FinalConfig} -> case MergeResult of {error,Reason} -> %% suite0 configure finished now, report that @@ -236,82 +266,219 @@ ct_suite_init(Mod, Func, [Config]) when is_list(Config) -> Else end. -add_defaults(Mod,Func,FuncInfo,DoInit) -> - case (catch Mod:suite()) of +add_defaults(Mod,Func, GroupPath, DoInit) -> + Suite = get_suite_name(Mod, GroupPath), + case (catch Suite:suite()) of {'EXIT',{undef,_}} -> - SuiteInfo = merge_with_suite_defaults(Mod,[]), + SuiteInfo = merge_with_suite_defaults(Suite,[]), SuiteInfoNoCTH = [I || I <- SuiteInfo, element(1,I) =/= ct_hooks], - case add_defaults1(Mod,Func,FuncInfo,SuiteInfoNoCTH,DoInit) of + case add_defaults1(Mod,Func, GroupPath, SuiteInfoNoCTH, DoInit) of Error = {error,_} -> {SuiteInfo,Error}; MergedInfo -> {SuiteInfo,MergedInfo} end; - {'EXIT',Reason} -> + {'EXIT',Reason} -> + ErrStr = io_lib:format("~n*** ERROR *** " + "~w:suite/0 failed: ~p~n", + [Suite,Reason]), + io:format(ErrStr, []), + io:format(user, ErrStr, []), {suite0_failed,{exited,Reason}}; SuiteInfo when is_list(SuiteInfo) -> case lists:all(fun(E) when is_tuple(E) -> true; (_) -> false end, SuiteInfo) of true -> - SuiteInfo1 = merge_with_suite_defaults(Mod,SuiteInfo), + SuiteInfo1 = merge_with_suite_defaults(Suite, SuiteInfo), SuiteInfoNoCTH = [I || I <- SuiteInfo1, element(1,I) =/= ct_hooks], - case add_defaults1(Mod,Func,FuncInfo, - SuiteInfoNoCTH,DoInit) of + case add_defaults1(Mod,Func, GroupPath, + SuiteInfoNoCTH, DoInit) of Error = {error,_} -> {SuiteInfo1,Error}; MergedInfo -> {SuiteInfo1,MergedInfo} end; false -> + ErrStr = io_lib:format("~n*** ERROR *** " + "Invalid return value from " + "~w:suite/0: ~p~n", [Suite,SuiteInfo]), + io:format(ErrStr, []), + io:format(user, ErrStr, []), {suite0_failed,bad_return_value} end; - _ -> + SuiteInfo -> + ErrStr = io_lib:format("~n*** ERROR *** " + "Invalid return value from " + "~w:suite/0: ~p~n", [Suite,SuiteInfo]), + io:format(ErrStr, []), + io:format(user, ErrStr, []), {suite0_failed,bad_return_value} end. -add_defaults1(_Mod,init_per_suite,[],SuiteInfo,_DoInit) -> - SuiteInfo; - -add_defaults1(Mod,Func,FuncInfo,SuiteInfo,DoInit) -> - %% mustn't re-require suite variables in test case info function, - %% can result in weird behaviour (suite values get overwritten) +add_defaults1(Mod,Func, GroupPath, SuiteInfo, DoInit) -> + Suite = get_suite_name(Mod, GroupPath), + %% GroupPathInfo (for subgroup on level X) = + %% [LevelXGroupInfo, LevelX-1GroupInfo, ..., TopLevelGroupInfo] + GroupPathInfo = + lists:map(fun(GroupProps) -> + Name = ?val(name, GroupProps), + case catch Suite:group(Name) of + GrInfo when is_list(GrInfo) -> GrInfo; + _ -> [] + end + end, GroupPath), + Args = if Func == init_per_group; Func == ct_init_per_group; + Func == end_per_group; Func == ct_end_per_group -> + [?val(name, hd(GroupPath))]; + true -> + [] + end, + TestCaseInfo = + case catch apply(Mod,Func,Args) of + TCInfo when is_list(TCInfo) -> TCInfo; + _ -> [] + end, + %% let test case info (also for all config funcs) override group info, + %% and lower level group info override higher level info + TCAndGroupInfo = [TestCaseInfo | remove_info_in_prev(TestCaseInfo, + GroupPathInfo)], + %% find and save require terms found in suite info SuiteReqs = [SDDef || SDDef <- SuiteInfo, ((require == element(1,SDDef)) or (default_config == element(1,SDDef)))], - FuncReqs = - [FIDef || FIDef <- FuncInfo, - require == element(1,FIDef)], - case [element(2,Clash) || Clash <- SuiteReqs, - require == element(1, Clash), - true == lists:keymember(element(2,Clash),2, - FuncReqs)] of + case check_for_clashes(TestCaseInfo, GroupPathInfo, SuiteReqs) of [] -> - add_defaults2(Mod,Func,FuncInfo,SuiteInfo,SuiteReqs,DoInit); + add_defaults2(Mod,Func, TCAndGroupInfo,SuiteInfo,SuiteReqs, DoInit); Clashes -> {error,{config_name_already_in_use,Clashes}} end. -add_defaults2(Mod,init_per_suite,IPSInfo,SuiteInfo,SuiteReqs,false) -> - %% not common practise to use a test case info function for - %% init_per_suite (usually handled by suite/0), but let's support - %% it just in case... - add_defaults2(Mod,init_per_suite,IPSInfo,SuiteInfo,SuiteReqs,true); - -add_defaults2(_Mod,_Func,FuncInfo,SuiteInfo,_,false) -> - %% include require elements from test case info, but not from suite/0 - %% (since we've already required those vars) - FuncInfo ++ - [SFDef || SFDef <- SuiteInfo, - require /= element(1,SFDef), - false == lists:keymember(element(1,SFDef),1,FuncInfo)]; - -add_defaults2(_Mod,_Func,FuncInfo,SuiteInfo,SuiteReqs,true) -> - %% We must include require elements from suite/0 here since - %% there's no init_per_suite call before this first test case. - %% Let other test case info elements override those from suite/0. - FuncInfo ++ SuiteReqs ++ - [SDDef || SDDef <- SuiteInfo, - require /= element(1,SDDef), - false == lists:keymember(element(1,SDDef),1,FuncInfo)]. +get_suite_name(?MODULE, [Cfg|_]) when is_list(Cfg), Cfg /= [] -> + get_suite_name(?MODULE, Cfg); + +get_suite_name(?MODULE, Cfg) when is_list(Cfg), Cfg /= [] -> + case ?val(tc_group_properties, Cfg) of + undefined -> + case ?val(suite, Cfg) of + undefined -> ?MODULE; + Suite -> Suite + end; + GrProps -> + case ?val(suite, GrProps) of + undefined -> ?MODULE; + Suite -> Suite + end + end; +get_suite_name(Mod, _) -> + Mod. + +%% Check that alias names are not already in use +check_for_clashes(TCInfo, GrPathInfo, SuiteInfo) -> + {CurrGrInfo,SearchIn} = case GrPathInfo of + [] -> {[],[SuiteInfo]}; + [Curr|Path] -> {Curr,[SuiteInfo|Path]} + end, + ReqNames = fun(Info) -> [element(2,R) || R <- Info, + size(R) == 3, + require == element(1,R)] + end, + ExistingNames = lists:flatten([ReqNames(L) || L <- SearchIn]), + CurrGrReqNs = ReqNames(CurrGrInfo), + GrClashes = [Name || Name <- CurrGrReqNs, + true == lists:member(Name, ExistingNames)], + AllReqNs = CurrGrReqNs ++ ExistingNames, + TCClashes = [Name || Name <- ReqNames(TCInfo), + true == lists:member(Name, AllReqNs)], + TCClashes ++ GrClashes. + +%% Delete the info terms in Terms from all following info lists +remove_info_in_prev(Terms, [[] | Rest]) -> + [[] | remove_info_in_prev(Terms, Rest)]; +remove_info_in_prev(Terms, [Info | Rest]) -> + UniqueInInfo = [U || U <- Info, + ((timetrap == element(1,U)) and + (not lists:keymember(timetrap,1,Terms))) or + ((require == element(1,U)) and + (not lists:member(U,Terms))) or + ((default_config == element(1,U)) and + (not keysmember([default_config,1, + element(2,U),2], Terms)))], + OtherTermsInInfo = [T || T <- Info, + timetrap /= element(1,T), + require /= element(1,T), + default_config /= element(1,T), + false == lists:keymember(element(1,T),1, + Terms)], + KeptInfo = UniqueInInfo ++ OtherTermsInInfo, + [KeptInfo | remove_info_in_prev(Terms ++ KeptInfo, Rest)]; +remove_info_in_prev(_, []) -> + []. + +keysmember([Key,Pos|Next], List) -> + case [Elem || Elem <- List, Key == element(Pos,Elem)] of + [] -> false; + Found -> keysmember(Next, Found) + end; +keysmember([], _) -> true. + + +add_defaults2(Mod,init_per_suite, IPSInfo, SuiteInfo,SuiteReqs, false) -> + add_defaults2(Mod,init_per_suite, IPSInfo, SuiteInfo,SuiteReqs, true); + +add_defaults2(_Mod,IPG, IPGAndGroupInfo, SuiteInfo,SuiteReqs, DoInit) when + IPG == init_per_group ; IPG == ct_init_per_group -> + %% If DoInit == true, we have to process the suite() list, otherwise + %% it has already been handled (see clause for init_per_suite) + case DoInit of + true -> + %% note: we know for sure this is a top level group + Info = lists:flatten([IPGAndGroupInfo, SuiteReqs]), + Info ++ remove_info_in_prev(Info, [SuiteInfo]); + false -> + SuiteInfo1 = + remove_info_in_prev(lists:flatten([IPGAndGroupInfo, + SuiteReqs]), [SuiteInfo]), + %% don't require terms in prev groups (already processed) + case IPGAndGroupInfo of + [IPGInfo] -> + lists:flatten([IPGInfo,SuiteInfo1]); + [IPGInfo | [CurrGroupInfo | PrevGroupInfo]] -> + PrevGroupInfo1 = delete_require_terms(PrevGroupInfo), + lists:flatten([IPGInfo,CurrGroupInfo,PrevGroupInfo1, + SuiteInfo1]) + end + end; + +add_defaults2(_Mod,_Func, TCAndGroupInfo, SuiteInfo,SuiteReqs, false) -> + %% Include require elements from test case info and current group, + %% but not from previous groups or suite/0 (since we've already required + %% those vars). Let test case info elements override group and suite + %% info elements. + SuiteInfo1 = remove_info_in_prev(lists:flatten([TCAndGroupInfo, + SuiteReqs]), [SuiteInfo]), + %% don't require terms in prev groups (already processed) + case TCAndGroupInfo of + [TCInfo] -> + lists:flatten([TCInfo,SuiteInfo1]); + [TCInfo | [CurrGroupInfo | PrevGroupInfo]] -> + PrevGroupInfo1 = delete_require_terms(PrevGroupInfo), + lists:flatten([TCInfo,CurrGroupInfo,PrevGroupInfo1, + SuiteInfo1]) + end; + +add_defaults2(_Mod,_Func, TCInfo, SuiteInfo,SuiteReqs, true) -> + %% Here we have to process the suite info list also (no call to + %% init_per_suite before this first test case). This TC can't belong + %% to a group, or the clause for (ct_)init_per_group would've caught this. + Info = lists:flatten([TCInfo, SuiteReqs]), + lists:flatten([Info,remove_info_in_prev(Info, [SuiteInfo])]). + +delete_require_terms([Info | Prev]) -> + Info1 = [T || T <- Info, + require /= element(1,T), + default_config /= element(1,T)], + [Info1 | delete_require_terms(Prev)]; +delete_require_terms([]) -> + []. merge_with_suite_defaults(Mod,SuiteInfo) -> case lists:keysearch(suite_defaults,1,Mod:module_info(attributes)) of @@ -335,16 +502,17 @@ timetrap_first([Trap = {timetrap,_} | Rest],Info,Found) -> timetrap_first([Other | Rest],Info,Found) -> timetrap_first(Rest,[Other | Info],Found); timetrap_first([],Info,[]) -> - [{timetrap,{minutes,30}} | lists:reverse(Info)]; + [{timetrap,{minutes,30}} | ?rev(Info)]; timetrap_first([],Info,Found) -> - lists:reverse(Found) ++ lists:reverse(Info). + ?rev(Found) ++ ?rev(Info). configure([{require,Required}|Rest],Info,SuiteInfo,Scope,Config) -> case ct:require(Required) of ok -> configure(Rest,Info,SuiteInfo,Scope,Config); Error = {error,Reason} -> - case required_default('_UNDEF',Required,Info,SuiteInfo,Scope) of + case required_default('_UNDEF',Required,Info, + SuiteInfo,Scope) of ok -> configure(Rest,Info,SuiteInfo,Scope,Config); _ -> @@ -386,18 +554,24 @@ configure([],_,_,_,Config) -> {ok,[Config]}. %% the require element in Info may come from suite/0 and -%% should be scoped 'suite', or come from the testcase info -%% function and should then be scoped 'testcase' -required_default(Name,Key,Info,SuiteInfo,{Func,true}) -> +%% should be scoped 'suite', or come from the group info +%% function and be scoped 'group', or come from the testcase +%% info function and then be scoped 'testcase' + +required_default(Name,Key,Info,SuiteInfo,{FuncSpec,true}) -> case try_set_default(Name,Key,SuiteInfo,suite) of ok -> ok; _ -> - required_default(Name,Key,Info,[],{Func,false}) + required_default(Name,Key,Info,[],{FuncSpec,false}) end; required_default(Name,Key,Info,_,{init_per_suite,_}) -> try_set_default(Name,Key,Info,suite); -required_default(Name,Key,Info,_,_) -> +required_default(Name,Key,Info,_,{{init_per_group,GrName,_},_}) -> + try_set_default(Name,Key,Info,{group,GrName}); +required_default(Name,Key,Info,_,{{ct_init_per_group,GrName,_},_}) -> + try_set_default(Name,Key,Info,{group,GrName}); +required_default(Name,Key,Info,_,_FuncSpec) -> try_set_default(Name,Key,Info,testcase). try_set_default(Name,Key,Info,Where) -> @@ -437,7 +611,7 @@ try_set_default(Name,Key,Info,Where) -> %%% @doc Test server framework callback, called by the test_server %%% when a test case is finished. end_tc(Mod, Fun, Args) -> - %% Have to keep end_tc/3 for backwards compatabilty issues + %% Have to keep end_tc/3 for backwards compatibility issues end_tc(Mod, Fun, Args, '$end_tc_dummy'). end_tc(?MODULE,error_in_suite,_, _) -> % bad start! ok; @@ -451,7 +625,6 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) -> {value,{watchdog,Dog}} -> test_server:timetrap_cancel(Dog); false -> ok end, - %% save the testcase process pid so that it can be used %% to look up the attached trace window later case ct_util:get_testdata(interpret) of @@ -461,12 +634,15 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) -> _ -> ok end, - ct_util:delete_testdata(comment), ct_util:delete_suite_data(last_saved_config), FuncSpec = case group_or_func(Func,Args) of - {_,GroupName,_Props} = Group -> + {_,GroupName,_Props} = Group -> + if Func == end_per_group; Func == ct_end_per_group -> + ct_config:delete_default_config({group,GroupName}); + true -> ok + end, case lists:keysearch(save_config,1,Args) of {value,{save_config,SaveConfig}} -> ct_util:save_suite_data( @@ -685,12 +861,14 @@ mark_as_failed1(_,_,_,[]) -> ok. group_or_func(Func, Config) when Func == init_per_group; - Func == end_per_group -> - case proplists:get_value(tc_group_properties,Config) of + Func == end_per_group; + Func == ct_init_per_group; + Func == ct_end_per_group -> + case ?val(tc_group_properties, Config) of undefined -> {Func,unknown,[]}; GrProps -> - GrName = proplists:get_value(name,GrProps), + GrName = ?val(name,GrProps), {Func,GrName,proplists:delete(name,GrProps)} end; group_or_func(Func, _Config) -> @@ -714,7 +892,7 @@ get_suite(Mod, all) -> %% (and only) test case so we can report Error properly [{?MODULE,error_in_suite,[[Error]]}]; ConfTests -> - get_all(Mod, ConfTests) + get_all(Mod, ConfTests) end; _ -> E = "Bad return value from "++atom_to_list(Mod)++":groups/0", @@ -728,7 +906,7 @@ get_suite(Mod, all) -> %% group get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) -> - Name = proplists:get_value(name, Props), + Name = ?val(name, Props), case catch apply(Mod, groups, []) of {'EXIT',_} -> [Group]; @@ -746,14 +924,25 @@ get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) -> %% a *subgroup* specified *only* as skipped (and not %% as an explicit test) should not be returned, or %% init/end functions for top groups will be executed - case catch proplists:get_value(name, element(2, hd(ConfTests))) of + case catch ?val(name, element(2, hd(ConfTests))) of Name -> % top group delete_subs(ConfTests, ConfTests); _ -> [] end; false -> - delete_subs(ConfTests, ConfTests) + ConfTests1 = delete_subs(ConfTests, ConfTests), + case ?val(override, Props) of + undefined -> + ConfTests1; + [] -> + ConfTests1; + ORSpec -> + ORSpec1 = if is_tuple(ORSpec) -> [ORSpec]; + true -> ORSpec end, + search_and_override(ConfTests1, + ORSpec1, Mod) + end end end; _ -> @@ -767,6 +956,36 @@ get_suite(Mod, Name) -> %%%----------------------------------------------------------------- +get_all_cases(Suite) -> + case get_suite(Suite, all) of + [{?MODULE,error_in_suite,[[{error,_}=Error]]}] -> + Error; + [{?MODULE,error_in_suite,[[Error]]}] -> + {error,Error}; + Tests -> + Cases = get_all_cases1(Suite, Tests), + ?rev(lists:foldl(fun(TC, TCs) -> + case lists:member(TC, TCs) of + true -> TCs; + false -> [TC | TCs] + end + end, [], Cases)) + end. + +get_all_cases1(Suite, [{conf,_Props,_Init,GrTests,_End} | Tests]) -> + get_all_cases1(Suite, GrTests) ++ get_all_cases1(Suite, Tests); + +get_all_cases1(Suite, [Test | Tests]) when is_atom(Test) -> + [{Suite,Test} | get_all_cases1(Suite, Tests)]; + +get_all_cases1(Suite, [Test | Tests]) -> + [Test | get_all_cases1(Suite, Tests)]; + +get_all_cases1(_, []) -> + []. + +%%%----------------------------------------------------------------- + find_groups(Mod, Name, TCs, GroupDefs) -> Found = find(Mod, Name, TCs, GroupDefs, [], GroupDefs, false), trim(Found). @@ -869,14 +1088,14 @@ delete_subs([], All) -> All. delete_conf({conf,Props,_,_,_}, Confs) -> - Name = proplists:get_value(name, Props), + Name = ?val(name, Props), [Conf || Conf = {conf,Props0,_,_,_} <- Confs, - Name =/= proplists:get_value(name, Props0)]. + Name =/= ?val(name, Props0)]. is_sub({conf,Props,_,_,_}=Conf, [{conf,_,_,Tests,_} | Confs]) -> - Name = proplists:get_value(name, Props), + Name = ?val(name, Props), case lists:any(fun({conf,Props0,_,_,_}) -> - case proplists:get_value(name, Props0) of + case ?val(name, Props0) of N when N == Name -> true; _ -> @@ -978,15 +1197,20 @@ make_conf(Mod, Name, Props, TestSpec) -> _ -> ok end, - {InitConf,EndConf} = + {InitConf,EndConf,ExtraProps} = case erlang:function_exported(Mod,init_per_group,2) of true -> - {{Mod,init_per_group},{Mod,end_per_group}}; + {{Mod,init_per_group},{Mod,end_per_group},[]}; false -> + ct_logs:log("TEST INFO", "init_per_group/2 and " + "end_per_group/2 missing for group " + "~p in ~p, using default.", + [Name,Mod]), {{?MODULE,ct_init_per_group}, - {?MODULE,ct_end_per_group}} + {?MODULE,ct_end_per_group}, + [{suite,Mod}]} end, - {conf,[{name,Name}|Props],InitConf,TestSpec,EndConf}. + {conf,[{name,Name}|Props++ExtraProps],InitConf,TestSpec,EndConf}. %%%----------------------------------------------------------------- @@ -1024,29 +1248,116 @@ expand_groups([H | T], ConfTests, Mod) -> expand_groups([], _ConfTests, _Mod) -> []; expand_groups({group,Name}, ConfTests, Mod) -> - FindConf = - fun({conf,Props,_,_,_}) -> - case proplists:get_value(name, Props) of - Name -> true; - _ -> false + expand_groups({group,Name,default,[]}, ConfTests, Mod); +expand_groups({group,Name,default}, ConfTests, Mod) -> + expand_groups({group,Name,default,[]}, ConfTests, Mod); +expand_groups({group,Name,ORProps}, ConfTests, Mod) when is_list(ORProps) -> + expand_groups({group,Name,ORProps,[]}, ConfTests, Mod); +expand_groups({group,Name,ORProps,SubORSpec}, ConfTests, Mod) -> + FindConf = + fun(Conf = {conf,Props,Init,Ts,End}) -> + case ?val(name, Props) of + Name when ORProps == default -> + [Conf]; + Name -> + [{conf,[{name,Name}|ORProps],Init,Ts,End}]; + _ -> + [] end end, - case lists:filter(FindConf, ConfTests) of - [ConfTest|_] -> - expand_groups(ConfTest, ConfTests, Mod); + case lists:flatmap(FindConf, ConfTests) of [] -> - E = "Invalid reference to group "++ - atom_to_list(Name)++" in "++ - atom_to_list(Mod)++":all/0", - throw({error,list_to_atom(E)}) + throw({error,invalid_ref_msg(Name, Mod)}); + Matching when SubORSpec == [] -> + Matching; + Matching -> + override_props(Matching, SubORSpec, Name,Mod) end; expand_groups(SeqOrTC, _ConfTests, _Mod) -> SeqOrTC. +%% search deep for the matching conf test and modify it and any +%% sub tests according to the override specification +search_and_override([Conf = {conf,Props,Init,Tests,End}], ORSpec, Mod) -> + Name = ?val(name, Props), + case lists:keysearch(Name, 1, ORSpec) of + {value,{Name,default}} -> + [Conf]; + {value,{Name,ORProps}} -> + [{conf,[{name,Name}|ORProps],Init,Tests,End}]; + {value,{Name,default,[]}} -> + [Conf]; + {value,{Name,default,SubORSpec}} -> + override_props([Conf], SubORSpec, Name,Mod); + {value,{Name,ORProps,SubORSpec}} -> + override_props([{conf,[{name,Name}|ORProps], + Init,Tests,End}], SubORSpec, Name,Mod); + _ -> + [{conf,Props,Init,search_and_override(Tests,ORSpec,Mod),End}] + end. + +%% Modify the Tests element according to the override specification +override_props([{conf,Props,Init,Tests,End} | Confs], SubORSpec, Name,Mod) -> + {Subs,SubORSpec1} = override_sub_props(Tests, [], SubORSpec, Mod), + [{conf,Props,Init,Subs,End} | override_props(Confs, SubORSpec1, Name,Mod)]; +override_props([], [], _,_) -> + []; +override_props([], SubORSpec, Name,Mod) -> + Es = [invalid_ref_msg(Name, element(1,Spec), Mod) || Spec <- SubORSpec], + throw({error,Es}). + +override_sub_props([], New, ORSpec, _) -> + {?rev(New),ORSpec}; +override_sub_props([T = {conf,Props,Init,Tests,End} | Ts], + New, ORSpec, Mod) -> + Name = ?val(name, Props), + case lists:keysearch(Name, 1, ORSpec) of + {value,Spec} -> % group found in spec + Props1 = + case element(2, Spec) of + default -> Props; + ORProps -> [{name,Name} | ORProps] + end, + case catch element(3, Spec) of + Undef when Undef == [] ; 'EXIT' == element(1, Undef) -> + override_sub_props(Ts, [{conf,Props1,Init,Tests,End} | New], + lists:keydelete(Name, 1, ORSpec), Mod); + SubORSpec when is_list(SubORSpec) -> + case override_sub_props(Tests, [], SubORSpec, Mod) of + {Subs,[]} -> + override_sub_props(Ts, [{conf,Props1,Init, + Subs,End} | New], + lists:keydelete(Name, 1, ORSpec), + Mod); + {_,NonEmptySpec} -> + Es = [invalid_ref_msg(Name, element(1, GrRef), + Mod) || GrRef <- NonEmptySpec], + throw({error,Es}) + end; + BadGrSpec -> + throw({error,{invalid_form,BadGrSpec}}) + end; + _ -> % not a group in spec + override_sub_props(Ts, [T | New], ORSpec, Mod) + end; +override_sub_props([TC | Ts], New, ORSpec, Mod) -> + override_sub_props(Ts, [TC | New], ORSpec, Mod). + +invalid_ref_msg(Name, Mod) -> + E = "Invalid reference to group "++ + atom_to_list(Name)++" in "++ + atom_to_list(Mod)++":all/0", + list_to_atom(E). + +invalid_ref_msg(Name0, Name1, Mod) -> + E = "Invalid reference to group "++ + atom_to_list(Name1)++" from "++atom_to_list(Name0)++ + " in "++atom_to_list(Mod)++":all/0", + list_to_atom(E). %%!============================================================ %%! The support for sequences by means of using sequences/0 -%%! will be removed in OTP R14. The code below is only kept +%%! will be removed in OTP R15. The code below is only kept %%! for backwards compatibility. From OTP R13 groups with %%! sequence property should be used instead! %%!============================================================ @@ -1159,13 +1470,15 @@ error_in_suite(Config) -> %% if the group config functions are missing in the suite, %% use these instead ct_init_per_group(GroupName, Config) -> - ct_logs:log("WARNING", "init_per_group/2 for ~w missing " + ct:comment(io_lib:format("start of ~p", [GroupName])), + ct_logs:log("TEST INFO", "init_per_group/2 for ~w missing " "in suite, using default.", [GroupName]), Config. ct_end_per_group(GroupName, _) -> - ct_logs:log("WARNING", "end_per_group/2 for ~w missing " + ct:comment(io_lib:format("end of ~p", [GroupName])), + ct_logs:log("TEST INFO", "end_per_group/2 for ~w missing " "in suite, using default.", [GroupName]), ok. @@ -1178,8 +1491,8 @@ report(What,Data) -> loginfo -> %% logfiles and direcories have been created for a test and the %% top level test index page needs to be refreshed - TestName = filename:basename(proplists:get_value(topdir, Data), ".logs"), - RunDir = proplists:get_value(rundir, Data), + TestName = filename:basename(?val(topdir, Data), ".logs"), + RunDir = ?val(rundir, Data), ct_logs:make_all_suites_index({TestName,RunDir}), ok; tests_start -> @@ -1218,6 +1531,10 @@ report(What,Data) -> tests_done -> ok; tc_start -> + %% Data = {{Suite,Func},LogFileName} + ct_event:sync_notify(#event{name=tc_logfile, + node=node(), + data=Data}), ok; tc_done -> {_Suite,Case,Result} = Data, @@ -1242,12 +1559,20 @@ report(What,Data) -> ok; {end_per_group,_} -> ok; + {ct_init_per_group,_} -> + ok; + {ct_end_per_group,_} -> + ok; {_,ok} -> add_to_stats(ok); {_,{skipped,{failed,{_,init_per_testcase,_}}}} -> add_to_stats(auto_skipped); {_,{skipped,{require_failed,_}}} -> add_to_stats(auto_skipped); + {_,{skipped,{timetrap_error,_}}} -> + add_to_stats(auto_skipped); + {_,{skipped,{invalid_time_format,_}}} -> + add_to_stats(auto_skipped); {_,{skipped,_}} -> add_to_stats(user_skipped); {_,{SkipOrFail,_Reason}} -> @@ -1273,11 +1598,24 @@ report(What,Data) -> node=node(), data=Data}), ct_hooks:on_tc_skip(What, Data), - if Case /= end_per_suite, Case /= end_per_group -> + if Case /= end_per_suite, + Case /= end_per_group, + Case /= ct_end_per_group -> add_to_stats(auto_skipped); true -> ok end; + framework_error -> + case Data of + {{M,F},E} -> + ct_event:sync_notify(#event{name=tc_done, + node=node(), + data={M,F,{framework_error,E}}}); + _ -> + ct_event:sync_notify(#event{name=tc_done, + node=node(), + data=Data}) + end; _ -> ok end, @@ -1332,30 +1670,21 @@ add_data_dir(File,Config) when is_list(File) -> end. %%%----------------------------------------------------------------- -%%% @spec overview_html_header(TestName) -> Header -overview_html_header(TestName) -> - TestName1 = lists:flatten(io_lib:format("~p", [TestName])), - Label = case application:get_env(common_test, test_label) of - {ok,Lbl} when Lbl =/= undefined -> - "<H1><FONT color=\"green\">" ++ Lbl ++ "</FONT></H1>\n"; - _ -> - "" - end, - Bgr = case ct_logs:basic_html() of - true -> - ""; - false -> - CTPath = code:lib_dir(common_test), - TileFile = filename:join(filename:join(CTPath,"priv"),"tile1.jpg"), - " background=\"" ++ TileFile ++ "\"" - end, - - ["<html>\n", - "<head><title>Test ", TestName1, " results</title>\n", - "<meta http-equiv=\"cache-control\" content=\"no-cache\">\n", - "</head>\n", - "<body", Bgr, " bgcolor=\"white\" text=\"black\" ", - "link=\"blue\" vlink=\"purple\" alink=\"red\">\n", - Label, - "<H2>Results from test ", TestName1, "</H2>\n"]. +%%% @spec get_logopts() -> [LogOpt] +get_logopts() -> + case ct_util:get_testdata(logopts) of + undefined -> + []; + LogOpts -> + LogOpts + end. +%%%----------------------------------------------------------------- +%%% @spec format_comment(Comment) -> HtmlComment +format_comment(Comment) -> + "<font color=\"green\">" ++ Comment ++ "</font>". + +%%%----------------------------------------------------------------- +%%% @spec get_html_wrapper(TestName, PrintLabel, Cwd) -> Header +get_html_wrapper(TestName, PrintLabel, Cwd) -> + ct_logs:get_ts_html_wrapper(TestName, PrintLabel, Cwd). |