From 15214a3d1918f4388d2aa9f12a3fd7005da2b65d Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Tue, 15 Nov 2011 22:58:24 +0100 Subject: Implement the group info function OTP-9235 --- lib/common_test/src/ct.erl | 36 ++- lib/common_test/src/ct_framework.erl | 284 +++++++++++++++------ lib/common_test/test/ct_config_SUITE.erl | 2 +- lib/common_test/test/ct_error_SUITE.erl | 15 +- .../error/test/cfg_error_11_SUITE.erl | 12 +- lib/common_test/test/ct_groups_test_2_SUITE.erl | 8 +- lib/common_test/test/ct_test_server_if_1_SUITE.erl | 16 +- 7 files changed, 264 insertions(+), 109 deletions(-) (limited to 'lib/common_test') diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index d72b8bc0e1..6fd7693185 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -65,7 +65,7 @@ pal/1, pal/2, pal/3, fail/1, fail/2, comment/1, comment/2, testcases/2, userdata/2, userdata/3, - timetrap/1, sleep/1]). + timetrap/1, get_timetrap_info/0, sleep/1]). %% New API for manipulating with config handlers -export([add_config/2, remove_config/2]). @@ -703,7 +703,8 @@ userdata(TestDir, Suite) -> get_userdata(Info, "suite/0") end. -get_userdata({'EXIT',{undef,_}}, Spec) -> +get_userdata({'EXIT',{Undef,_}}, Spec) when Undef == undef; + Undef == function_clause -> {error,list_to_atom(Spec ++ " is not defined")}; get_userdata({'EXIT',Reason}, Spec) -> {error,{list_to_atom("error in " ++ Spec),Reason}}; @@ -719,16 +720,27 @@ get_userdata(_BadTerm, Spec) -> {error,list_to_atom(Spec ++ " must return a list")}. %%%----------------------------------------------------------------- -%%% @spec userdata(TestDir, Suite, Case) -> TCUserData | {error,Reason} +%%% @spec userdata(TestDir, Suite, GroupOrCase) -> TCUserData | {error,Reason} %%% TestDir = string() %%% Suite = atom() -%%% Case = atom() +%%% GroupOrCase = {group,GroupName} | atom() +%%% GroupName = atom() %%% TCUserData = [term()] %%% Reason = term() %%% %%% @doc Returns any data specified with the tag userdata -%%% in the list of tuples returned from Suite:Case/0. -userdata(TestDir, Suite, Case) -> +%%% in the list of tuples returned from Suite:group(GroupName) +%%% or Suite:Case(). +userdata(TestDir, Suite, {group,GroupName}) -> + case make_and_load(TestDir, Suite) of + E = {error,_} -> + E; + _ -> + Info = (catch apply(Suite, group, [GroupName])), + get_userdata(Info, "group("++atom_to_list(GroupName)++")") + end; + +userdata(TestDir, Suite, Case) when is_atom(Case) -> case make_and_load(TestDir, Suite) of E = {error,_} -> E; @@ -905,6 +917,18 @@ timetrap(Time) -> test_server:timetrap_cancel(), test_server:timetrap(Time). +%%%----------------------------------------------------------------- +%%% @spec get_timetrap_info() -> {Time,Scale} +%%% Time = integer() | infinity +%%% Scale = true | false +%%% +%%% @doc

Read info about the timetrap set for the current test case. +%%% Scale indicates if Common Test will attempt to automatically +%%% compensate timetraps for runtime delays introduced by e.g. tools like +%%% cover.

+get_timetrap_info() -> + test_server:get_timetrap_info(). + %%%----------------------------------------------------------------- %%% @spec sleep(Time) -> ok %%% Time = {hours,Hours} | {minutes,Mins} | {seconds,Secs} | Millisecs | infinity diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 0897675591..f99eafaa34 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -36,6 +36,9 @@ -include("ct_event.hrl"). -include("ct_util.hrl"). +-define(val(Key, List), proplists:get_value(Key, List)). +-define(val(Key, List, Def), proplists:get_value(Key, List, Def)). + %%%----------------------------------------------------------------- %%% @spec init_tc(Mod,Func,Args) -> {ok,NewArgs} | {error,Reason} | %%% {skip,Reason} | {auto_skip,Reason} @@ -98,6 +101,17 @@ init_tc(Mod,Func,Config) -> {skip,InitFailed} end. +init_tc1(?MODULE,error_in_suite,[Config0],_) when is_list(Config0) -> + ct_logs:init_tc(false), + ct_event:notify(#event{name=tc_start, + node=node(), + data={?MODULE,error_in_suite}}), + case ?val(error, Config0) of + undefined -> + {skip,"unknown_error_in_suite"}; + Reason -> + {skip,Reason} + end; init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) -> Config1 = case ct_util:read_suite_data(last_saved_config) of @@ -122,19 +136,22 @@ init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) -> %% release all name -> key bindings (once per suite) ct_config:release_allocated() end, - TestCaseInfo = - case catch apply(Mod,Func,[]) of - Result when is_list(Result) -> Result; - _ -> [] - end, + + GroupPath = ?val(tc_group_path, Config, []), + AllGroups = [?val(tc_group_properties, Config, []) | GroupPath], + %% clear all config data default values set by previous %% testcase info function (these should only survive the %% testcase, not the whole suite) - ct_config:delete_default_config(testcase), - case add_defaults(Mod,Func,TestCaseInfo,DoInit) of + FuncSpec = group_or_func(Func,Config0), + if is_tuple(FuncSpec) -> % group + ok; + true -> + ct_config:delete_default_config(testcase) + end, + case add_defaults(Mod,Func,AllGroups,DoInit) of Error = {suite0_failed,_} -> ct_logs:init_tc(false), - FuncSpec = group_or_func(Func,Config0), ct_event:notify(#event{name=tc_start, node=node(), data={Mod,FuncSpec}}), @@ -144,13 +161,13 @@ init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) -> case MergeResult of {error,Reason} when DoInit == false -> ct_logs:init_tc(false), - FuncSpec = group_or_func(Func,Config0), ct_event:notify(#event{name=tc_start, node=node(), data={Mod,FuncSpec}}), {skip,Reason}; _ -> - init_tc2(Mod,Func,SuiteInfo,MergeResult,Config,DoInit) + init_tc2(Mod,Func,SuiteInfo,MergeResult, + Config,DoInit) end end; init_tc1(_Mod,_Func,Args,_DoInit) -> @@ -203,8 +220,9 @@ init_tc2(Mod,Func,SuiteInfo,MergeResult,Config,DoInit) -> ct_event:notify(#event{name=tc_start, node=node(), data={Mod,FuncSpec}}), - - case catch configure(MergedInfo1,MergedInfo1,SuiteInfo,{Func,DoInit},Config) of + + case catch configure(MergedInfo1,MergedInfo1,SuiteInfo, + {FuncSpec,DoInit},Config) of {suite0_failed,Reason} -> ct_util:set_testdata({curr_tc,{Mod,{suite0_failed,{require,Reason}}}}), {skip,{require_failed_in_suite0,Reason}}; @@ -212,7 +230,7 @@ init_tc2(Mod,Func,SuiteInfo,MergeResult,Config,DoInit) -> {auto_skip,{require_failed,Reason}}; {'EXIT',Reason} -> {auto_skip,Reason}; - {ok, FinalConfig} -> + {ok,FinalConfig} -> case MergeResult of {error,Reason} -> %% suite0 configure finished now, report that @@ -241,12 +259,12 @@ ct_suite_init(Mod, Func, [Config]) when is_list(Config) -> Else end. -add_defaults(Mod,Func,FuncInfo,DoInit) -> +add_defaults(Mod,Func, GroupPath, DoInit) -> case (catch Mod:suite()) of {'EXIT',{undef,_}} -> SuiteInfo = merge_with_suite_defaults(Mod,[]), SuiteInfoNoCTH = [I || I <- SuiteInfo, element(1,I) =/= ct_hooks], - case add_defaults1(Mod,Func,FuncInfo,SuiteInfoNoCTH,DoInit) of + case add_defaults1(Mod,Func, GroupPath, SuiteInfoNoCTH, DoInit) of Error = {error,_} -> {SuiteInfo,Error}; MergedInfo -> {SuiteInfo,MergedInfo} end; @@ -262,11 +280,11 @@ add_defaults(Mod,Func,FuncInfo,DoInit) -> (_) -> false end, SuiteInfo) of true -> - SuiteInfo1 = merge_with_suite_defaults(Mod,SuiteInfo), + SuiteInfo1 = merge_with_suite_defaults(Mod, SuiteInfo), SuiteInfoNoCTH = [I || I <- SuiteInfo1, element(1,I) =/= ct_hooks], - case add_defaults1(Mod,Func,FuncInfo, - SuiteInfoNoCTH,DoInit) of + case add_defaults1(Mod,Func, GroupPath, + SuiteInfoNoCTH, DoInit) of Error = {error,_} -> {SuiteInfo1,Error}; MergedInfo -> {SuiteInfo1,MergedInfo} end; @@ -287,51 +305,147 @@ add_defaults(Mod,Func,FuncInfo,DoInit) -> {suite0_failed,bad_return_value} end. -add_defaults1(_Mod,init_per_suite,[],SuiteInfo,_DoInit) -> - SuiteInfo; - -add_defaults1(Mod,Func,FuncInfo,SuiteInfo,DoInit) -> - %% mustn't re-require suite variables in test case info function, - %% can result in weird behaviour (suite values get overwritten) +add_defaults1(Mod,Func, GroupPath, SuiteInfo, DoInit) -> + %% GroupPathInfo (for subgroup on level X) = + %% [LevelXGroupInfo, LevelX-1GroupInfo, ..., TopLevelGroupInfo] + GroupPathInfo = + lists:map(fun(GroupProps) -> + Name = ?val(name, GroupProps), + case catch Mod:group(Name) of + GrInfo when is_list(GrInfo) -> GrInfo; + _ -> [] + end + end, GroupPath), + TestCaseInfo = + case catch apply(Mod,Func,[]) of + TCInfo when is_list(TCInfo) -> TCInfo; + _ -> [] + end, + %% let test case info (also for all config funcs) override group info, + %% and lower level group info override higher level info + TCAndGroupInfo = [TestCaseInfo | remove_info_in_prev(TestCaseInfo, + GroupPathInfo)], + %% find and save require terms found in suite info SuiteReqs = [SDDef || SDDef <- SuiteInfo, ((require == element(1,SDDef)) or (default_config == element(1,SDDef)))], - FuncReqs = - [FIDef || FIDef <- FuncInfo, - require == element(1,FIDef)], - case [element(2,Clash) || Clash <- SuiteReqs, - require == element(1, Clash), - true == lists:keymember(element(2,Clash),2, - FuncReqs)] of + + case check_for_clashes(TestCaseInfo, GroupPathInfo, SuiteReqs) of [] -> - add_defaults2(Mod,Func,FuncInfo,SuiteInfo,SuiteReqs,DoInit); + add_defaults2(Mod,Func, TCAndGroupInfo,SuiteInfo,SuiteReqs, DoInit); Clashes -> {error,{config_name_already_in_use,Clashes}} end. -add_defaults2(Mod,init_per_suite,IPSInfo,SuiteInfo,SuiteReqs,false) -> - %% not common practise to use a test case info function for - %% init_per_suite (usually handled by suite/0), but let's support - %% it just in case... - add_defaults2(Mod,init_per_suite,IPSInfo,SuiteInfo,SuiteReqs,true); - -add_defaults2(_Mod,_Func,FuncInfo,SuiteInfo,_,false) -> - %% include require elements from test case info, but not from suite/0 - %% (since we've already required those vars) - FuncInfo ++ - [SFDef || SFDef <- SuiteInfo, - require /= element(1,SFDef), - false == lists:keymember(element(1,SFDef),1,FuncInfo)]; - -add_defaults2(_Mod,_Func,FuncInfo,SuiteInfo,SuiteReqs,true) -> - %% We must include require elements from suite/0 here since - %% there's no init_per_suite call before this first test case. - %% Let other test case info elements override those from suite/0. - FuncInfo ++ SuiteReqs ++ - [SDDef || SDDef <- SuiteInfo, - require /= element(1,SDDef), - false == lists:keymember(element(1,SDDef),1,FuncInfo)]. +%% Check that alias names are not already in use +check_for_clashes(TCInfo, GrPathInfo, SuiteInfo) -> + {CurrGrInfo,SearchIn} = case GrPathInfo of + [] -> {[],[SuiteInfo]}; + [Curr|Path] -> {Curr,[SuiteInfo|Path]} + end, + ReqNames = fun(Info) -> [element(2,R) || R <- Info, + size(R) == 3, + require == element(1,R)] + end, + ExistingNames = lists:flatten([ReqNames(L) || L <- SearchIn]), + CurrGrReqNs = ReqNames(CurrGrInfo), + GrClashes = [Name || Name <- CurrGrReqNs, + true == lists:member(Name, ExistingNames)], + AllReqNs = CurrGrReqNs ++ ExistingNames, + TCClashes = [Name || Name <- ReqNames(TCInfo), + true == lists:member(Name, AllReqNs)], + TCClashes ++ GrClashes. + +%% Delete the info terms in Terms from all following info lists +remove_info_in_prev(Terms, [[] | Rest]) -> + [[] | remove_info_in_prev(Terms, Rest)]; +remove_info_in_prev(Terms, [Info | Rest]) -> + UniqueInInfo = [U || U <- Info, + ((timetrap == element(1,U)) and + (not lists:keymember(timetrap,1,Terms))) or + ((require == element(1,U)) and + (not lists:member(U,Terms))) or + ((default_config == element(1,U)) and + (not keysmember([default_config,1, + element(2,U),2], Terms)))], + OtherTermsInInfo = [T || T <- Info, + timetrap /= element(1,T), + require /= element(1,T), + default_config /= element(1,T), + false == lists:keymember(element(1,T),1, + Terms)], + KeptInfo = UniqueInInfo ++ OtherTermsInInfo, + [KeptInfo | remove_info_in_prev(Terms ++ KeptInfo, Rest)]; +remove_info_in_prev(_, []) -> + []. + +keysmember([Key,Pos|Next], List) -> + case [Elem || Elem <- List, Key == element(Pos,Elem)] of + [] -> false; + Found -> keysmember(Next, Found) + end; +keysmember([], _) -> true. + + +add_defaults2(Mod,init_per_suite, IPSInfo, SuiteInfo,SuiteReqs, false) -> + add_defaults2(Mod,init_per_suite, IPSInfo, SuiteInfo,SuiteReqs, true); + +add_defaults2(_Mod,IPG, IPGAndGroupInfo, SuiteInfo,SuiteReqs, DoInit) when + IPG == init_per_group ; IPG == ct_init_per_group -> + %% If DoInit == true, we have to process the suite() list, otherwise + %% it has already been handled (see clause for init_per_suite) + case DoInit of + true -> + %% note: we know for sure this is a top level group + Info = lists:flatten([IPGAndGroupInfo, SuiteReqs]), + Info ++ remove_info_in_prev(Info, [SuiteInfo]); + false -> + SuiteInfo1 = + remove_info_in_prev(lists:flatten([IPGAndGroupInfo, + SuiteReqs]), [SuiteInfo]), + %% don't require terms in prev groups (already processed) + case IPGAndGroupInfo of + [IPGInfo] -> + lists:flatten([IPGInfo,SuiteInfo1]); + [IPGInfo | [CurrGroupInfo | PrevGroupInfo]] -> + PrevGroupInfo1 = delete_require_terms(PrevGroupInfo), + lists:flatten([IPGInfo,CurrGroupInfo,PrevGroupInfo1, + SuiteInfo1]) + end + end; + +add_defaults2(_Mod,_Func, TCAndGroupInfo, SuiteInfo,SuiteReqs, false) -> + %% Include require elements from test case info and current group, + %% but not from previous groups or suite/0 (since we've already required + %% those vars). Let test case info elements override group and suite + %% info elements. + SuiteInfo1 = remove_info_in_prev(lists:flatten([TCAndGroupInfo, + SuiteReqs]), [SuiteInfo]), + %% don't require terms in prev groups (already processed) + case TCAndGroupInfo of + [TCInfo] -> + lists:flatten([TCInfo,SuiteInfo1]); + [TCInfo | [CurrGroupInfo | PrevGroupInfo]] -> + PrevGroupInfo1 = delete_require_terms(PrevGroupInfo), + lists:flatten([TCInfo,CurrGroupInfo,PrevGroupInfo1, + SuiteInfo1]) + end; + +add_defaults2(_Mod,_Func, TCInfo, SuiteInfo,SuiteReqs, true) -> + %% Here we have to process the suite info list also (no call to + %% init_per_suite before this first test case). This TC can't belong + %% to a group, or the clause for (ct_)init_per_group would've caught this. + Info = lists:flatten([TCInfo, SuiteReqs]), + lists:flatten([Info,remove_info_in_prev(Info, [SuiteInfo])]). + +delete_require_terms([Info | Prev]) -> + Info1 = [T || T <- Info, + require /= element(1,T), + default_config /= element(1,T)], + [Info1 | delete_require_terms(Prev)]; +delete_require_terms([]) -> + []. merge_with_suite_defaults(Mod,SuiteInfo) -> case lists:keysearch(suite_defaults,1,Mod:module_info(attributes)) of @@ -364,7 +478,8 @@ configure([{require,Required}|Rest],Info,SuiteInfo,Scope,Config) -> ok -> configure(Rest,Info,SuiteInfo,Scope,Config); Error = {error,Reason} -> - case required_default('_UNDEF',Required,Info,SuiteInfo,Scope) of + case required_default('_UNDEF',Required,Info, + SuiteInfo,Scope) of ok -> configure(Rest,Info,SuiteInfo,Scope,Config); _ -> @@ -406,18 +521,24 @@ configure([],_,_,_,Config) -> {ok,[Config]}. %% the require element in Info may come from suite/0 and -%% should be scoped 'suite', or come from the testcase info -%% function and should then be scoped 'testcase' -required_default(Name,Key,Info,SuiteInfo,{Func,true}) -> +%% should be scoped 'suite', or come from the group info +%% function and be scoped 'group', or come from the testcase +%% info function and then be scoped 'testcase' + +required_default(Name,Key,Info,SuiteInfo,{FuncSpec,true}) -> case try_set_default(Name,Key,SuiteInfo,suite) of ok -> ok; _ -> - required_default(Name,Key,Info,[],{Func,false}) + required_default(Name,Key,Info,[],{FuncSpec,false}) end; required_default(Name,Key,Info,_,{init_per_suite,_}) -> try_set_default(Name,Key,Info,suite); -required_default(Name,Key,Info,_,_) -> +required_default(Name,Key,Info,_,{{init_per_group,GrName,_},_}) -> + try_set_default(Name,Key,Info,{group,GrName}); +required_default(Name,Key,Info,_,{{ct_init_per_group,GrName,_},_}) -> + try_set_default(Name,Key,Info,{group,GrName}); +required_default(Name,Key,Info,_,_FuncSpec) -> try_set_default(Name,Key,Info,testcase). try_set_default(Name,Key,Info,Where) -> @@ -484,7 +605,11 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) -> ct_util:delete_suite_data(last_saved_config), FuncSpec = case group_or_func(Func,Args) of - {_,GroupName,_Props} = Group -> + {_,GroupName,_Props} = Group -> + if Func == end_per_group; Func == ct_end_per_group -> + ct_config:delete_default_config({group,GroupName}); + true -> ok + end, case lists:keysearch(save_config,1,Args) of {value,{save_config,SaveConfig}} -> ct_util:save_suite_data( @@ -703,12 +828,14 @@ mark_as_failed1(_,_,_,[]) -> ok. group_or_func(Func, Config) when Func == init_per_group; - Func == end_per_group -> - case proplists:get_value(tc_group_properties,Config) of + Func == end_per_group; + Func == ct_init_per_group; + Func == ct_end_per_group -> + case ?val(tc_group_properties, Config) of undefined -> {Func,unknown,[]}; GrProps -> - GrName = proplists:get_value(name,GrProps), + GrName = ?val(name,GrProps), {Func,GrName,proplists:delete(name,GrProps)} end; group_or_func(Func, _Config) -> @@ -746,7 +873,7 @@ get_suite(Mod, all) -> %% group get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) -> - Name = proplists:get_value(name, Props), + Name = ?val(name, Props), case catch apply(Mod, groups, []) of {'EXIT',_} -> [Group]; @@ -764,7 +891,7 @@ get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) -> %% a *subgroup* specified *only* as skipped (and not %% as an explicit test) should not be returned, or %% init/end functions for top groups will be executed - case catch proplists:get_value(name, element(2, hd(ConfTests))) of + case catch ?val(name, element(2, hd(ConfTests))) of Name -> % top group delete_subs(ConfTests, ConfTests); _ -> @@ -918,14 +1045,14 @@ delete_subs([], All) -> All. delete_conf({conf,Props,_,_,_}, Confs) -> - Name = proplists:get_value(name, Props), + Name = ?val(name, Props), [Conf || Conf = {conf,Props0,_,_,_} <- Confs, - Name =/= proplists:get_value(name, Props0)]. + Name =/= ?val(name, Props0)]. is_sub({conf,Props,_,_,_}=Conf, [{conf,_,_,Tests,_} | Confs]) -> - Name = proplists:get_value(name, Props), + Name = ?val(name, Props), case lists:any(fun({conf,Props0,_,_,_}) -> - case proplists:get_value(name, Props0) of + case ?val(name, Props0) of N when N == Name -> true; _ -> @@ -1080,7 +1207,7 @@ expand_groups([], _ConfTests, _Mod) -> expand_groups({group,Name}, ConfTests, Mod) -> FindConf = fun({conf,Props,_,_,_}) -> - case proplists:get_value(name, Props) of + case ?val(name, Props) of Name -> true; _ -> false end @@ -1234,8 +1361,8 @@ report(What,Data) -> loginfo -> %% logfiles and direcories have been created for a test and the %% top level test index page needs to be refreshed - TestName = filename:basename(proplists:get_value(topdir, Data), ".logs"), - RunDir = proplists:get_value(rundir, Data), + TestName = filename:basename(?val(topdir, Data), ".logs"), + RunDir = ?val(rundir, Data), ct_logs:make_all_suites_index({TestName,RunDir}), ok; tests_start -> @@ -1342,6 +1469,17 @@ report(What,Data) -> true -> ok end; + framework_error -> + case Data of + {{M,F},E} -> + ct_event:sync_notify(#event{name=tc_done, + node=node(), + data={M,F,{framework_error,E}}}); + _ -> + ct_event:sync_notify(#event{name=tc_done, + node=node(), + data=Data}) + end; _ -> ok end, diff --git a/lib/common_test/test/ct_config_SUITE.erl b/lib/common_test/test/ct_config_SUITE.erl index 8ce75f582a..18218bee47 100644 --- a/lib/common_test/test/ct_config_SUITE.erl +++ b/lib/common_test/test/ct_config_SUITE.erl @@ -228,7 +228,7 @@ expected_events(config_static_SUITE)-> {?eh,tc_start,{config_static_SUITE,test_config_name_already_in_use2}}, {?eh,tc_done, {config_static_SUITE,test_config_name_already_in_use2, - {skipped,{config_name_already_in_use,[x1,alias]}}}}, + {skipped,{config_name_already_in_use,[alias,x1]}}}}, {?eh,test_stats,{4,0,{2,0}}}, {?eh,tc_start,{config_static_SUITE,test_alias_tclocal}}, {?eh,tc_done,{config_static_SUITE,test_alias_tclocal,ok}}, diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index c1a455c6d8..2f081decca 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -493,7 +493,7 @@ test_events(cfg_error) -> {?eh,tc_start,{cfg_error_9_SUITE,tc1}}, {?eh,tc_done,{cfg_error_9_SUITE,tc1, {skipped,{failed,{cfg_error_9_SUITE,init_per_testcase, - tc1_should_be_skipped}}}}}, + {tc1_should_be_skipped,'_'}}}}}}, {?eh,test_stats,{9,0,{0,15}}}, {?eh,tc_start,{cfg_error_9_SUITE,tc2}}, {?eh,tc_done,{cfg_error_9_SUITE,tc2, @@ -535,12 +535,7 @@ test_events(cfg_error) -> {?eh,tc_start,{cfg_error_9_SUITE,tc13}}, {?eh,tc_done,{cfg_error_9_SUITE,tc13, {failed,{cfg_error_9_SUITE,end_per_testcase, - {'EXIT',{{badmatch,undefined}, - [{cfg_error_9_SUITE,end_per_testcase,2}, - {test_server,my_apply,3}, - {test_server,do_end_per_testcase,4}, - {test_server,run_test_case_eval1,6}, - {test_server,run_test_case_eval,9}]}}}}}}, + {'EXIT',{{badmatch,undefined},'_'}}}}}}, {?eh,test_stats,{12,3,{0,18}}}, {?eh,tc_start,{cfg_error_9_SUITE,tc14}}, {?eh,tc_done, @@ -568,8 +563,8 @@ test_events(cfg_error) -> {?eh,tc_start,{cfg_error_11_SUITE,init_per_suite}}, {?eh,tc_done,{cfg_error_11_SUITE,init_per_suite,ok}}, {?eh,tc_start,{cfg_error_11_SUITE,tc1}}, - {?eh,tc_done,{cfg_error_11_SUITE,tc1, - {skipped,{config_name_already_in_use,[dummy0]}}}}, + {?eh,tc_done, {cfg_error_11_SUITE,tc1, + {skipped,{config_name_already_in_use,[dummy_alias]}}}}, {?eh,test_stats,{12,6,{1,19}}}, {?eh,tc_start,{cfg_error_11_SUITE,tc2}}, {?eh,tc_done,{cfg_error_11_SUITE,tc2,ok}}, @@ -577,7 +572,7 @@ test_events(cfg_error) -> {?eh,tc_start,{cfg_error_11_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_11_SUITE,end_per_suite,ok}}, {?eh,tc_start,{cfg_error_12_SUITE,tc1}}, - {?eh,tc_done,{cfg_error_12_SUITE,tc1,{failed,{timetrap_timeout,500}}}}, + {?eh,tc_done,{ct_framework,init_tc,{framework_error,{timetrap,500}}}}, {?eh,test_stats,{13,7,{1,19}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc2}}, {?eh,tc_done,{cfg_error_12_SUITE,tc2,{failed, diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_11_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_11_SUITE.erl index ce94533110..aa7857aa92 100644 --- a/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_11_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_11_SUITE.erl @@ -29,10 +29,7 @@ suite() -> [{timetrap,{seconds,2}}, {require, dummy0}, {default_config, dummy0, "suite/0"}, - {require, dummy1}, {default_config, dummy1, "suite/0"}, - {require, dummy2}, {default_config, dummy2, "suite/0"}]. - - + {require, dummy_alias, dummy1}, {default_config, dummy1, "suite/0"}]. %%-------------------------------------------------------------------- %% Function: init_per_suite(Config0) -> @@ -119,16 +116,17 @@ groups() -> %% Reason = term() %%-------------------------------------------------------------------- all() -> - [tc1, tc2]. + [tc1,tc2]. tc1() -> - [{require, dummy0}, {default_config, dummy0, "tc1"}]. + [{require, dummy0}, {default_config, dummy0, "tc1"}, + {require, dummy_alias, dummy2}, {default_config, dummy2, "tc1"}]. tc1(_) -> dummy. tc2() -> - [{timetrap,1}]. + [{timetrap,10}]. tc2(_) -> dummy. diff --git a/lib/common_test/test/ct_groups_test_2_SUITE.erl b/lib/common_test/test/ct_groups_test_2_SUITE.erl index 940d791b15..2392b0b850 100644 --- a/lib/common_test/test/ct_groups_test_2_SUITE.erl +++ b/lib/common_test/test/ct_groups_test_2_SUITE.erl @@ -171,16 +171,16 @@ test_events(missing_conf) -> {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, {?eh,start_info,{1,1,2}}, - {?eh,tc_start,{ct_framework,ct_init_per_group}}, - {?eh,tc_done,{ct_framework,ct_init_per_group,ok}}, + {?eh,tc_start,{ct_framework,{ct_init_per_group,group1,[]}}}, + {?eh,tc_done,{ct_framework,{ct_init_per_group,group1,[]},ok}}, {?eh,tc_start,{missing_conf_SUITE,tc1}}, {?eh,tc_done,{missing_conf_SUITE,tc1,ok}}, {?eh,test_stats,{1,0,{0,0}}}, {?eh,tc_start,{missing_conf_SUITE,tc2}}, {?eh,tc_done,{missing_conf_SUITE,tc2,ok}}, {?eh,test_stats,{2,0,{0,0}}}, - {?eh,tc_start,{ct_framework,ct_end_per_group}}, - {?eh,tc_done,{ct_framework,ct_end_per_group,ok}}, + {?eh,tc_start,{ct_framework,{ct_end_per_group,group1,[]}}}, + {?eh,tc_done,{ct_framework,{ct_end_per_group,group1,[]},ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} ]; diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE.erl index 4471915e69..efc0309781 100644 --- a/lib/common_test/test/ct_test_server_if_1_SUITE.erl +++ b/lib/common_test/test/ct_test_server_if_1_SUITE.erl @@ -228,39 +228,39 @@ test_events(ts_if_1) -> {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}}, {?eh,tc_start,{ct_framework,error_in_suite}}, - {?eh,test_stats,{3,6,{3,7}}}, + {?eh,test_stats,{3,5,{4,7}}}, {?eh,tc_start,{ct_framework,error_in_suite}}, - {?eh,test_stats,{3,7,{3,7}}}, + {?eh,test_stats,{3,5,{5,7}}}, {?eh,tc_start,{ts_if_5_SUITE,init_per_suite}}, {?eh,tc_done,{ts_if_5_SUITE,init_per_suite, {skipped,{require_failed_in_suite0,{not_available,undef_variable}}}}}, {?eh,tc_auto_skip,{ts_if_5_SUITE,my_test_case, {require_failed_in_suite0,{not_available,undef_variable}}}}, - {?eh,test_stats,{3,7,{3,8}}}, + {?eh,test_stats,{3,5,{5,8}}}, {?eh,tc_auto_skip,{ts_if_5_SUITE,end_per_suite, {require_failed_in_suite0,{not_available,undef_variable}}}}, {?eh,tc_start,{ts_if_6_SUITE,tc1}}, {?eh,tc_done,{ts_if_6_SUITE,tc1,{failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}}, - {?eh,test_stats,{3,7,{4,8}}}, + {?eh,test_stats,{3,5,{6,8}}}, {?eh,tc_start,{ts_if_7_SUITE,tc1}}, {?eh,tc_done,{ts_if_7_SUITE,tc1,ok}}, - {?eh,test_stats,{4,7,{4,8}}}, + {?eh,test_stats,{4,5,{6,8}}}, {?eh,tc_start,{ts_if_8_SUITE,tc1}}, {?eh,tc_done,{ts_if_8_SUITE,tc1,{failed,{error,failed_on_purpose}}}}, - {?eh,test_stats,{4,8,{4,8}}}, + {?eh,test_stats,{4,6,{6,8}}}, {?eh,tc_user_skip,{skipped_by_spec_1_SUITE,all,"should be skipped"}}, - {?eh,test_stats,{4,8,{5,8}}}, + {?eh,test_stats,{4,6,{7,8}}}, {?eh,tc_start,{skipped_by_spec_2_SUITE,init_per_suite}}, {?eh,tc_done,{skipped_by_spec_2_SUITE,init_per_suite,ok}}, {?eh,tc_user_skip,{skipped_by_spec_2_SUITE,tc1,"should be skipped"}}, - {?eh,test_stats,{4,8,{6,8}}}, + {?eh,test_stats,{4,6,{8,8}}}, {?eh,tc_start,{skipped_by_spec_2_SUITE,end_per_suite}}, {?eh,tc_done,{skipped_by_spec_2_SUITE,end_per_suite,ok}}, -- cgit v1.2.3