aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--lib/common_test/src/ct.erl36
-rw-r--r--lib/common_test/src/ct_framework.erl284
-rw-r--r--lib/common_test/test/ct_config_SUITE.erl2
-rw-r--r--lib/common_test/test/ct_error_SUITE.erl15
-rw-r--r--lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_11_SUITE.erl12
-rw-r--r--lib/common_test/test/ct_groups_test_2_SUITE.erl8
-rw-r--r--lib/common_test/test/ct_test_server_if_1_SUITE.erl16
-rw-r--r--lib/test_server/src/test_server.erl43
-rw-r--r--lib/test_server/src/test_server_ctrl.erl16
9 files changed, 310 insertions, 122 deletions
diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl
index d72b8bc0e1..6fd7693185 100644
--- a/lib/common_test/src/ct.erl
+++ b/lib/common_test/src/ct.erl
@@ -65,7 +65,7 @@
pal/1, pal/2, pal/3,
fail/1, fail/2, comment/1, comment/2,
testcases/2, userdata/2, userdata/3,
- timetrap/1, sleep/1]).
+ timetrap/1, get_timetrap_info/0, sleep/1]).
%% New API for manipulating with config handlers
-export([add_config/2, remove_config/2]).
@@ -703,7 +703,8 @@ userdata(TestDir, Suite) ->
get_userdata(Info, "suite/0")
end.
-get_userdata({'EXIT',{undef,_}}, Spec) ->
+get_userdata({'EXIT',{Undef,_}}, Spec) when Undef == undef;
+ Undef == function_clause ->
{error,list_to_atom(Spec ++ " is not defined")};
get_userdata({'EXIT',Reason}, Spec) ->
{error,{list_to_atom("error in " ++ Spec),Reason}};
@@ -719,16 +720,27 @@ get_userdata(_BadTerm, Spec) ->
{error,list_to_atom(Spec ++ " must return a list")}.
%%%-----------------------------------------------------------------
-%%% @spec userdata(TestDir, Suite, Case) -> TCUserData | {error,Reason}
+%%% @spec userdata(TestDir, Suite, GroupOrCase) -> TCUserData | {error,Reason}
%%% TestDir = string()
%%% Suite = atom()
-%%% Case = atom()
+%%% GroupOrCase = {group,GroupName} | atom()
+%%% GroupName = atom()
%%% TCUserData = [term()]
%%% Reason = term()
%%%
%%% @doc Returns any data specified with the tag <code>userdata</code>
-%%% in the list of tuples returned from <code>Suite:Case/0</code>.
-userdata(TestDir, Suite, Case) ->
+%%% in the list of tuples returned from <code>Suite:group(GroupName)</code>
+%%% or <code>Suite:Case()</code>.
+userdata(TestDir, Suite, {group,GroupName}) ->
+ case make_and_load(TestDir, Suite) of
+ E = {error,_} ->
+ E;
+ _ ->
+ Info = (catch apply(Suite, group, [GroupName])),
+ get_userdata(Info, "group("++atom_to_list(GroupName)++")")
+ end;
+
+userdata(TestDir, Suite, Case) when is_atom(Case) ->
case make_and_load(TestDir, Suite) of
E = {error,_} ->
E;
@@ -906,6 +918,18 @@ timetrap(Time) ->
test_server:timetrap(Time).
%%%-----------------------------------------------------------------
+%%% @spec get_timetrap_info() -> {Time,Scale}
+%%% Time = integer() | infinity
+%%% Scale = true | false
+%%%
+%%% @doc <p>Read info about the timetrap set for the current test case.
+%%% <c>Scale</c> indicates if Common Test will attempt to automatically
+%%% compensate timetraps for runtime delays introduced by e.g. tools like
+%%% cover.</p>
+get_timetrap_info() ->
+ test_server:get_timetrap_info().
+
+%%%-----------------------------------------------------------------
%%% @spec sleep(Time) -> ok
%%% Time = {hours,Hours} | {minutes,Mins} | {seconds,Secs} | Millisecs | infinity
%%% Hours = integer()
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index 0897675591..f99eafaa34 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -36,6 +36,9 @@
-include("ct_event.hrl").
-include("ct_util.hrl").
+-define(val(Key, List), proplists:get_value(Key, List)).
+-define(val(Key, List, Def), proplists:get_value(Key, List, Def)).
+
%%%-----------------------------------------------------------------
%%% @spec init_tc(Mod,Func,Args) -> {ok,NewArgs} | {error,Reason} |
%%% {skip,Reason} | {auto_skip,Reason}
@@ -98,6 +101,17 @@ init_tc(Mod,Func,Config) ->
{skip,InitFailed}
end.
+init_tc1(?MODULE,error_in_suite,[Config0],_) when is_list(Config0) ->
+ ct_logs:init_tc(false),
+ ct_event:notify(#event{name=tc_start,
+ node=node(),
+ data={?MODULE,error_in_suite}}),
+ case ?val(error, Config0) of
+ undefined ->
+ {skip,"unknown_error_in_suite"};
+ Reason ->
+ {skip,Reason}
+ end;
init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) ->
Config1 =
case ct_util:read_suite_data(last_saved_config) of
@@ -122,19 +136,22 @@ init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) ->
%% release all name -> key bindings (once per suite)
ct_config:release_allocated()
end,
- TestCaseInfo =
- case catch apply(Mod,Func,[]) of
- Result when is_list(Result) -> Result;
- _ -> []
- end,
+
+ GroupPath = ?val(tc_group_path, Config, []),
+ AllGroups = [?val(tc_group_properties, Config, []) | GroupPath],
+
%% clear all config data default values set by previous
%% testcase info function (these should only survive the
%% testcase, not the whole suite)
- ct_config:delete_default_config(testcase),
- case add_defaults(Mod,Func,TestCaseInfo,DoInit) of
+ FuncSpec = group_or_func(Func,Config0),
+ if is_tuple(FuncSpec) -> % group
+ ok;
+ true ->
+ ct_config:delete_default_config(testcase)
+ end,
+ case add_defaults(Mod,Func,AllGroups,DoInit) of
Error = {suite0_failed,_} ->
ct_logs:init_tc(false),
- FuncSpec = group_or_func(Func,Config0),
ct_event:notify(#event{name=tc_start,
node=node(),
data={Mod,FuncSpec}}),
@@ -144,13 +161,13 @@ init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) ->
case MergeResult of
{error,Reason} when DoInit == false ->
ct_logs:init_tc(false),
- FuncSpec = group_or_func(Func,Config0),
ct_event:notify(#event{name=tc_start,
node=node(),
data={Mod,FuncSpec}}),
{skip,Reason};
_ ->
- init_tc2(Mod,Func,SuiteInfo,MergeResult,Config,DoInit)
+ init_tc2(Mod,Func,SuiteInfo,MergeResult,
+ Config,DoInit)
end
end;
init_tc1(_Mod,_Func,Args,_DoInit) ->
@@ -203,8 +220,9 @@ init_tc2(Mod,Func,SuiteInfo,MergeResult,Config,DoInit) ->
ct_event:notify(#event{name=tc_start,
node=node(),
data={Mod,FuncSpec}}),
-
- case catch configure(MergedInfo1,MergedInfo1,SuiteInfo,{Func,DoInit},Config) of
+
+ case catch configure(MergedInfo1,MergedInfo1,SuiteInfo,
+ {FuncSpec,DoInit},Config) of
{suite0_failed,Reason} ->
ct_util:set_testdata({curr_tc,{Mod,{suite0_failed,{require,Reason}}}}),
{skip,{require_failed_in_suite0,Reason}};
@@ -212,7 +230,7 @@ init_tc2(Mod,Func,SuiteInfo,MergeResult,Config,DoInit) ->
{auto_skip,{require_failed,Reason}};
{'EXIT',Reason} ->
{auto_skip,Reason};
- {ok, FinalConfig} ->
+ {ok,FinalConfig} ->
case MergeResult of
{error,Reason} ->
%% suite0 configure finished now, report that
@@ -241,12 +259,12 @@ ct_suite_init(Mod, Func, [Config]) when is_list(Config) ->
Else
end.
-add_defaults(Mod,Func,FuncInfo,DoInit) ->
+add_defaults(Mod,Func, GroupPath, DoInit) ->
case (catch Mod:suite()) of
{'EXIT',{undef,_}} ->
SuiteInfo = merge_with_suite_defaults(Mod,[]),
SuiteInfoNoCTH = [I || I <- SuiteInfo, element(1,I) =/= ct_hooks],
- case add_defaults1(Mod,Func,FuncInfo,SuiteInfoNoCTH,DoInit) of
+ case add_defaults1(Mod,Func, GroupPath, SuiteInfoNoCTH, DoInit) of
Error = {error,_} -> {SuiteInfo,Error};
MergedInfo -> {SuiteInfo,MergedInfo}
end;
@@ -262,11 +280,11 @@ add_defaults(Mod,Func,FuncInfo,DoInit) ->
(_) -> false
end, SuiteInfo) of
true ->
- SuiteInfo1 = merge_with_suite_defaults(Mod,SuiteInfo),
+ SuiteInfo1 = merge_with_suite_defaults(Mod, SuiteInfo),
SuiteInfoNoCTH = [I || I <- SuiteInfo1,
element(1,I) =/= ct_hooks],
- case add_defaults1(Mod,Func,FuncInfo,
- SuiteInfoNoCTH,DoInit) of
+ case add_defaults1(Mod,Func, GroupPath,
+ SuiteInfoNoCTH, DoInit) of
Error = {error,_} -> {SuiteInfo1,Error};
MergedInfo -> {SuiteInfo1,MergedInfo}
end;
@@ -287,51 +305,147 @@ add_defaults(Mod,Func,FuncInfo,DoInit) ->
{suite0_failed,bad_return_value}
end.
-add_defaults1(_Mod,init_per_suite,[],SuiteInfo,_DoInit) ->
- SuiteInfo;
-
-add_defaults1(Mod,Func,FuncInfo,SuiteInfo,DoInit) ->
- %% mustn't re-require suite variables in test case info function,
- %% can result in weird behaviour (suite values get overwritten)
+add_defaults1(Mod,Func, GroupPath, SuiteInfo, DoInit) ->
+ %% GroupPathInfo (for subgroup on level X) =
+ %% [LevelXGroupInfo, LevelX-1GroupInfo, ..., TopLevelGroupInfo]
+ GroupPathInfo =
+ lists:map(fun(GroupProps) ->
+ Name = ?val(name, GroupProps),
+ case catch Mod:group(Name) of
+ GrInfo when is_list(GrInfo) -> GrInfo;
+ _ -> []
+ end
+ end, GroupPath),
+ TestCaseInfo =
+ case catch apply(Mod,Func,[]) of
+ TCInfo when is_list(TCInfo) -> TCInfo;
+ _ -> []
+ end,
+ %% let test case info (also for all config funcs) override group info,
+ %% and lower level group info override higher level info
+ TCAndGroupInfo = [TestCaseInfo | remove_info_in_prev(TestCaseInfo,
+ GroupPathInfo)],
+ %% find and save require terms found in suite info
SuiteReqs =
[SDDef || SDDef <- SuiteInfo,
((require == element(1,SDDef)) or
(default_config == element(1,SDDef)))],
- FuncReqs =
- [FIDef || FIDef <- FuncInfo,
- require == element(1,FIDef)],
- case [element(2,Clash) || Clash <- SuiteReqs,
- require == element(1, Clash),
- true == lists:keymember(element(2,Clash),2,
- FuncReqs)] of
+
+ case check_for_clashes(TestCaseInfo, GroupPathInfo, SuiteReqs) of
[] ->
- add_defaults2(Mod,Func,FuncInfo,SuiteInfo,SuiteReqs,DoInit);
+ add_defaults2(Mod,Func, TCAndGroupInfo,SuiteInfo,SuiteReqs, DoInit);
Clashes ->
{error,{config_name_already_in_use,Clashes}}
end.
-add_defaults2(Mod,init_per_suite,IPSInfo,SuiteInfo,SuiteReqs,false) ->
- %% not common practise to use a test case info function for
- %% init_per_suite (usually handled by suite/0), but let's support
- %% it just in case...
- add_defaults2(Mod,init_per_suite,IPSInfo,SuiteInfo,SuiteReqs,true);
-
-add_defaults2(_Mod,_Func,FuncInfo,SuiteInfo,_,false) ->
- %% include require elements from test case info, but not from suite/0
- %% (since we've already required those vars)
- FuncInfo ++
- [SFDef || SFDef <- SuiteInfo,
- require /= element(1,SFDef),
- false == lists:keymember(element(1,SFDef),1,FuncInfo)];
-
-add_defaults2(_Mod,_Func,FuncInfo,SuiteInfo,SuiteReqs,true) ->
- %% We must include require elements from suite/0 here since
- %% there's no init_per_suite call before this first test case.
- %% Let other test case info elements override those from suite/0.
- FuncInfo ++ SuiteReqs ++
- [SDDef || SDDef <- SuiteInfo,
- require /= element(1,SDDef),
- false == lists:keymember(element(1,SDDef),1,FuncInfo)].
+%% Check that alias names are not already in use
+check_for_clashes(TCInfo, GrPathInfo, SuiteInfo) ->
+ {CurrGrInfo,SearchIn} = case GrPathInfo of
+ [] -> {[],[SuiteInfo]};
+ [Curr|Path] -> {Curr,[SuiteInfo|Path]}
+ end,
+ ReqNames = fun(Info) -> [element(2,R) || R <- Info,
+ size(R) == 3,
+ require == element(1,R)]
+ end,
+ ExistingNames = lists:flatten([ReqNames(L) || L <- SearchIn]),
+ CurrGrReqNs = ReqNames(CurrGrInfo),
+ GrClashes = [Name || Name <- CurrGrReqNs,
+ true == lists:member(Name, ExistingNames)],
+ AllReqNs = CurrGrReqNs ++ ExistingNames,
+ TCClashes = [Name || Name <- ReqNames(TCInfo),
+ true == lists:member(Name, AllReqNs)],
+ TCClashes ++ GrClashes.
+
+%% Delete the info terms in Terms from all following info lists
+remove_info_in_prev(Terms, [[] | Rest]) ->
+ [[] | remove_info_in_prev(Terms, Rest)];
+remove_info_in_prev(Terms, [Info | Rest]) ->
+ UniqueInInfo = [U || U <- Info,
+ ((timetrap == element(1,U)) and
+ (not lists:keymember(timetrap,1,Terms))) or
+ ((require == element(1,U)) and
+ (not lists:member(U,Terms))) or
+ ((default_config == element(1,U)) and
+ (not keysmember([default_config,1,
+ element(2,U),2], Terms)))],
+ OtherTermsInInfo = [T || T <- Info,
+ timetrap /= element(1,T),
+ require /= element(1,T),
+ default_config /= element(1,T),
+ false == lists:keymember(element(1,T),1,
+ Terms)],
+ KeptInfo = UniqueInInfo ++ OtherTermsInInfo,
+ [KeptInfo | remove_info_in_prev(Terms ++ KeptInfo, Rest)];
+remove_info_in_prev(_, []) ->
+ [].
+
+keysmember([Key,Pos|Next], List) ->
+ case [Elem || Elem <- List, Key == element(Pos,Elem)] of
+ [] -> false;
+ Found -> keysmember(Next, Found)
+ end;
+keysmember([], _) -> true.
+
+
+add_defaults2(Mod,init_per_suite, IPSInfo, SuiteInfo,SuiteReqs, false) ->
+ add_defaults2(Mod,init_per_suite, IPSInfo, SuiteInfo,SuiteReqs, true);
+
+add_defaults2(_Mod,IPG, IPGAndGroupInfo, SuiteInfo,SuiteReqs, DoInit) when
+ IPG == init_per_group ; IPG == ct_init_per_group ->
+ %% If DoInit == true, we have to process the suite() list, otherwise
+ %% it has already been handled (see clause for init_per_suite)
+ case DoInit of
+ true ->
+ %% note: we know for sure this is a top level group
+ Info = lists:flatten([IPGAndGroupInfo, SuiteReqs]),
+ Info ++ remove_info_in_prev(Info, [SuiteInfo]);
+ false ->
+ SuiteInfo1 =
+ remove_info_in_prev(lists:flatten([IPGAndGroupInfo,
+ SuiteReqs]), [SuiteInfo]),
+ %% don't require terms in prev groups (already processed)
+ case IPGAndGroupInfo of
+ [IPGInfo] ->
+ lists:flatten([IPGInfo,SuiteInfo1]);
+ [IPGInfo | [CurrGroupInfo | PrevGroupInfo]] ->
+ PrevGroupInfo1 = delete_require_terms(PrevGroupInfo),
+ lists:flatten([IPGInfo,CurrGroupInfo,PrevGroupInfo1,
+ SuiteInfo1])
+ end
+ end;
+
+add_defaults2(_Mod,_Func, TCAndGroupInfo, SuiteInfo,SuiteReqs, false) ->
+ %% Include require elements from test case info and current group,
+ %% but not from previous groups or suite/0 (since we've already required
+ %% those vars). Let test case info elements override group and suite
+ %% info elements.
+ SuiteInfo1 = remove_info_in_prev(lists:flatten([TCAndGroupInfo,
+ SuiteReqs]), [SuiteInfo]),
+ %% don't require terms in prev groups (already processed)
+ case TCAndGroupInfo of
+ [TCInfo] ->
+ lists:flatten([TCInfo,SuiteInfo1]);
+ [TCInfo | [CurrGroupInfo | PrevGroupInfo]] ->
+ PrevGroupInfo1 = delete_require_terms(PrevGroupInfo),
+ lists:flatten([TCInfo,CurrGroupInfo,PrevGroupInfo1,
+ SuiteInfo1])
+ end;
+
+add_defaults2(_Mod,_Func, TCInfo, SuiteInfo,SuiteReqs, true) ->
+ %% Here we have to process the suite info list also (no call to
+ %% init_per_suite before this first test case). This TC can't belong
+ %% to a group, or the clause for (ct_)init_per_group would've caught this.
+ Info = lists:flatten([TCInfo, SuiteReqs]),
+ lists:flatten([Info,remove_info_in_prev(Info, [SuiteInfo])]).
+
+delete_require_terms([Info | Prev]) ->
+ Info1 = [T || T <- Info,
+ require /= element(1,T),
+ default_config /= element(1,T)],
+ [Info1 | delete_require_terms(Prev)];
+delete_require_terms([]) ->
+ [].
merge_with_suite_defaults(Mod,SuiteInfo) ->
case lists:keysearch(suite_defaults,1,Mod:module_info(attributes)) of
@@ -364,7 +478,8 @@ configure([{require,Required}|Rest],Info,SuiteInfo,Scope,Config) ->
ok ->
configure(Rest,Info,SuiteInfo,Scope,Config);
Error = {error,Reason} ->
- case required_default('_UNDEF',Required,Info,SuiteInfo,Scope) of
+ case required_default('_UNDEF',Required,Info,
+ SuiteInfo,Scope) of
ok ->
configure(Rest,Info,SuiteInfo,Scope,Config);
_ ->
@@ -406,18 +521,24 @@ configure([],_,_,_,Config) ->
{ok,[Config]}.
%% the require element in Info may come from suite/0 and
-%% should be scoped 'suite', or come from the testcase info
-%% function and should then be scoped 'testcase'
-required_default(Name,Key,Info,SuiteInfo,{Func,true}) ->
+%% should be scoped 'suite', or come from the group info
+%% function and be scoped 'group', or come from the testcase
+%% info function and then be scoped 'testcase'
+
+required_default(Name,Key,Info,SuiteInfo,{FuncSpec,true}) ->
case try_set_default(Name,Key,SuiteInfo,suite) of
ok ->
ok;
_ ->
- required_default(Name,Key,Info,[],{Func,false})
+ required_default(Name,Key,Info,[],{FuncSpec,false})
end;
required_default(Name,Key,Info,_,{init_per_suite,_}) ->
try_set_default(Name,Key,Info,suite);
-required_default(Name,Key,Info,_,_) ->
+required_default(Name,Key,Info,_,{{init_per_group,GrName,_},_}) ->
+ try_set_default(Name,Key,Info,{group,GrName});
+required_default(Name,Key,Info,_,{{ct_init_per_group,GrName,_},_}) ->
+ try_set_default(Name,Key,Info,{group,GrName});
+required_default(Name,Key,Info,_,_FuncSpec) ->
try_set_default(Name,Key,Info,testcase).
try_set_default(Name,Key,Info,Where) ->
@@ -484,7 +605,11 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) ->
ct_util:delete_suite_data(last_saved_config),
FuncSpec =
case group_or_func(Func,Args) of
- {_,GroupName,_Props} = Group ->
+ {_,GroupName,_Props} = Group ->
+ if Func == end_per_group; Func == ct_end_per_group ->
+ ct_config:delete_default_config({group,GroupName});
+ true -> ok
+ end,
case lists:keysearch(save_config,1,Args) of
{value,{save_config,SaveConfig}} ->
ct_util:save_suite_data(
@@ -703,12 +828,14 @@ mark_as_failed1(_,_,_,[]) ->
ok.
group_or_func(Func, Config) when Func == init_per_group;
- Func == end_per_group ->
- case proplists:get_value(tc_group_properties,Config) of
+ Func == end_per_group;
+ Func == ct_init_per_group;
+ Func == ct_end_per_group ->
+ case ?val(tc_group_properties, Config) of
undefined ->
{Func,unknown,[]};
GrProps ->
- GrName = proplists:get_value(name,GrProps),
+ GrName = ?val(name,GrProps),
{Func,GrName,proplists:delete(name,GrProps)}
end;
group_or_func(Func, _Config) ->
@@ -746,7 +873,7 @@ get_suite(Mod, all) ->
%% group
get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) ->
- Name = proplists:get_value(name, Props),
+ Name = ?val(name, Props),
case catch apply(Mod, groups, []) of
{'EXIT',_} ->
[Group];
@@ -764,7 +891,7 @@ get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) ->
%% a *subgroup* specified *only* as skipped (and not
%% as an explicit test) should not be returned, or
%% init/end functions for top groups will be executed
- case catch proplists:get_value(name, element(2, hd(ConfTests))) of
+ case catch ?val(name, element(2, hd(ConfTests))) of
Name -> % top group
delete_subs(ConfTests, ConfTests);
_ ->
@@ -918,14 +1045,14 @@ delete_subs([], All) ->
All.
delete_conf({conf,Props,_,_,_}, Confs) ->
- Name = proplists:get_value(name, Props),
+ Name = ?val(name, Props),
[Conf || Conf = {conf,Props0,_,_,_} <- Confs,
- Name =/= proplists:get_value(name, Props0)].
+ Name =/= ?val(name, Props0)].
is_sub({conf,Props,_,_,_}=Conf, [{conf,_,_,Tests,_} | Confs]) ->
- Name = proplists:get_value(name, Props),
+ Name = ?val(name, Props),
case lists:any(fun({conf,Props0,_,_,_}) ->
- case proplists:get_value(name, Props0) of
+ case ?val(name, Props0) of
N when N == Name ->
true;
_ ->
@@ -1080,7 +1207,7 @@ expand_groups([], _ConfTests, _Mod) ->
expand_groups({group,Name}, ConfTests, Mod) ->
FindConf =
fun({conf,Props,_,_,_}) ->
- case proplists:get_value(name, Props) of
+ case ?val(name, Props) of
Name -> true;
_ -> false
end
@@ -1234,8 +1361,8 @@ report(What,Data) ->
loginfo ->
%% logfiles and direcories have been created for a test and the
%% top level test index page needs to be refreshed
- TestName = filename:basename(proplists:get_value(topdir, Data), ".logs"),
- RunDir = proplists:get_value(rundir, Data),
+ TestName = filename:basename(?val(topdir, Data), ".logs"),
+ RunDir = ?val(rundir, Data),
ct_logs:make_all_suites_index({TestName,RunDir}),
ok;
tests_start ->
@@ -1342,6 +1469,17 @@ report(What,Data) ->
true ->
ok
end;
+ framework_error ->
+ case Data of
+ {{M,F},E} ->
+ ct_event:sync_notify(#event{name=tc_done,
+ node=node(),
+ data={M,F,{framework_error,E}}});
+ _ ->
+ ct_event:sync_notify(#event{name=tc_done,
+ node=node(),
+ data=Data})
+ end;
_ ->
ok
end,
diff --git a/lib/common_test/test/ct_config_SUITE.erl b/lib/common_test/test/ct_config_SUITE.erl
index 8ce75f582a..18218bee47 100644
--- a/lib/common_test/test/ct_config_SUITE.erl
+++ b/lib/common_test/test/ct_config_SUITE.erl
@@ -228,7 +228,7 @@ expected_events(config_static_SUITE)->
{?eh,tc_start,{config_static_SUITE,test_config_name_already_in_use2}},
{?eh,tc_done,
{config_static_SUITE,test_config_name_already_in_use2,
- {skipped,{config_name_already_in_use,[x1,alias]}}}},
+ {skipped,{config_name_already_in_use,[alias,x1]}}}},
{?eh,test_stats,{4,0,{2,0}}},
{?eh,tc_start,{config_static_SUITE,test_alias_tclocal}},
{?eh,tc_done,{config_static_SUITE,test_alias_tclocal,ok}},
diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl
index c1a455c6d8..2f081decca 100644
--- a/lib/common_test/test/ct_error_SUITE.erl
+++ b/lib/common_test/test/ct_error_SUITE.erl
@@ -493,7 +493,7 @@ test_events(cfg_error) ->
{?eh,tc_start,{cfg_error_9_SUITE,tc1}},
{?eh,tc_done,{cfg_error_9_SUITE,tc1,
{skipped,{failed,{cfg_error_9_SUITE,init_per_testcase,
- tc1_should_be_skipped}}}}},
+ {tc1_should_be_skipped,'_'}}}}}},
{?eh,test_stats,{9,0,{0,15}}},
{?eh,tc_start,{cfg_error_9_SUITE,tc2}},
{?eh,tc_done,{cfg_error_9_SUITE,tc2,
@@ -535,12 +535,7 @@ test_events(cfg_error) ->
{?eh,tc_start,{cfg_error_9_SUITE,tc13}},
{?eh,tc_done,{cfg_error_9_SUITE,tc13,
{failed,{cfg_error_9_SUITE,end_per_testcase,
- {'EXIT',{{badmatch,undefined},
- [{cfg_error_9_SUITE,end_per_testcase,2},
- {test_server,my_apply,3},
- {test_server,do_end_per_testcase,4},
- {test_server,run_test_case_eval1,6},
- {test_server,run_test_case_eval,9}]}}}}}},
+ {'EXIT',{{badmatch,undefined},'_'}}}}}},
{?eh,test_stats,{12,3,{0,18}}},
{?eh,tc_start,{cfg_error_9_SUITE,tc14}},
{?eh,tc_done,
@@ -568,8 +563,8 @@ test_events(cfg_error) ->
{?eh,tc_start,{cfg_error_11_SUITE,init_per_suite}},
{?eh,tc_done,{cfg_error_11_SUITE,init_per_suite,ok}},
{?eh,tc_start,{cfg_error_11_SUITE,tc1}},
- {?eh,tc_done,{cfg_error_11_SUITE,tc1,
- {skipped,{config_name_already_in_use,[dummy0]}}}},
+ {?eh,tc_done, {cfg_error_11_SUITE,tc1,
+ {skipped,{config_name_already_in_use,[dummy_alias]}}}},
{?eh,test_stats,{12,6,{1,19}}},
{?eh,tc_start,{cfg_error_11_SUITE,tc2}},
{?eh,tc_done,{cfg_error_11_SUITE,tc2,ok}},
@@ -577,7 +572,7 @@ test_events(cfg_error) ->
{?eh,tc_start,{cfg_error_11_SUITE,end_per_suite}},
{?eh,tc_done,{cfg_error_11_SUITE,end_per_suite,ok}},
{?eh,tc_start,{cfg_error_12_SUITE,tc1}},
- {?eh,tc_done,{cfg_error_12_SUITE,tc1,{failed,{timetrap_timeout,500}}}},
+ {?eh,tc_done,{ct_framework,init_tc,{framework_error,{timetrap,500}}}},
{?eh,test_stats,{13,7,{1,19}}},
{?eh,tc_start,{cfg_error_12_SUITE,tc2}},
{?eh,tc_done,{cfg_error_12_SUITE,tc2,{failed,
diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_11_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_11_SUITE.erl
index ce94533110..aa7857aa92 100644
--- a/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_11_SUITE.erl
+++ b/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_11_SUITE.erl
@@ -29,10 +29,7 @@
suite() ->
[{timetrap,{seconds,2}},
{require, dummy0}, {default_config, dummy0, "suite/0"},
- {require, dummy1}, {default_config, dummy1, "suite/0"},
- {require, dummy2}, {default_config, dummy2, "suite/0"}].
-
-
+ {require, dummy_alias, dummy1}, {default_config, dummy1, "suite/0"}].
%%--------------------------------------------------------------------
%% Function: init_per_suite(Config0) ->
@@ -119,16 +116,17 @@ groups() ->
%% Reason = term()
%%--------------------------------------------------------------------
all() ->
- [tc1, tc2].
+ [tc1,tc2].
tc1() ->
- [{require, dummy0}, {default_config, dummy0, "tc1"}].
+ [{require, dummy0}, {default_config, dummy0, "tc1"},
+ {require, dummy_alias, dummy2}, {default_config, dummy2, "tc1"}].
tc1(_) ->
dummy.
tc2() ->
- [{timetrap,1}].
+ [{timetrap,10}].
tc2(_) ->
dummy.
diff --git a/lib/common_test/test/ct_groups_test_2_SUITE.erl b/lib/common_test/test/ct_groups_test_2_SUITE.erl
index 940d791b15..2392b0b850 100644
--- a/lib/common_test/test/ct_groups_test_2_SUITE.erl
+++ b/lib/common_test/test/ct_groups_test_2_SUITE.erl
@@ -171,16 +171,16 @@ test_events(missing_conf) ->
{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
{?eh,start_info,{1,1,2}},
- {?eh,tc_start,{ct_framework,ct_init_per_group}},
- {?eh,tc_done,{ct_framework,ct_init_per_group,ok}},
+ {?eh,tc_start,{ct_framework,{ct_init_per_group,group1,[]}}},
+ {?eh,tc_done,{ct_framework,{ct_init_per_group,group1,[]},ok}},
{?eh,tc_start,{missing_conf_SUITE,tc1}},
{?eh,tc_done,{missing_conf_SUITE,tc1,ok}},
{?eh,test_stats,{1,0,{0,0}}},
{?eh,tc_start,{missing_conf_SUITE,tc2}},
{?eh,tc_done,{missing_conf_SUITE,tc2,ok}},
{?eh,test_stats,{2,0,{0,0}}},
- {?eh,tc_start,{ct_framework,ct_end_per_group}},
- {?eh,tc_done,{ct_framework,ct_end_per_group,ok}},
+ {?eh,tc_start,{ct_framework,{ct_end_per_group,group1,[]}}},
+ {?eh,tc_done,{ct_framework,{ct_end_per_group,group1,[]},ok}},
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}
];
diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE.erl
index 4471915e69..efc0309781 100644
--- a/lib/common_test/test/ct_test_server_if_1_SUITE.erl
+++ b/lib/common_test/test/ct_test_server_if_1_SUITE.erl
@@ -228,39 +228,39 @@ test_events(ts_if_1) ->
{failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}},
{?eh,tc_start,{ct_framework,error_in_suite}},
- {?eh,test_stats,{3,6,{3,7}}},
+ {?eh,test_stats,{3,5,{4,7}}},
{?eh,tc_start,{ct_framework,error_in_suite}},
- {?eh,test_stats,{3,7,{3,7}}},
+ {?eh,test_stats,{3,5,{5,7}}},
{?eh,tc_start,{ts_if_5_SUITE,init_per_suite}},
{?eh,tc_done,{ts_if_5_SUITE,init_per_suite,
{skipped,{require_failed_in_suite0,{not_available,undef_variable}}}}},
{?eh,tc_auto_skip,{ts_if_5_SUITE,my_test_case,
{require_failed_in_suite0,{not_available,undef_variable}}}},
- {?eh,test_stats,{3,7,{3,8}}},
+ {?eh,test_stats,{3,5,{5,8}}},
{?eh,tc_auto_skip,{ts_if_5_SUITE,end_per_suite,
{require_failed_in_suite0,{not_available,undef_variable}}}},
{?eh,tc_start,{ts_if_6_SUITE,tc1}},
{?eh,tc_done,{ts_if_6_SUITE,tc1,{failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}},
- {?eh,test_stats,{3,7,{4,8}}},
+ {?eh,test_stats,{3,5,{6,8}}},
{?eh,tc_start,{ts_if_7_SUITE,tc1}},
{?eh,tc_done,{ts_if_7_SUITE,tc1,ok}},
- {?eh,test_stats,{4,7,{4,8}}},
+ {?eh,test_stats,{4,5,{6,8}}},
{?eh,tc_start,{ts_if_8_SUITE,tc1}},
{?eh,tc_done,{ts_if_8_SUITE,tc1,{failed,{error,failed_on_purpose}}}},
- {?eh,test_stats,{4,8,{4,8}}},
+ {?eh,test_stats,{4,6,{6,8}}},
{?eh,tc_user_skip,{skipped_by_spec_1_SUITE,all,"should be skipped"}},
- {?eh,test_stats,{4,8,{5,8}}},
+ {?eh,test_stats,{4,6,{7,8}}},
{?eh,tc_start,{skipped_by_spec_2_SUITE,init_per_suite}},
{?eh,tc_done,{skipped_by_spec_2_SUITE,init_per_suite,ok}},
{?eh,tc_user_skip,{skipped_by_spec_2_SUITE,tc1,"should be skipped"}},
- {?eh,test_stats,{4,8,{6,8}}},
+ {?eh,test_stats,{4,6,{8,8}}},
{?eh,tc_start,{skipped_by_spec_2_SUITE,end_per_suite}},
{?eh,tc_done,{skipped_by_spec_2_SUITE,end_per_suite,ok}},
diff --git a/lib/test_server/src/test_server.erl b/lib/test_server/src/test_server.erl
index 98a2e21e21..1b344c6c7e 100644
--- a/lib/test_server/src/test_server.erl
+++ b/lib/test_server/src/test_server.erl
@@ -36,7 +36,8 @@
-export([capture_start/0,capture_stop/0,capture_get/0]).
-export([messages_get/0]).
-export([hours/1,minutes/1,seconds/1,sleep/1,adjusted_sleep/1,timecall/3]).
--export([timetrap_scale_factor/0,timetrap/1,timetrap_cancel/1,timetrap_cancel/0]).
+-export([timetrap_scale_factor/0,timetrap/1,get_timetrap_info/0,
+ timetrap_cancel/1,timetrap_cancel/0]).
-export([m_out_of_n/3,do_times/4,do_times/2]).
-export([call_crash/3,call_crash/4,call_crash/5]).
-export([temp_name/1]).
@@ -1257,11 +1258,15 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) ->
end.
do_end_tc_call(M,F, Loc, Res, Return) ->
+ IsSuite = case lists:reverse(atom_to_list(M)) of
+ [$E,$T,$I,$U,$S,$_|_] -> true;
+ _ -> false
+ end,
FwMod = os:getenv("TEST_SERVER_FRAMEWORK"),
{Mod,Func} =
if FwMod == M ; FwMod == "undefined"; FwMod == false ->
{M,F};
- is_list(Loc) and (length(Loc)>1) ->
+ (not IsSuite) and is_list(Loc) and (length(Loc)>1) ->
%% If failure in other module (M) than suite, try locate
%% suite name in Loc list and call end_tc with Suite:TestCase
%% instead of M:F.
@@ -1524,7 +1529,10 @@ get_loc(Pid) ->
process_info(Pid, [current_stacktrace,dictionary]),
lists:foreach(fun({Key,Val}) -> put(Key, Val) end, Dict),
Stk = [rewrite_loc_item(Loc) || Loc <- Stk0],
- put(test_server_loc, Stk),
+ case get(test_server_loc) of
+ undefined -> put(test_server_loc, Stk);
+ _ -> ok
+ end,
get_loc().
%% find the latest known Suite:Testcase
@@ -1856,7 +1864,9 @@ fail() ->
break(Comment) ->
case erase(test_server_timetraps) of
undefined -> ok;
- List -> lists:foreach(fun({Ref,_}) -> timetrap_cancel(Ref) end, List)
+ List -> lists:foreach(fun({Ref,_,_}) ->
+ timetrap_cancel(Ref)
+ end, List)
end,
io:format(user,
"\n\n\n--- SEMIAUTOMATIC TESTING ---"
@@ -1945,8 +1955,8 @@ timetrap1(Timeout, Scale) ->
TCPid = self(),
Ref = spawn_link(test_server_sup,timetrap,[Timeout,Scale,TCPid]),
case get(test_server_timetraps) of
- undefined -> put(test_server_timetraps,[{Ref,TCPid}]);
- List -> put(test_server_timetraps,[{Ref,TCPid}|List])
+ undefined -> put(test_server_timetraps,[{Ref,TCPid,{Timeout,Scale}}]);
+ List -> put(test_server_timetraps,[{Ref,TCPid,{Timeout,Scale}}|List])
end,
Ref.
@@ -2057,7 +2067,7 @@ timetrap_cancel(infinity) ->
timetrap_cancel(Handle) ->
case get(test_server_timetraps) of
undefined -> ok;
- [{Handle,_}] -> erase(test_server_timetraps);
+ [{Handle,_,_}] -> erase(test_server_timetraps);
Timers -> put(test_server_timetraps,
lists:keydelete(Handle, 1, Timers))
end,
@@ -2073,13 +2083,30 @@ timetrap_cancel() ->
ok;
Timers ->
case lists:keysearch(self(), 2, Timers) of
- {value,{Handle,_}} ->
+ {value,{Handle,_,_}} ->
timetrap_cancel(Handle);
_ ->
ok
end
end.
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%% get_timetrap_info() -> {Timeout,Scale} | undefined
+%%
+%% Read timetrap info for current test case
+get_timetrap_info() ->
+ case get(test_server_timetraps) of
+ undefined ->
+ undefined;
+ Timers ->
+ case lists:keysearch(self(), 2, Timers) of
+ {value,{_,_,Info}} ->
+ Info;
+ _ ->
+ undefined
+ end
+ end.
+
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% hours(N) -> Milliseconds
%% minutes(N) -> Milliseconds
diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl
index 642bb14c88..3c4b2ead8a 100644
--- a/lib/test_server/src/test_server_ctrl.erl
+++ b/lib/test_server/src/test_server_ctrl.erl
@@ -2725,21 +2725,27 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0,
"(configuration case ~w)", [What]);
(_) -> ok
end,
-
CfgProps = if StartConf ->
if Shuffle == undefined ->
[{tc_group_properties,Props}];
true ->
- [{tc_group_properties,[Shuffle|delete_shuffle(Props)]}]
+ [{tc_group_properties,
+ [Shuffle|delete_shuffle(Props)]}]
end;
not StartConf ->
{TcOk,TcSkip,TcFail} = get_tc_results(Status1),
[{tc_group_properties,get_props(Mode0)},
- {tc_group_result,[{ok,TcOk},{skipped,TcSkip},{failed,TcFail}]}]
+ {tc_group_result,[{ok,TcOk},
+ {skipped,TcSkip},
+ {failed,TcFail}]}]
end,
+ GroupPath = lists:flatmap(fun({_Ref,[],_T}) -> [];
+ ({_Ref,GrProps,_T}) -> [GrProps] end, Mode0),
ActualCfg =
- update_config(hd(Config), [{priv_dir,get(test_server_priv_dir)},
- {data_dir,get_data_dir(Mod)}] ++ CfgProps),
+ update_config(hd(Config),
+ [{priv_dir,get(test_server_priv_dir)},
+ {data_dir,get_data_dir(Mod)},
+ {tc_group_path,GroupPath} | CfgProps]),
CurrMode = curr_mode(Ref, Mode0, Mode),
ConfCaseResult = run_test_case(Ref, 0, Mod, Func, [ActualCfg], skip_init, target,
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-01 06:35:07 +0000