aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test/src
diff options
context:
space:
mode:
Diffstat (limited to 'lib/common_test/src')
-rw-r--r--lib/common_test/src/ct.erl36
-rw-r--r--lib/common_test/src/ct_framework.erl496
-rw-r--r--lib/common_test/src/ct_run.erl8
-rw-r--r--lib/common_test/src/ct_testspec.erl8
4 files changed, 427 insertions, 121 deletions
diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl
index 860bfb45ee..e0e82283c4 100644
--- a/lib/common_test/src/ct.erl
+++ b/lib/common_test/src/ct.erl
@@ -66,7 +66,7 @@
capture_start/0, capture_stop/0, capture_get/0, capture_get/1,
fail/1, fail/2, comment/1, comment/2,
testcases/2, userdata/2, userdata/3,
- timetrap/1, sleep/1]).
+ timetrap/1, get_timetrap_info/0, sleep/1]).
%% New API for manipulating with config handlers
-export([add_config/2, remove_config/2]).
@@ -763,7 +763,8 @@ userdata(TestDir, Suite) ->
get_userdata(Info, "suite/0")
end.
-get_userdata({'EXIT',{undef,_}}, Spec) ->
+get_userdata({'EXIT',{Undef,_}}, Spec) when Undef == undef;
+ Undef == function_clause ->
{error,list_to_atom(Spec ++ " is not defined")};
get_userdata({'EXIT',Reason}, Spec) ->
{error,{list_to_atom("error in " ++ Spec),Reason}};
@@ -779,16 +780,27 @@ get_userdata(_BadTerm, Spec) ->
{error,list_to_atom(Spec ++ " must return a list")}.
%%%-----------------------------------------------------------------
-%%% @spec userdata(TestDir, Suite, Case) -> TCUserData | {error,Reason}
+%%% @spec userdata(TestDir, Suite, GroupOrCase) -> TCUserData | {error,Reason}
%%% TestDir = string()
%%% Suite = atom()
-%%% Case = atom()
+%%% GroupOrCase = {group,GroupName} | atom()
+%%% GroupName = atom()
%%% TCUserData = [term()]
%%% Reason = term()
%%%
%%% @doc Returns any data specified with the tag <code>userdata</code>
-%%% in the list of tuples returned from <code>Suite:Case/0</code>.
-userdata(TestDir, Suite, Case) ->
+%%% in the list of tuples returned from <code>Suite:group(GroupName)</code>
+%%% or <code>Suite:Case()</code>.
+userdata(TestDir, Suite, {group,GroupName}) ->
+ case make_and_load(TestDir, Suite) of
+ E = {error,_} ->
+ E;
+ _ ->
+ Info = (catch apply(Suite, group, [GroupName])),
+ get_userdata(Info, "group("++atom_to_list(GroupName)++")")
+ end;
+
+userdata(TestDir, Suite, Case) when is_atom(Case) ->
case make_and_load(TestDir, Suite) of
E = {error,_} ->
E;
@@ -966,6 +978,18 @@ timetrap(Time) ->
test_server:timetrap(Time).
%%%-----------------------------------------------------------------
+%%% @spec get_timetrap_info() -> {Time,Scale}
+%%% Time = integer() | infinity
+%%% Scale = true | false
+%%%
+%%% @doc <p>Read info about the timetrap set for the current test case.
+%%% <c>Scale</c> indicates if Common Test will attempt to automatically
+%%% compensate timetraps for runtime delays introduced by e.g. tools like
+%%% cover.</p>
+get_timetrap_info() ->
+ test_server:get_timetrap_info().
+
+%%%-----------------------------------------------------------------
%%% @spec sleep(Time) -> ok
%%% Time = {hours,Hours} | {minutes,Mins} | {seconds,Secs} | Millisecs | infinity
%%% Hours = integer()
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index ee0162c5e3..c24a7c238b 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -36,6 +36,10 @@
-include("ct_event.hrl").
-include("ct_util.hrl").
+-define(val(Key, List), proplists:get_value(Key, List)).
+-define(val(Key, List, Def), proplists:get_value(Key, List, Def)).
+-define(rev(L), lists:reverse(L)).
+
%%%-----------------------------------------------------------------
%%% @spec init_tc(Mod,Func,Args) -> {ok,NewArgs} | {error,Reason} |
%%% {skip,Reason} | {auto_skip,Reason}
@@ -48,6 +52,8 @@
%%% @doc Test server framework callback, called by the test_server
%%% when a new test case is started.
init_tc(Mod,Func,Config) ->
+ %% in case Mod == ct_framework, lookup the suite name
+ Suite = get_suite_name(Mod, Config),
%% check if previous testcase was interpreted and has left
%% a "dead" trace window behind - if so, kill it
case ct_util:get_testdata(interpret) of
@@ -57,34 +63,36 @@ init_tc(Mod,Func,Config) ->
_ ->
ok
end,
-
%% check if we need to add defaults explicitly because
%% there's no init_per_suite exported from Mod
{InitFailed,DoInit} =
case ct_util:get_testdata(curr_tc) of
- {Mod,{suite0_failed,_}=Failure} ->
+ {Suite,{suite0_failed,_}=Failure} ->
{Failure,false};
- {Mod,_} ->
+ {?MODULE,_} -> % should not really happen
{false,false};
- _ when Func == init_per_suite ->
+ {Suite,_} -> % Func is not 1st case in suite
{false,false};
- _ ->
+ _ when Func == init_per_suite -> % defaults will be added anyway
+ {false,false};
+ _ -> % first case in suite
{false,true}
end,
case InitFailed of
false ->
- ct_util:set_testdata({curr_tc,{Mod,Func}}),
- case ct_util:read_suite_data({seq,Mod,Func}) of
+ ct_util:set_testdata({curr_tc,{Suite,Func}}),
+ case ct_util:read_suite_data({seq,Suite,Func}) of
undefined ->
init_tc1(Mod,Func,Config,DoInit);
Seq when is_atom(Seq) ->
- case ct_util:read_suite_data({seq,Mod,Seq}) of
+ case ct_util:read_suite_data({seq,Suite,Seq}) of
[Func|TCs] -> % this is the 1st case in Seq
%% make sure no cases in this seq are marked as failed
%% from an earlier execution in the same suite
- lists:foreach(fun(TC) ->
- ct_util:save_suite_data({seq,Mod,TC},Seq)
- end, TCs);
+ lists:foreach(
+ fun(TC) ->
+ ct_util:save_suite_data({seq,Suite,TC},Seq)
+ end, TCs);
_ ->
ok
end,
@@ -98,6 +106,17 @@ init_tc(Mod,Func,Config) ->
{skip,InitFailed}
end.
+init_tc1(?MODULE,error_in_suite,[Config0],_) when is_list(Config0) ->
+ ct_logs:init_tc(false),
+ ct_event:notify(#event{name=tc_start,
+ node=node(),
+ data={?MODULE,error_in_suite}}),
+ case ?val(error, Config0) of
+ undefined ->
+ {skip,"unknown_error_in_suite"};
+ Reason ->
+ {skip,Reason}
+ end;
init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) ->
Config1 =
case ct_util:read_suite_data(last_saved_config) of
@@ -122,35 +141,40 @@ init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) ->
%% release all name -> key bindings (once per suite)
ct_config:release_allocated()
end,
- TestCaseInfo =
- case catch apply(Mod,Func,[]) of
- Result when is_list(Result) -> Result;
- _ -> []
- end,
+
+ GroupPath = ?val(tc_group_path, Config, []),
+ AllGroups = [?val(tc_group_properties, Config, []) | GroupPath],
+
%% clear all config data default values set by previous
%% testcase info function (these should only survive the
%% testcase, not the whole suite)
- ct_config:delete_default_config(testcase),
- case add_defaults(Mod,Func,TestCaseInfo,DoInit) of
+ FuncSpec = group_or_func(Func,Config0),
+ if is_tuple(FuncSpec) -> % group
+ ok;
+ true ->
+ ct_config:delete_default_config(testcase)
+ end,
+ %% in case Mod == ct_framework, lookup the suite name
+ Suite = get_suite_name(Mod, Config),
+ case add_defaults(Mod,Func,AllGroups,DoInit) of
Error = {suite0_failed,_} ->
ct_logs:init_tc(false),
- FuncSpec = group_or_func(Func,Config0),
ct_event:notify(#event{name=tc_start,
node=node(),
data={Mod,FuncSpec}}),
- ct_util:set_testdata({curr_tc,{Mod,Error}}),
+ ct_util:set_testdata({curr_tc,{Suite,Error}}),
{error,Error};
{SuiteInfo,MergeResult} ->
case MergeResult of
{error,Reason} when DoInit == false ->
ct_logs:init_tc(false),
- FuncSpec = group_or_func(Func,Config0),
ct_event:notify(#event{name=tc_start,
node=node(),
data={Mod,FuncSpec}}),
{skip,Reason};
_ ->
- init_tc2(Mod,Func,SuiteInfo,MergeResult,Config,DoInit)
+ init_tc2(Mod,Func,SuiteInfo,MergeResult,
+ Config,DoInit)
end
end;
init_tc1(_Mod,_Func,Args,_DoInit) ->
@@ -203,8 +227,9 @@ init_tc2(Mod,Func,SuiteInfo,MergeResult,Config,DoInit) ->
ct_event:notify(#event{name=tc_start,
node=node(),
data={Mod,FuncSpec}}),
-
- case catch configure(MergedInfo1,MergedInfo1,SuiteInfo,{Func,DoInit},Config) of
+
+ case catch configure(MergedInfo1,MergedInfo1,SuiteInfo,
+ {FuncSpec,DoInit},Config) of
{suite0_failed,Reason} ->
ct_util:set_testdata({curr_tc,{Mod,{suite0_failed,{require,Reason}}}}),
{skip,{require_failed_in_suite0,Reason}};
@@ -212,7 +237,7 @@ init_tc2(Mod,Func,SuiteInfo,MergeResult,Config,DoInit) ->
{auto_skip,{require_failed,Reason}};
{'EXIT',Reason} ->
{auto_skip,Reason};
- {ok, FinalConfig} ->
+ {ok,FinalConfig} ->
case MergeResult of
{error,Reason} ->
%% suite0 configure finished now, report that
@@ -241,19 +266,20 @@ ct_suite_init(Mod, Func, [Config]) when is_list(Config) ->
Else
end.
-add_defaults(Mod,Func,FuncInfo,DoInit) ->
- case (catch Mod:suite()) of
+add_defaults(Mod,Func, GroupPath, DoInit) ->
+ Suite = get_suite_name(Mod, GroupPath),
+ case (catch Suite:suite()) of
{'EXIT',{undef,_}} ->
- SuiteInfo = merge_with_suite_defaults(Mod,[]),
+ SuiteInfo = merge_with_suite_defaults(Suite,[]),
SuiteInfoNoCTH = [I || I <- SuiteInfo, element(1,I) =/= ct_hooks],
- case add_defaults1(Mod,Func,FuncInfo,SuiteInfoNoCTH,DoInit) of
+ case add_defaults1(Mod,Func, GroupPath, SuiteInfoNoCTH, DoInit) of
Error = {error,_} -> {SuiteInfo,Error};
MergedInfo -> {SuiteInfo,MergedInfo}
end;
{'EXIT',Reason} ->
ErrStr = io_lib:format("~n*** ERROR *** "
"~w:suite/0 failed: ~p~n",
- [Mod,Reason]),
+ [Suite,Reason]),
io:format(ErrStr, []),
io:format(user, ErrStr, []),
{suite0_failed,{exited,Reason}};
@@ -262,18 +288,18 @@ add_defaults(Mod,Func,FuncInfo,DoInit) ->
(_) -> false
end, SuiteInfo) of
true ->
- SuiteInfo1 = merge_with_suite_defaults(Mod,SuiteInfo),
+ SuiteInfo1 = merge_with_suite_defaults(Suite, SuiteInfo),
SuiteInfoNoCTH = [I || I <- SuiteInfo1,
element(1,I) =/= ct_hooks],
- case add_defaults1(Mod,Func,FuncInfo,
- SuiteInfoNoCTH,DoInit) of
+ case add_defaults1(Mod,Func, GroupPath,
+ SuiteInfoNoCTH, DoInit) of
Error = {error,_} -> {SuiteInfo1,Error};
MergedInfo -> {SuiteInfo1,MergedInfo}
end;
false ->
ErrStr = io_lib:format("~n*** ERROR *** "
"Invalid return value from "
- "~w:suite/0: ~p~n", [Mod,SuiteInfo]),
+ "~w:suite/0: ~p~n", [Suite,SuiteInfo]),
io:format(ErrStr, []),
io:format(user, ErrStr, []),
{suite0_failed,bad_return_value}
@@ -281,57 +307,178 @@ add_defaults(Mod,Func,FuncInfo,DoInit) ->
SuiteInfo ->
ErrStr = io_lib:format("~n*** ERROR *** "
"Invalid return value from "
- "~w:suite/0: ~p~n", [Mod,SuiteInfo]),
+ "~w:suite/0: ~p~n", [Suite,SuiteInfo]),
io:format(ErrStr, []),
io:format(user, ErrStr, []),
{suite0_failed,bad_return_value}
end.
-add_defaults1(_Mod,init_per_suite,[],SuiteInfo,_DoInit) ->
- SuiteInfo;
-
-add_defaults1(Mod,Func,FuncInfo,SuiteInfo,DoInit) ->
- %% mustn't re-require suite variables in test case info function,
- %% can result in weird behaviour (suite values get overwritten)
+add_defaults1(Mod,Func, GroupPath, SuiteInfo, DoInit) ->
+ Suite = get_suite_name(Mod, GroupPath),
+ %% GroupPathInfo (for subgroup on level X) =
+ %% [LevelXGroupInfo, LevelX-1GroupInfo, ..., TopLevelGroupInfo]
+ GroupPathInfo =
+ lists:map(fun(GroupProps) ->
+ Name = ?val(name, GroupProps),
+ case catch Suite:group(Name) of
+ GrInfo when is_list(GrInfo) -> GrInfo;
+ _ -> []
+ end
+ end, GroupPath),
+ Args = if Func == init_per_group; Func == ct_init_per_group;
+ Func == end_per_group; Func == ct_end_per_group ->
+ [?val(name, hd(GroupPath))];
+ true ->
+ []
+ end,
+ TestCaseInfo =
+ case catch apply(Mod,Func,Args) of
+ TCInfo when is_list(TCInfo) -> TCInfo;
+ _ -> []
+ end,
+ %% let test case info (also for all config funcs) override group info,
+ %% and lower level group info override higher level info
+ TCAndGroupInfo = [TestCaseInfo | remove_info_in_prev(TestCaseInfo,
+ GroupPathInfo)],
+ %% find and save require terms found in suite info
SuiteReqs =
[SDDef || SDDef <- SuiteInfo,
((require == element(1,SDDef)) or
(default_config == element(1,SDDef)))],
- FuncReqs =
- [FIDef || FIDef <- FuncInfo,
- require == element(1,FIDef)],
- case [element(2,Clash) || Clash <- SuiteReqs,
- require == element(1, Clash),
- true == lists:keymember(element(2,Clash),2,
- FuncReqs)] of
+ case check_for_clashes(TestCaseInfo, GroupPathInfo, SuiteReqs) of
[] ->
- add_defaults2(Mod,Func,FuncInfo,SuiteInfo,SuiteReqs,DoInit);
+ add_defaults2(Mod,Func, TCAndGroupInfo,SuiteInfo,SuiteReqs, DoInit);
Clashes ->
{error,{config_name_already_in_use,Clashes}}
end.
-add_defaults2(Mod,init_per_suite,IPSInfo,SuiteInfo,SuiteReqs,false) ->
- %% not common practise to use a test case info function for
- %% init_per_suite (usually handled by suite/0), but let's support
- %% it just in case...
- add_defaults2(Mod,init_per_suite,IPSInfo,SuiteInfo,SuiteReqs,true);
-
-add_defaults2(_Mod,_Func,FuncInfo,SuiteInfo,_,false) ->
- %% include require elements from test case info, but not from suite/0
- %% (since we've already required those vars)
- FuncInfo ++
- [SFDef || SFDef <- SuiteInfo,
- require /= element(1,SFDef),
- false == lists:keymember(element(1,SFDef),1,FuncInfo)];
-
-add_defaults2(_Mod,_Func,FuncInfo,SuiteInfo,SuiteReqs,true) ->
- %% We must include require elements from suite/0 here since
- %% there's no init_per_suite call before this first test case.
- %% Let other test case info elements override those from suite/0.
- FuncInfo ++ SuiteReqs ++
- [SDDef || SDDef <- SuiteInfo,
- require /= element(1,SDDef),
- false == lists:keymember(element(1,SDDef),1,FuncInfo)].
+get_suite_name(?MODULE, [Cfg|_]) when is_list(Cfg), Cfg /= [] ->
+ get_suite_name(?MODULE, Cfg);
+
+get_suite_name(?MODULE, Cfg) when is_list(Cfg), Cfg /= [] ->
+ case ?val(tc_group_properties, Cfg) of
+ undefined ->
+ case ?val(suite, Cfg) of
+ undefined -> ?MODULE;
+ Suite -> Suite
+ end;
+ GrProps ->
+ case ?val(suite, GrProps) of
+ undefined -> ?MODULE;
+ Suite -> Suite
+ end
+ end;
+get_suite_name(Mod, _) ->
+ Mod.
+
+%% Check that alias names are not already in use
+check_for_clashes(TCInfo, GrPathInfo, SuiteInfo) ->
+ {CurrGrInfo,SearchIn} = case GrPathInfo of
+ [] -> {[],[SuiteInfo]};
+ [Curr|Path] -> {Curr,[SuiteInfo|Path]}
+ end,
+ ReqNames = fun(Info) -> [element(2,R) || R <- Info,
+ size(R) == 3,
+ require == element(1,R)]
+ end,
+ ExistingNames = lists:flatten([ReqNames(L) || L <- SearchIn]),
+ CurrGrReqNs = ReqNames(CurrGrInfo),
+ GrClashes = [Name || Name <- CurrGrReqNs,
+ true == lists:member(Name, ExistingNames)],
+ AllReqNs = CurrGrReqNs ++ ExistingNames,
+ TCClashes = [Name || Name <- ReqNames(TCInfo),
+ true == lists:member(Name, AllReqNs)],
+ TCClashes ++ GrClashes.
+
+%% Delete the info terms in Terms from all following info lists
+remove_info_in_prev(Terms, [[] | Rest]) ->
+ [[] | remove_info_in_prev(Terms, Rest)];
+remove_info_in_prev(Terms, [Info | Rest]) ->
+ UniqueInInfo = [U || U <- Info,
+ ((timetrap == element(1,U)) and
+ (not lists:keymember(timetrap,1,Terms))) or
+ ((require == element(1,U)) and
+ (not lists:member(U,Terms))) or
+ ((default_config == element(1,U)) and
+ (not keysmember([default_config,1,
+ element(2,U),2], Terms)))],
+ OtherTermsInInfo = [T || T <- Info,
+ timetrap /= element(1,T),
+ require /= element(1,T),
+ default_config /= element(1,T),
+ false == lists:keymember(element(1,T),1,
+ Terms)],
+ KeptInfo = UniqueInInfo ++ OtherTermsInInfo,
+ [KeptInfo | remove_info_in_prev(Terms ++ KeptInfo, Rest)];
+remove_info_in_prev(_, []) ->
+ [].
+
+keysmember([Key,Pos|Next], List) ->
+ case [Elem || Elem <- List, Key == element(Pos,Elem)] of
+ [] -> false;
+ Found -> keysmember(Next, Found)
+ end;
+keysmember([], _) -> true.
+
+
+add_defaults2(Mod,init_per_suite, IPSInfo, SuiteInfo,SuiteReqs, false) ->
+ add_defaults2(Mod,init_per_suite, IPSInfo, SuiteInfo,SuiteReqs, true);
+
+add_defaults2(_Mod,IPG, IPGAndGroupInfo, SuiteInfo,SuiteReqs, DoInit) when
+ IPG == init_per_group ; IPG == ct_init_per_group ->
+ %% If DoInit == true, we have to process the suite() list, otherwise
+ %% it has already been handled (see clause for init_per_suite)
+ case DoInit of
+ true ->
+ %% note: we know for sure this is a top level group
+ Info = lists:flatten([IPGAndGroupInfo, SuiteReqs]),
+ Info ++ remove_info_in_prev(Info, [SuiteInfo]);
+ false ->
+ SuiteInfo1 =
+ remove_info_in_prev(lists:flatten([IPGAndGroupInfo,
+ SuiteReqs]), [SuiteInfo]),
+ %% don't require terms in prev groups (already processed)
+ case IPGAndGroupInfo of
+ [IPGInfo] ->
+ lists:flatten([IPGInfo,SuiteInfo1]);
+ [IPGInfo | [CurrGroupInfo | PrevGroupInfo]] ->
+ PrevGroupInfo1 = delete_require_terms(PrevGroupInfo),
+ lists:flatten([IPGInfo,CurrGroupInfo,PrevGroupInfo1,
+ SuiteInfo1])
+ end
+ end;
+
+add_defaults2(_Mod,_Func, TCAndGroupInfo, SuiteInfo,SuiteReqs, false) ->
+ %% Include require elements from test case info and current group,
+ %% but not from previous groups or suite/0 (since we've already required
+ %% those vars). Let test case info elements override group and suite
+ %% info elements.
+ SuiteInfo1 = remove_info_in_prev(lists:flatten([TCAndGroupInfo,
+ SuiteReqs]), [SuiteInfo]),
+ %% don't require terms in prev groups (already processed)
+ case TCAndGroupInfo of
+ [TCInfo] ->
+ lists:flatten([TCInfo,SuiteInfo1]);
+ [TCInfo | [CurrGroupInfo | PrevGroupInfo]] ->
+ PrevGroupInfo1 = delete_require_terms(PrevGroupInfo),
+ lists:flatten([TCInfo,CurrGroupInfo,PrevGroupInfo1,
+ SuiteInfo1])
+ end;
+
+add_defaults2(_Mod,_Func, TCInfo, SuiteInfo,SuiteReqs, true) ->
+ %% Here we have to process the suite info list also (no call to
+ %% init_per_suite before this first test case). This TC can't belong
+ %% to a group, or the clause for (ct_)init_per_group would've caught this.
+ Info = lists:flatten([TCInfo, SuiteReqs]),
+ lists:flatten([Info,remove_info_in_prev(Info, [SuiteInfo])]).
+
+delete_require_terms([Info | Prev]) ->
+ Info1 = [T || T <- Info,
+ require /= element(1,T),
+ default_config /= element(1,T)],
+ [Info1 | delete_require_terms(Prev)];
+delete_require_terms([]) ->
+ [].
merge_with_suite_defaults(Mod,SuiteInfo) ->
case lists:keysearch(suite_defaults,1,Mod:module_info(attributes)) of
@@ -355,16 +502,17 @@ timetrap_first([Trap = {timetrap,_} | Rest],Info,Found) ->
timetrap_first([Other | Rest],Info,Found) ->
timetrap_first(Rest,[Other | Info],Found);
timetrap_first([],Info,[]) ->
- [{timetrap,{minutes,30}} | lists:reverse(Info)];
+ [{timetrap,{minutes,30}} | ?rev(Info)];
timetrap_first([],Info,Found) ->
- lists:reverse(Found) ++ lists:reverse(Info).
+ ?rev(Found) ++ ?rev(Info).
configure([{require,Required}|Rest],Info,SuiteInfo,Scope,Config) ->
case ct:require(Required) of
ok ->
configure(Rest,Info,SuiteInfo,Scope,Config);
Error = {error,Reason} ->
- case required_default('_UNDEF',Required,Info,SuiteInfo,Scope) of
+ case required_default('_UNDEF',Required,Info,
+ SuiteInfo,Scope) of
ok ->
configure(Rest,Info,SuiteInfo,Scope,Config);
_ ->
@@ -406,18 +554,24 @@ configure([],_,_,_,Config) ->
{ok,[Config]}.
%% the require element in Info may come from suite/0 and
-%% should be scoped 'suite', or come from the testcase info
-%% function and should then be scoped 'testcase'
-required_default(Name,Key,Info,SuiteInfo,{Func,true}) ->
+%% should be scoped 'suite', or come from the group info
+%% function and be scoped 'group', or come from the testcase
+%% info function and then be scoped 'testcase'
+
+required_default(Name,Key,Info,SuiteInfo,{FuncSpec,true}) ->
case try_set_default(Name,Key,SuiteInfo,suite) of
ok ->
ok;
_ ->
- required_default(Name,Key,Info,[],{Func,false})
+ required_default(Name,Key,Info,[],{FuncSpec,false})
end;
required_default(Name,Key,Info,_,{init_per_suite,_}) ->
try_set_default(Name,Key,Info,suite);
-required_default(Name,Key,Info,_,_) ->
+required_default(Name,Key,Info,_,{{init_per_group,GrName,_},_}) ->
+ try_set_default(Name,Key,Info,{group,GrName});
+required_default(Name,Key,Info,_,{{ct_init_per_group,GrName,_},_}) ->
+ try_set_default(Name,Key,Info,{group,GrName});
+required_default(Name,Key,Info,_,_FuncSpec) ->
try_set_default(Name,Key,Info,testcase).
try_set_default(Name,Key,Info,Where) ->
@@ -484,7 +638,11 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) ->
ct_util:delete_suite_data(last_saved_config),
FuncSpec =
case group_or_func(Func,Args) of
- {_,GroupName,_Props} = Group ->
+ {_,GroupName,_Props} = Group ->
+ if Func == end_per_group; Func == ct_end_per_group ->
+ ct_config:delete_default_config({group,GroupName});
+ true -> ok
+ end,
case lists:keysearch(save_config,1,Args) of
{value,{save_config,SaveConfig}} ->
ct_util:save_suite_data(
@@ -703,12 +861,14 @@ mark_as_failed1(_,_,_,[]) ->
ok.
group_or_func(Func, Config) when Func == init_per_group;
- Func == end_per_group ->
- case proplists:get_value(tc_group_properties,Config) of
+ Func == end_per_group;
+ Func == ct_init_per_group;
+ Func == ct_end_per_group ->
+ case ?val(tc_group_properties, Config) of
undefined ->
{Func,unknown,[]};
GrProps ->
- GrName = proplists:get_value(name,GrProps),
+ GrName = ?val(name,GrProps),
{Func,GrName,proplists:delete(name,GrProps)}
end;
group_or_func(Func, _Config) ->
@@ -732,7 +892,7 @@ get_suite(Mod, all) ->
%% (and only) test case so we can report Error properly
[{?MODULE,error_in_suite,[[Error]]}];
ConfTests ->
- get_all(Mod, ConfTests)
+ get_all(Mod, ConfTests)
end;
_ ->
E = "Bad return value from "++atom_to_list(Mod)++":groups/0",
@@ -746,7 +906,7 @@ get_suite(Mod, all) ->
%% group
get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) ->
- Name = proplists:get_value(name, Props),
+ Name = ?val(name, Props),
case catch apply(Mod, groups, []) of
{'EXIT',_} ->
[Group];
@@ -764,14 +924,25 @@ get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) ->
%% a *subgroup* specified *only* as skipped (and not
%% as an explicit test) should not be returned, or
%% init/end functions for top groups will be executed
- case catch proplists:get_value(name, element(2, hd(ConfTests))) of
+ case catch ?val(name, element(2, hd(ConfTests))) of
Name -> % top group
delete_subs(ConfTests, ConfTests);
_ ->
[]
end;
false ->
- delete_subs(ConfTests, ConfTests)
+ ConfTests1 = delete_subs(ConfTests, ConfTests),
+ case ?val(override, Props) of
+ undefined ->
+ ConfTests1;
+ [] ->
+ ConfTests1;
+ ORSpec ->
+ ORSpec1 = if is_tuple(ORSpec) -> [ORSpec];
+ true -> ORSpec end,
+ search_and_override(ConfTests1,
+ ORSpec1, Mod)
+ end
end
end;
_ ->
@@ -793,13 +964,12 @@ get_all_cases(Suite) ->
{error,Error};
Tests ->
Cases = get_all_cases1(Suite, Tests),
- lists:reverse(
- lists:foldl(fun(TC, TCs) ->
- case lists:member(TC, TCs) of
+ ?rev(lists:foldl(fun(TC, TCs) ->
+ case lists:member(TC, TCs) of
true -> TCs;
- false -> [TC | TCs]
- end
- end, [], Cases))
+ false -> [TC | TCs]
+ end
+ end, [], Cases))
end.
get_all_cases1(Suite, [{conf,_Props,_Init,GrTests,_End} | Tests]) ->
@@ -918,14 +1088,14 @@ delete_subs([], All) ->
All.
delete_conf({conf,Props,_,_,_}, Confs) ->
- Name = proplists:get_value(name, Props),
+ Name = ?val(name, Props),
[Conf || Conf = {conf,Props0,_,_,_} <- Confs,
- Name =/= proplists:get_value(name, Props0)].
+ Name =/= ?val(name, Props0)].
is_sub({conf,Props,_,_,_}=Conf, [{conf,_,_,Tests,_} | Confs]) ->
- Name = proplists:get_value(name, Props),
+ Name = ?val(name, Props),
case lists:any(fun({conf,Props0,_,_,_}) ->
- case proplists:get_value(name, Props0) of
+ case ?val(name, Props0) of
N when N == Name ->
true;
_ ->
@@ -1078,29 +1248,116 @@ expand_groups([H | T], ConfTests, Mod) ->
expand_groups([], _ConfTests, _Mod) ->
[];
expand_groups({group,Name}, ConfTests, Mod) ->
- FindConf =
- fun({conf,Props,_,_,_}) ->
- case proplists:get_value(name, Props) of
- Name -> true;
- _ -> false
+ expand_groups({group,Name,default,[]}, ConfTests, Mod);
+expand_groups({group,Name,default}, ConfTests, Mod) ->
+ expand_groups({group,Name,default,[]}, ConfTests, Mod);
+expand_groups({group,Name,ORProps}, ConfTests, Mod) when is_list(ORProps) ->
+ expand_groups({group,Name,ORProps,[]}, ConfTests, Mod);
+expand_groups({group,Name,ORProps,SubORSpec}, ConfTests, Mod) ->
+ FindConf =
+ fun(Conf = {conf,Props,Init,Ts,End}) ->
+ case ?val(name, Props) of
+ Name when ORProps == default ->
+ [Conf];
+ Name ->
+ [{conf,[{name,Name}|ORProps],Init,Ts,End}];
+ _ ->
+ []
end
end,
- case lists:filter(FindConf, ConfTests) of
- [ConfTest|_] ->
- expand_groups(ConfTest, ConfTests, Mod);
+ case lists:flatmap(FindConf, ConfTests) of
[] ->
- E = "Invalid reference to group "++
- atom_to_list(Name)++" in "++
- atom_to_list(Mod)++":all/0",
- throw({error,list_to_atom(E)})
+ throw({error,invalid_ref_msg(Name, Mod)});
+ Matching when SubORSpec == [] ->
+ Matching;
+ Matching ->
+ override_props(Matching, SubORSpec, Name,Mod)
end;
expand_groups(SeqOrTC, _ConfTests, _Mod) ->
SeqOrTC.
+%% search deep for the matching conf test and modify it and any
+%% sub tests according to the override specification
+search_and_override([Conf = {conf,Props,Init,Tests,End}], ORSpec, Mod) ->
+ Name = ?val(name, Props),
+ case lists:keysearch(Name, 1, ORSpec) of
+ {value,{Name,default}} ->
+ [Conf];
+ {value,{Name,ORProps}} ->
+ [{conf,[{name,Name}|ORProps],Init,Tests,End}];
+ {value,{Name,default,[]}} ->
+ [Conf];
+ {value,{Name,default,SubORSpec}} ->
+ override_props([Conf], SubORSpec, Name,Mod);
+ {value,{Name,ORProps,SubORSpec}} ->
+ override_props([{conf,[{name,Name}|ORProps],
+ Init,Tests,End}], SubORSpec, Name,Mod);
+ _ ->
+ [{conf,Props,Init,search_and_override(Tests,ORSpec,Mod),End}]
+ end.
+
+%% Modify the Tests element according to the override specification
+override_props([{conf,Props,Init,Tests,End} | Confs], SubORSpec, Name,Mod) ->
+ {Subs,SubORSpec1} = override_sub_props(Tests, [], SubORSpec, Mod),
+ [{conf,Props,Init,Subs,End} | override_props(Confs, SubORSpec1, Name,Mod)];
+override_props([], [], _,_) ->
+ [];
+override_props([], SubORSpec, Name,Mod) ->
+ Es = [invalid_ref_msg(Name, element(1,Spec), Mod) || Spec <- SubORSpec],
+ throw({error,Es}).
+
+override_sub_props([], New, ORSpec, _) ->
+ {?rev(New),ORSpec};
+override_sub_props([T = {conf,Props,Init,Tests,End} | Ts],
+ New, ORSpec, Mod) ->
+ Name = ?val(name, Props),
+ case lists:keysearch(Name, 1, ORSpec) of
+ {value,Spec} -> % group found in spec
+ Props1 =
+ case element(2, Spec) of
+ default -> Props;
+ ORProps -> [{name,Name} | ORProps]
+ end,
+ case catch element(3, Spec) of
+ Undef when Undef == [] ; 'EXIT' == element(1, Undef) ->
+ override_sub_props(Ts, [{conf,Props1,Init,Tests,End} | New],
+ lists:keydelete(Name, 1, ORSpec), Mod);
+ SubORSpec when is_list(SubORSpec) ->
+ case override_sub_props(Tests, [], SubORSpec, Mod) of
+ {Subs,[]} ->
+ override_sub_props(Ts, [{conf,Props1,Init,
+ Subs,End} | New],
+ lists:keydelete(Name, 1, ORSpec),
+ Mod);
+ {_,NonEmptySpec} ->
+ Es = [invalid_ref_msg(Name, element(1, GrRef),
+ Mod) || GrRef <- NonEmptySpec],
+ throw({error,Es})
+ end;
+ BadGrSpec ->
+ throw({error,{invalid_form,BadGrSpec}})
+ end;
+ _ -> % not a group in spec
+ override_sub_props(Ts, [T | New], ORSpec, Mod)
+ end;
+override_sub_props([TC | Ts], New, ORSpec, Mod) ->
+ override_sub_props(Ts, [TC | New], ORSpec, Mod).
+
+invalid_ref_msg(Name, Mod) ->
+ E = "Invalid reference to group "++
+ atom_to_list(Name)++" in "++
+ atom_to_list(Mod)++":all/0",
+ list_to_atom(E).
+
+invalid_ref_msg(Name0, Name1, Mod) ->
+ E = "Invalid reference to group "++
+ atom_to_list(Name1)++" from "++atom_to_list(Name0)++
+ " in "++atom_to_list(Mod)++":all/0",
+ list_to_atom(E).
%%!============================================================
%%! The support for sequences by means of using sequences/0
-%%! will be removed in OTP R14. The code below is only kept
+%%! will be removed in OTP R15. The code below is only kept
%%! for backwards compatibility. From OTP R13 groups with
%%! sequence property should be used instead!
%%!============================================================
@@ -1234,8 +1491,8 @@ report(What,Data) ->
loginfo ->
%% logfiles and direcories have been created for a test and the
%% top level test index page needs to be refreshed
- TestName = filename:basename(proplists:get_value(topdir, Data), ".logs"),
- RunDir = proplists:get_value(rundir, Data),
+ TestName = filename:basename(?val(topdir, Data), ".logs"),
+ RunDir = ?val(rundir, Data),
ct_logs:make_all_suites_index({TestName,RunDir}),
ok;
tests_start ->
@@ -1341,11 +1598,24 @@ report(What,Data) ->
node=node(),
data=Data}),
ct_hooks:on_tc_skip(What, Data),
- if Case /= end_per_suite, Case /= end_per_group ->
+ if Case /= end_per_suite,
+ Case /= end_per_group,
+ Case /= ct_end_per_group ->
add_to_stats(auto_skipped);
true ->
ok
end;
+ framework_error ->
+ case Data of
+ {{M,F},E} ->
+ ct_event:sync_notify(#event{name=tc_done,
+ node=node(),
+ data={M,F,{framework_error,E}}});
+ _ ->
+ ct_event:sync_notify(#event{name=tc_done,
+ node=node(),
+ data=Data})
+ end;
_ ->
ok
end,
diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index 0a9bb5af67..05b10bca32 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -1660,6 +1660,14 @@ final_tests1([{TestDir,Suite,GrsOrCs}|Tests], Final, Skip, Bad) when
({skipped,Group,TCs}) ->
[ct_framework:make_conf(TestDir, Suite,
Group, [skipped], TCs)];
+ ({GrSpec = {Group,_},TCs}) ->
+ Props = [{override,GrSpec}],
+ [ct_framework:make_conf(TestDir, Suite,
+ Group, Props, TCs)];
+ ({GrSpec = {Group,_,_},TCs}) ->
+ Props = [{override,GrSpec}],
+ [ct_framework:make_conf(TestDir, Suite,
+ Group, Props, TCs)];
({Group,TCs}) ->
[ct_framework:make_conf(TestDir, Suite,
Group, [], TCs)];
diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl
index 317910d5c8..b68cbd3aa1 100644
--- a/lib/common_test/src/ct_testspec.erl
+++ b/lib/common_test/src/ct_testspec.erl
@@ -878,7 +878,11 @@ separate([],_,_,_) ->
%% {Suite2,[GrOrCase21,GrOrCase22,...]},...]}
%% {{Node,Dir},[{Suite1,{skip,Cmt}},
%% {Suite2,[{GrOrCase21,{skip,Cmt}},GrOrCase22,...]},...]}
-%% GrOrCase = {GroupName,[Case1,Case2,...]} | Case
+%% GrOrCase = {GroupSpec,[Case1,Case2,...]} | Case
+%% GroupSpec = {GroupName,OverrideProps} |
+%% {GroupName,OverrideProps,SubGroupSpec}
+%% OverrideProps = Props | default
+%% SubGroupSpec = GroupSpec | []
insert_suites(Node,Dir,[S|Ss],Tests, MergeTests) ->
Tests1 = insert_cases(Node,Dir,S,all,Tests,MergeTests),
@@ -889,7 +893,7 @@ insert_suites(Node,Dir,S,Tests,MergeTests) ->
insert_suites(Node,Dir,[S],Tests,MergeTests).
insert_groups(Node,Dir,Suite,Group,Cases,Tests,MergeTests)
- when is_atom(Group) ->
+ when is_atom(Group); is_tuple(Group) ->
insert_groups(Node,Dir,Suite,[Group],Cases,Tests,MergeTests);
insert_groups(Node,Dir,Suite,Groups,Cases,Tests,false) when
((Cases == all) or is_list(Cases)) and is_list(Groups) ->