aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test/src
diff options
context:
space:
mode:
Diffstat (limited to 'lib/common_test/src')
-rw-r--r--lib/common_test/src/ct_framework.erl87
-rw-r--r--lib/common_test/src/ct_groups.erl121
-rw-r--r--lib/common_test/src/ct_hooks.erl24
-rw-r--r--lib/common_test/src/ct_run.erl7
-rw-r--r--lib/common_test/src/ct_testspec.erl109
-rw-r--r--lib/common_test/src/cth_log_redirect.erl9
-rw-r--r--lib/common_test/src/test_server.erl216
7 files changed, 381 insertions, 192 deletions
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index 8fb7a03bb0..8e25bfece0 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -52,9 +52,23 @@
%%%
%%% @doc Test server framework callback, called by the test_server
%%% when a new test case is started.
-init_tc(Mod,Func,Config) ->
+init_tc(Mod,EPTC={end_per_testcase,_},[Config]) ->
%% in case Mod == ct_framework, lookup the suite name
Suite = get_suite_name(Mod, Config),
+ case ct_hooks:init_tc(Suite,EPTC,Config) of
+ NewConfig when is_list(NewConfig) ->
+ {ok,[NewConfig]};
+ Other->
+ Other
+ end;
+
+init_tc(Mod,Func0,Args) ->
+ %% in case Mod == ct_framework, lookup the suite name
+ Suite = get_suite_name(Mod, Args),
+ {Func,HookFunc} = case Func0 of
+ {init_per_testcase,F} -> {F,Func0};
+ _ -> {Func0,Func0}
+ end,
%% check if previous testcase was interpreted and has left
%% a "dead" trace window behind - if so, kill it
@@ -86,7 +100,7 @@ init_tc(Mod,Func,Config) ->
end, [create]),
case ct_util:read_suite_data({seq,Suite,Func}) of
undefined ->
- init_tc1(Mod,Suite,Func,Config);
+ init_tc1(Mod,Suite,Func,HookFunc,Args);
Seq when is_atom(Seq) ->
case ct_util:read_suite_data({seq,Suite,Seq}) of
[Func|TCs] -> % this is the 1st case in Seq
@@ -102,27 +116,27 @@ init_tc(Mod,Func,Config) ->
_ ->
ok
end,
- init_tc1(Mod,Suite,Func,Config);
+ init_tc1(Mod,Suite,Func,HookFunc,Args);
{failed,Seq,BadFunc} ->
{auto_skip,{sequence_failed,Seq,BadFunc}}
end
end
- end.
+ end.
-init_tc1(?MODULE,_,error_in_suite,[Config0]) when is_list(Config0) ->
+init_tc1(?MODULE,_,error_in_suite,_,[Config0]) when is_list(Config0) ->
ct_logs:init_tc(false),
ct_event:notify(#event{name=tc_start,
node=node(),
data={?MODULE,error_in_suite}}),
- ct_suite_init(?MODULE, error_in_suite, [], Config0),
- case ?val(error, Config0) of
+ ct_suite_init(?MODULE,error_in_suite,[],Config0),
+ case ?val(error,Config0) of
undefined ->
{fail,"unknown_error_in_suite"};
Reason ->
{fail,Reason}
end;
-init_tc1(Mod,Suite,Func,[Config0]) when is_list(Config0) ->
+init_tc1(Mod,Suite,Func,HookFunc,[Config0]) when is_list(Config0) ->
Config1 =
case ct_util:read_suite_data(last_saved_config) of
{{Suite,LastFunc},SavedConfig} -> % last testcase
@@ -156,11 +170,13 @@ init_tc1(Mod,Suite,Func,[Config0]) when is_list(Config0) ->
%% testcase info function (these should only survive the
%% testcase, not the whole suite)
FuncSpec = group_or_func(Func,Config0),
- if is_tuple(FuncSpec) -> % group
- ok;
- true ->
- ct_config:delete_default_config(testcase)
- end,
+ HookFunc1 =
+ if is_tuple(FuncSpec) -> % group
+ FuncSpec;
+ true ->
+ ct_config:delete_default_config(testcase),
+ HookFunc
+ end,
Initialize = fun() ->
ct_logs:init_tc(false),
ct_event:notify(#event{name=tc_start,
@@ -184,14 +200,15 @@ init_tc1(Mod,Suite,Func,[Config0]) when is_list(Config0) ->
Initialize(),
{fail,Reason};
_ ->
- init_tc2(Mod,Suite,Func,SuiteInfo,MergeResult,Config)
+ init_tc2(Mod,Suite,Func,HookFunc1,
+ SuiteInfo,MergeResult,Config)
end
end;
-init_tc1(_Mod,_Suite,_Func,Args) ->
+init_tc1(_Mod,_Suite,_Func,_HookFunc,Args) ->
{ok,Args}.
-init_tc2(Mod,Suite,Func,SuiteInfo,MergeResult,Config) ->
+init_tc2(Mod,Suite,Func,HookFunc,SuiteInfo,MergeResult,Config) ->
%% timetrap must be handled before require
MergedInfo = timetrap_first(MergeResult, [], []),
%% tell logger to use specified style sheet
@@ -238,7 +255,7 @@ init_tc2(Mod,Suite,Func,SuiteInfo,MergeResult,Config) ->
{ok,PostInitHook,Config1} ->
case get('$test_server_framework_test') of
undefined ->
- ct_suite_init(Suite, FuncSpec, PostInitHook, Config1);
+ ct_suite_init(Suite,HookFunc,PostInitHook,Config1);
Fun ->
PostInitHookResult = do_post_init_hook(PostInitHook,
Config1),
@@ -251,16 +268,16 @@ init_tc2(Mod,Suite,Func,SuiteInfo,MergeResult,Config) ->
end
end.
-ct_suite_init(Suite, FuncSpec, PostInitHook, Config) when is_list(Config) ->
- case ct_hooks:init_tc(Suite, FuncSpec, Config) of
+ct_suite_init(Suite,HookFunc,PostInitHook,Config) when is_list(Config) ->
+ case ct_hooks:init_tc(Suite,HookFunc,Config) of
NewConfig when is_list(NewConfig) ->
- PostInitHookResult = do_post_init_hook(PostInitHook, NewConfig),
+ PostInitHookResult = do_post_init_hook(PostInitHook,NewConfig),
{ok, [PostInitHookResult ++ NewConfig]};
Else ->
Else
end.
-do_post_init_hook(PostInitHook, Config) ->
+do_post_init_hook(PostInitHook,Config) ->
lists:flatmap(fun({Tag,Fun}) ->
case lists:keysearch(Tag,1,Config) of
{value,_} ->
@@ -657,9 +674,23 @@ end_tc(Mod,Func,{TCPid,Result,[Args]}, Return) when is_pid(TCPid) ->
end_tc(Mod,Func,{Result,[Args]}, Return) ->
end_tc(Mod,Func,self(),Result,Args,Return).
-end_tc(Mod,Func,TCPid,Result,Args,Return) ->
+end_tc(Mod,IPTC={init_per_testcase,_Func},_TCPid,Result,Args,Return) ->
%% in case Mod == ct_framework, lookup the suite name
Suite = get_suite_name(Mod, Args),
+ case ct_hooks:end_tc(Suite,IPTC,Args,Result,Return) of
+ '$ct_no_change' ->
+ ok;
+ HookResult ->
+ HookResult
+ end;
+
+end_tc(Mod,Func0,TCPid,Result,Args,Return) ->
+ %% in case Mod == ct_framework, lookup the suite name
+ Suite = get_suite_name(Mod, Args),
+ {EPTC,Func} = case Func0 of
+ {end_per_testcase,F} -> {true,F};
+ _ -> {false,Func0}
+ end,
test_server:timetrap_cancel(),
@@ -686,11 +717,15 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) ->
end,
ct_util:delete_suite_data(last_saved_config),
- FuncSpec = group_or_func(Func,Args),
-
+ {FuncSpec,HookFunc} =
+ if not EPTC ->
+ FS = group_or_func(Func,Args),
+ {FS,FS};
+ true ->
+ {Func,Func0}
+ end,
{Result1,FinalNotify} =
- case ct_hooks:end_tc(
- Suite, FuncSpec, Args, Result, Return) of
+ case ct_hooks:end_tc(Suite,HookFunc,Args,Result,Return) of
'$ct_no_change' ->
{ok,Result};
HookResult ->
diff --git a/lib/common_test/src/ct_groups.erl b/lib/common_test/src/ct_groups.erl
index 7636f15f59..92640cf323 100644
--- a/lib/common_test/src/ct_groups.erl
+++ b/lib/common_test/src/ct_groups.erl
@@ -81,7 +81,7 @@ find(Mod, all, all, [{Name,Props,Tests} | Gs], Known, Defs, _)
find(Mod, all, TCs, [{Name,Props,Tests} | Gs], Known, Defs, _)
when is_atom(Name), is_list(Props), is_list(Tests) ->
cyclic_test(Mod, Name, Known),
- Tests1 = rm_unwanted_tcs(Tests, TCs, []),
+ Tests1 = modify_tc_list(Tests, TCs, []),
trim(make_conf(Mod, Name, Props,
find(Mod, all, TCs, Tests1, [Name | Known],
Defs, true))) ++
@@ -91,7 +91,7 @@ find(Mod, all, TCs, [{Name,Props,Tests} | Gs], Known, Defs, _)
find(Mod, [Name|GrNames]=SPath, TCs, [{Name,Props,Tests} | Gs], Known,
Defs, FindAll) when is_atom(Name), is_list(Props), is_list(Tests) ->
cyclic_test(Mod, Name, Known),
- Tests1 = rm_unwanted_tcs(Tests, TCs, GrNames),
+ Tests1 = modify_tc_list(Tests, TCs, GrNames),
trim(make_conf(Mod, Name, Props,
find(Mod, GrNames, TCs, Tests1, [Name|Known],
Defs, FindAll))) ++
@@ -133,7 +133,7 @@ find(_Mod, [_|_], _TCs, [], _Known, _Defs, _) ->
find(Mod, GrNames, TCs, [{Name,Props,Tests} | Gs], Known,
Defs, FindAll) when is_atom(Name), is_list(Props), is_list(Tests) ->
cyclic_test(Mod, Name, Known),
- Tests1 = rm_unwanted_tcs(Tests, TCs, GrNames),
+ Tests1 = modify_tc_list(Tests, TCs, GrNames),
trim(make_conf(Mod, Name, Props,
find(Mod, GrNames, TCs, Tests1, [Name|Known],
Defs, FindAll))) ++
@@ -284,70 +284,57 @@ trim_test(Test) ->
%% GrNames is [] if the terminating group has been found. From
%% that point, all specified test should be included (as well as
%% sub groups for deeper search).
-rm_unwanted_tcs(Tests, all, []) ->
- Tests;
-
-rm_unwanted_tcs(Tests, TCs, []) ->
- sort_tests(lists:flatmap(fun(Test) when is_tuple(Test),
- (size(Test) > 2) ->
- [Test];
- (Test={group,_}) ->
- [Test];
- (Test={_M,TC}) ->
- case lists:member(TC, TCs) of
- true -> [Test];
- false -> []
- end;
- (Test) when is_atom(Test) ->
- case lists:keysearch(Test, 2, TCs) of
- {value,_} ->
- [Test];
- _ ->
- case lists:member(Test, TCs) of
- true -> [Test];
- false -> []
- end
- end;
- (Test) -> [Test]
- end, Tests), TCs);
-
-rm_unwanted_tcs(Tests, _TCs, _) ->
- [Test || Test <- Tests, not is_atom(Test)].
-
-%% make sure the order of tests is according to the order in TCs
-sort_tests(Tests, TCs) when is_list(TCs)->
- lists:sort(fun(T1, T2) ->
- case {is_tc(T1),is_tc(T2)} of
- {true,true} ->
- (position(T1, TCs) =<
- position(T2, TCs));
- {false,true} ->
- (position(T2, TCs) == (length(TCs)+1));
- _ -> true
-
- end
- end, Tests);
-sort_tests(Tests, _) ->
- Tests.
-
-is_tc(T) when is_atom(T) -> true;
-is_tc({group,_}) -> false;
-is_tc({_M,T}) when is_atom(T) -> true;
-is_tc(_) -> false.
-
-position(T, TCs) ->
- position(T, TCs, 1).
-
-position(T, [T|_TCs], Pos) ->
- Pos;
-position(T, [{_,T}|_TCs], Pos) ->
- Pos;
-position({M,T}, [T|_TCs], Pos) when M /= group ->
- Pos;
-position(T, [_|TCs], Pos) ->
- position(T, TCs, Pos+1);
-position(_, [], Pos) ->
- Pos.
+modify_tc_list(GrSpecTs, all, []) ->
+ GrSpecTs;
+
+modify_tc_list(GrSpecTs, TSCs, []) ->
+ modify_tc_list1(GrSpecTs, TSCs);
+
+modify_tc_list(GrSpecTs, _TSCs, _) ->
+ [Test || Test <- GrSpecTs, not is_atom(Test)].
+
+modify_tc_list1(GrSpecTs, TSCs) ->
+ %% remove all cases in group tc list that should not be executed
+ GrSpecTs1 =
+ lists:flatmap(fun(Test) when is_tuple(Test),
+ (size(Test) > 2) ->
+ [Test];
+ (Test={group,_}) ->
+ [Test];
+ (Test={_M,TC}) ->
+ case lists:member(TC, TSCs) of
+ true -> [Test];
+ false -> []
+ end;
+ (Test) when is_atom(Test) ->
+ case lists:keysearch(Test, 2, TSCs) of
+ {value,_} ->
+ [Test];
+ _ ->
+ case lists:member(Test, TSCs) of
+ true -> [Test];
+ false -> []
+ end
+ end;
+ (Test) -> [Test]
+ end, GrSpecTs),
+ {TSCs2,GrSpecTs3} =
+ lists:foldr(
+ fun(TC, {TSCs1,GrSpecTs2}) ->
+ case lists:member(TC,GrSpecTs1) of
+ true ->
+ {[TC|TSCs1],lists:delete(TC,GrSpecTs2)};
+ false ->
+ case lists:keymember(TC, 2, GrSpecTs) of
+ {value,Test} ->
+ {[Test|TSCs1],
+ lists:keydelete(TC, 2, GrSpecTs2)};
+ false ->
+ {TSCs1,GrSpecTs2}
+ end
+ end
+ end, {[],GrSpecTs1}, TSCs),
+ TSCs2 ++ GrSpecTs3.
%%%-----------------------------------------------------------------
diff --git a/lib/common_test/src/ct_hooks.erl b/lib/common_test/src/ct_hooks.erl
index 86d18696dc..83ad33fdd8 100644
--- a/lib/common_test/src/ct_hooks.erl
+++ b/lib/common_test/src/ct_hooks.erl
@@ -93,7 +93,11 @@ init_tc(Mod, {init_per_group, GroupName, Properties}, Config) ->
call(fun call_generic/3, Config, [pre_init_per_group, GroupName]);
init_tc(_Mod, {end_per_group, GroupName, _}, Config) ->
call(fun call_generic/3, Config, [pre_end_per_group, GroupName]);
-init_tc(_Mod, TC, Config) ->
+init_tc(_Mod, {init_per_testcase,TC}, Config) ->
+ call(fun call_generic/3, Config, [pre_init_per_testcase, TC]);
+init_tc(_Mod, {end_per_testcase,TC}, Config) ->
+ call(fun call_generic/3, Config, [pre_end_per_testcase, TC]);
+init_tc(_Mod, TC = error_in_suite, Config) ->
call(fun call_generic/3, Config, [pre_init_per_testcase, TC]).
%% @doc Called as each test case is completed. This includes all configuration
@@ -126,10 +130,17 @@ end_tc(Mod, {end_per_group, GroupName, Properties}, Config, Result, _Return) ->
[post_end_per_group, GroupName, Config], '$ct_no_change'),
maybe_stop_locker(Mod, GroupName, Properties),
Res;
-end_tc(_Mod, TC, Config, Result, _Return) ->
+end_tc(_Mod, {init_per_testcase,TC}, Config, Result, _Return) ->
+ call(fun call_generic/3, Result, [post_init_per_testcase, TC, Config],
+ '$ct_no_change');
+end_tc(_Mod, {end_per_testcase,TC}, Config, Result, _Return) ->
+ call(fun call_generic/3, Result, [post_end_per_testcase, TC, Config],
+ '$ct_no_change');
+end_tc(_Mod, TC = error_in_suite, Config, Result, _Return) ->
call(fun call_generic/3, Result, [post_end_per_testcase, TC, Config],
'$ct_no_change').
+
%% Case = TestCase | {TestCase,GroupName}
on_tc_skip(How, {Suite, Case, Reason}) ->
call(fun call_cleanup/3, {How, Reason}, [on_tc_skip, Suite, Case]).
@@ -244,6 +255,8 @@ remove(_, Else) ->
%% Translate scopes, i.e. init_per_group,group1 -> end_per_group,group1 etc
scope([pre_init_per_testcase, TC|_]) ->
+ [post_init_per_testcase, TC];
+scope([pre_end_per_testcase, TC|_]) ->
[post_end_per_testcase, TC];
scope([pre_init_per_group, GroupName|_]) ->
[post_end_per_group, GroupName];
@@ -317,7 +330,8 @@ get_hooks() ->
%% If we are doing a cleanup call i.e. {post,pre}_end_per_*, all priorities
%% are reversed. Probably want to make this sorting algorithm pluginable
%% as some point...
-resort(Calls,Hooks,[F|_R]) when F == post_end_per_testcase;
+resort(Calls,Hooks,[F|_R]) when F == pre_end_per_testcase;
+ F == post_end_per_testcase;
F == pre_end_per_group;
F == post_end_per_group;
F == pre_end_per_suite;
@@ -367,10 +381,10 @@ pos(Id,[_|Rest],Num) ->
catch_apply(M,F,A, Default) ->
try
- apply(M,F,A)
+ erlang:apply(M,F,A)
catch _:Reason ->
case erlang:get_stacktrace() of
- %% Return the default if it was the CTH module which did not have the function.
+ %% Return the default if it was the CTH module which did not have the function.
[{M,F,A,_}|_] when Reason == undef ->
Default;
Trace ->
diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index 39b2c2a8cd..e156c9b773 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -2055,6 +2055,13 @@ final_tests1([{TestDir,Suite,GrsOrCs}|Tests], Final, Skip, Bad) when
({skipped,Group,TCs}) ->
[ct_groups:make_conf(TestDir, Suite,
Group, [skipped], TCs)];
+ ({skipped,TC}) ->
+ case lists:member(TC, GrsOrCs) of
+ true ->
+ [];
+ false ->
+ [TC]
+ end;
({GrSpec = {GroupName,_},TCs}) ->
Props = [{override,GrSpec}],
[ct_groups:make_conf(TestDir, Suite,
diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl
index 5d5448f352..5cd52bd042 100644
--- a/lib/common_test/src/ct_testspec.erl
+++ b/lib/common_test/src/ct_testspec.erl
@@ -70,13 +70,17 @@ prepare_tests(TestSpec) when is_record(TestSpec,testspec) ->
Tests = TestSpec#testspec.tests,
%% Sort Tests into "flat" Run and Skip lists (not sorted per node).
{Run,Skip} = get_run_and_skip(Tests,[],[]),
+
%% Create initial list of {Node,{Run,Skip}} tuples
NodeList = lists:map(fun(N) -> {N,{[],[]}} end, list_nodes(TestSpec)),
+
%% Get all Run tests sorted per node basis.
NodeList1 = run_per_node(Run,NodeList,
- TestSpec#testspec.merge_tests),
+ TestSpec#testspec.merge_tests),
+
%% Get all Skip entries sorted per node basis.
NodeList2 = skip_per_node(Skip,NodeList1),
+
%% Change representation.
Result=
lists:map(fun({Node,{Run1,Skip1}}) ->
@@ -103,7 +107,7 @@ run_per_node([{{Node,Dir},Test}|Ts],Result,MergeTests) ->
true ->
merge_tests(Dir,Test,Run)
end,
- run_per_node(Ts,insert_in_order({Node,{Run1,Skip}},Result),
+ run_per_node(Ts,insert_in_order({Node,{Run1,Skip}},Result,replace),
MergeTests);
run_per_node([],Result,_) ->
Result.
@@ -140,7 +144,7 @@ merge_suites(Dir,Test,[]) ->
skip_per_node([{{Node,Dir},Test}|Ts],Result) ->
{value,{Node,{Run,Skip}}} = lists:keysearch(Node,1,Result),
Skip1 = [{Dir,Test}|Skip],
- skip_per_node(Ts,insert_in_order({Node,{Run,Skip1}},Result));
+ skip_per_node(Ts,insert_in_order({Node,{Run,Skip1}},Result,replace));
skip_per_node([],Result) ->
Result.
@@ -156,7 +160,7 @@ skip_per_node([],Result) ->
%%
%% Skip entry: {Suites,Comment} or {Suite,Cases,Comment}
%%
-get_run_and_skip([{{Node,Dir},Suites}|Tests],Run,Skip) ->
+get_run_and_skip([{{Node,Dir},Suites}|Tests],Run,Skip) ->
TestDir = ct_util:get_testdir(Dir,catch element(1,hd(Suites))),
case lists:keysearch(all,1,Suites) of
{value,_} -> % all Suites in Dir
@@ -183,18 +187,33 @@ prepare_suites(Node,Dir,[{Suite,Cases}|Suites],Run,Skip) ->
[[{{Node,Dir},{Suite,all}}]|Run],
[Skipped|Skip]);
false ->
- {RL,SL} = prepare_cases(Node,Dir,Suite,Cases),
- prepare_suites(Node,Dir,Suites,[RL|Run],[SL|Skip])
+ {Run1,Skip1} = prepare_cases(Node,Dir,Suite,Cases,Run,Skip),
+ prepare_suites(Node,Dir,Suites,Run1,Skip1)
end;
prepare_suites(_Node,_Dir,[],Run,Skip) ->
{lists:flatten(lists:reverse(Run)),
lists:flatten(lists:reverse(Skip))}.
-prepare_cases(Node,Dir,Suite,Cases) ->
+prepare_cases(Node,Dir,Suite,Cases,Run,Skip) ->
case get_skipped_cases(Node,Dir,Suite,Cases) of
- SkipAll=[{{Node,Dir},{Suite,_Cmt}}] -> % all cases to be skipped
- %% note: this adds an 'all' test even if only skip is specified
- {[{{Node,Dir},{Suite,all}}],SkipAll};
+ [SkipAll={{Node,Dir},{Suite,_Cmt}}] -> % all cases to be skipped
+ case lists:any(fun({{N,D},{S,all}}) when N == Node,
+ D == Dir,
+ S == Suite ->
+ true;
+ ({{N,D},{S,Cs}}) when N == Node,
+ D == Dir,
+ S == Suite ->
+ lists:member(all,Cs);
+ (_) -> false
+ end, lists:flatten(Run)) of
+ true ->
+ {Run,[SkipAll|Skip]};
+ false ->
+ %% note: this adds an 'all' test even if
+ %% only skip is specified
+ {[{{Node,Dir},{Suite,all}}|Run],[SkipAll|Skip]}
+ end;
Skipped ->
%% note: this adds a test even if only skip is specified
PrepC = lists:foldr(fun({{G,Cs},{skip,_Cmt}}, Acc) when
@@ -210,11 +229,11 @@ prepare_cases(Node,Dir,Suite,Cases) ->
true ->
Acc;
false ->
- [C|Acc]
+ [{skipped,C}|Acc]
end;
(C,Acc) -> [C|Acc]
end, [], Cases),
- {{{Node,Dir},{Suite,PrepC}},Skipped}
+ {[{{Node,Dir},{Suite,PrepC}}|Run],[Skipped|Skip]}
end.
get_skipped_suites(Node,Dir,Suites) ->
@@ -431,6 +450,7 @@ collect_tests({Replace,Terms},TestSpec=#testspec{alias=As,nodes=Ns},Relaxed) ->
merge_tests = MergeTestsDef}),
TestSpec2 = get_all_nodes(Terms2,TestSpec1),
{Terms3, TestSpec3} = filter_init_terms(Terms2, [], TestSpec2),
+
add_tests(Terms3,TestSpec3).
%% replace names (atoms) in the testspec matching those in 'define' terms by
@@ -1257,7 +1277,7 @@ insert_groups1(Suite,Groups,Suites0) ->
Suites0;
{value,{Suite,GrAndCases0}} ->
GrAndCases = insert_groups2(Groups,GrAndCases0),
- insert_in_order({Suite,GrAndCases},Suites0);
+ insert_in_order({Suite,GrAndCases},Suites0,replace);
false ->
insert_in_order({Suite,Groups},Suites0)
end.
@@ -1282,7 +1302,7 @@ insert_cases(Node,Dir,Suite,Cases,Tests,false) when is_list(Cases) ->
insert_cases(Node,Dir,Suite,Cases,Tests,true) when is_list(Cases) ->
{Tests1,Done} =
lists:foldr(fun(All={{N,D},[{all,_}]},{Merged,_}) when N == Node,
- D == Dir ->
+ D == Dir ->
{[All|Merged],true};
({{N,D},Suites0},{Merged,_}) when N == Node,
D == Dir ->
@@ -1312,7 +1332,7 @@ insert_cases1(Suite,Cases,Suites0) ->
Suites0;
{value,{Suite,Cases0}} ->
Cases1 = insert_in_order(Cases,Cases0),
- insert_in_order({Suite,Cases1},Suites0);
+ insert_in_order({Suite,Cases1},Suites0,replace);
false ->
insert_in_order({Suite,Cases},Suites0)
end.
@@ -1369,9 +1389,9 @@ skip_groups1(Suite,Groups,Cmt,Suites0) ->
case lists:keysearch(Suite,1,Suites0) of
{value,{Suite,GrAndCases0}} ->
GrAndCases1 = GrAndCases0 ++ SkipGroups,
- insert_in_order({Suite,GrAndCases1},Suites0);
+ insert_in_order({Suite,GrAndCases1},Suites0,replace);
false ->
- insert_in_order({Suite,SkipGroups},Suites0)
+ insert_in_order({Suite,SkipGroups},Suites0,replace)
end.
skip_cases(Node,Dir,Suite,Cases,Cmt,Tests,false) when is_list(Cases) ->
@@ -1380,7 +1400,7 @@ skip_cases(Node,Dir,Suite,Cases,Cmt,Tests,false) when is_list(Cases) ->
skip_cases(Node,Dir,Suite,Cases,Cmt,Tests,true) when is_list(Cases) ->
{Tests1,Done} =
lists:foldr(fun({{N,D},Suites0},{Merged,_}) when N == Node,
- D == Dir ->
+ D == Dir ->
Suites1 = skip_cases1(Suite,Cases,Cmt,Suites0),
{[{{N,D},Suites1}|Merged],true};
(T,{Merged,Match}) ->
@@ -1401,32 +1421,55 @@ skip_cases1(Suite,Cases,Cmt,Suites0) ->
case lists:keysearch(Suite,1,Suites0) of
{value,{Suite,Cases0}} ->
Cases1 = Cases0 ++ SkipCases,
- insert_in_order({Suite,Cases1},Suites0);
+ insert_in_order({Suite,Cases1},Suites0,replace);
false ->
- insert_in_order({Suite,SkipCases},Suites0)
+ case Suites0 of
+ [{all,_}=All|Skips]->
+ [All|Skips++[{Suite,SkipCases}]];
+ _ ->
+ insert_in_order({Suite,SkipCases},Suites0,replace)
+ end
end.
append(Elem, List) ->
List ++ [Elem].
-insert_in_order([E|Es],List) ->
- List1 = insert_elem(E,List,[]),
- insert_in_order(Es,List1);
-insert_in_order([],List) ->
+insert_in_order(Elems,Dest) ->
+ insert_in_order1(Elems,Dest,false).
+
+insert_in_order(Elems,Dest,replace) ->
+ insert_in_order1(Elems,Dest,true).
+
+insert_in_order1([_E|Es],all,Replace) ->
+ insert_in_order1(Es,all,Replace);
+
+insert_in_order1([E|Es],List,Replace) ->
+ List1 = insert_elem(E,List,[],Replace),
+ insert_in_order1(Es,List1,Replace);
+insert_in_order1([],List,_Replace) ->
List;
-insert_in_order(E,List) ->
- insert_elem(E,List,[]).
+insert_in_order1(E,List,Replace) ->
+ insert_elem(E,List,[],Replace).
+
-%% replace an existing entry (same key) or add last in list
-insert_elem({Key,_}=E,[{Key,_}|Rest],SoFar) ->
+insert_elem({Key,_}=E,[{Key,_}|Rest],SoFar,true) ->
lists:reverse([E|SoFar]) ++ Rest;
-insert_elem({E,_},[E|Rest],SoFar) ->
+insert_elem({E,_},[E|Rest],SoFar,true) ->
lists:reverse([E|SoFar]) ++ Rest;
-insert_elem(E,[E|Rest],SoFar) ->
+insert_elem(E,[E|Rest],SoFar,true) ->
+ lists:reverse([E|SoFar]) ++ Rest;
+
+insert_elem({all,_}=E,_,SoFar,_Replace) ->
+ lists:reverse([E|SoFar]);
+insert_elem(_E,[all|_],SoFar,_Replace) ->
+ lists:reverse(SoFar);
+insert_elem(_E,[{all,_}],SoFar,_Replace) ->
+ lists:reverse(SoFar);
+insert_elem({Key,_}=E,[{Key,[]}|Rest],SoFar,_Replace) ->
lists:reverse([E|SoFar]) ++ Rest;
-insert_elem(E,[E1|Rest],SoFar) ->
- insert_elem(E,Rest,[E1|SoFar]);
-insert_elem(E,[],SoFar) ->
+insert_elem(E,[E1|Rest],SoFar,Replace) ->
+ insert_elem(E,Rest,[E1|SoFar],Replace);
+insert_elem(E,[],SoFar,_Replace) ->
lists:reverse([E|SoFar]).
ref2node(all_nodes,_Refs) ->
diff --git a/lib/common_test/src/cth_log_redirect.erl b/lib/common_test/src/cth_log_redirect.erl
index e6970a2bad..a8c4a455e1 100644
--- a/lib/common_test/src/cth_log_redirect.erl
+++ b/lib/common_test/src/cth_log_redirect.erl
@@ -30,7 +30,8 @@
pre_init_per_suite/3, pre_end_per_suite/3, post_end_per_suite/4,
pre_init_per_group/3, post_init_per_group/4,
pre_end_per_group/3, post_end_per_group/4,
- pre_init_per_testcase/3, post_end_per_testcase/4]).
+ pre_init_per_testcase/3, post_init_per_testcase/4,
+ pre_end_per_testcase/3, post_end_per_testcase/4]).
%% Event handler Callbacks
-export([init/1,
@@ -89,6 +90,12 @@ pre_init_per_testcase(TC, Config, State) ->
set_curr_func(TC, Config),
{Config, State}.
+post_init_per_testcase(_TC, _Config, Return, State) ->
+ {Return, State}.
+
+pre_end_per_testcase(_TC, Config, State) ->
+ {Config, State}.
+
post_end_per_testcase(_TC, _Config, Result, State) ->
%% Make sure that the event queue is flushed
%% before ending this test case.
diff --git a/lib/common_test/src/test_server.erl b/lib/common_test/src/test_server.erl
index 671674c617..f62eb88ccf 100644
--- a/lib/common_test/src/test_server.erl
+++ b/lib/common_test/src/test_server.erl
@@ -714,10 +714,10 @@ call_end_conf(Mod,Func,TCPid,TCExitReason,Loc,Conf,TVal) ->
Starter ! {self(),{call_end_conf,Data,ok}}
end);
true ->
- do_call_end_conf(Starter,Mod,Func,Data,Conf,TVal)
+ do_call_end_conf(Starter,Mod,Func,Data,TCExitReason,Conf,TVal)
end.
-do_call_end_conf(Starter,Mod,Func,Data,Conf,TVal) ->
+do_call_end_conf(Starter,Mod,Func,Data,TCExitReason,Conf,TVal) ->
EndConfProc =
fun() ->
process_flag(trap_exit,true), % to catch timetraps
@@ -725,16 +725,29 @@ do_call_end_conf(Starter,Mod,Func,Data,Conf,TVal) ->
EndConfApply =
fun() ->
timetrap(TVal),
- try apply(Mod,end_per_testcase,[Func,Conf]) of
+ %% We can't handle fails or skips here
+ %% (neither input nor output). The error can
+ %% be read from Conf though (tc_status).
+ EndConf =
+ case do_init_tc_call(Mod,{end_per_testcase,Func},
+ [Conf],
+ {TCExitReason,[Conf]}) of
+ {_,[EPTCInit]} when is_list(EPTCInit) ->
+ EPTCInit;
+ _ ->
+ Conf
+ end,
+ try apply(Mod,end_per_testcase,[Func,EndConf]) of
_ -> ok
catch
_:Why ->
timer:sleep(1),
- group_leader() ! {printout,12,
- "WARNING! "
- "~w:end_per_testcase(~w, ~p)"
- " crashed!\n\tReason: ~p\n",
- [Mod,Func,Conf,Why]}
+ GLMsg = {printout,12,
+ "WARNING! "
+ "~w:end_per_testcase(~w, ~p)"
+ " crashed!\n\tReason: ~p\n",
+ [Mod,Func,Conf,Why]},
+ group_leader() ! GLMsg
end,
Supervisor ! {self(),end_conf}
end,
@@ -755,28 +768,38 @@ do_call_end_conf(Starter,Mod,Func,Data,Conf,TVal) ->
end,
spawn_link(EndConfProc).
-spawn_fw_call(Mod,{init_per_testcase,Func},CurrConf,Pid,
- {timetrap_timeout,TVal}=Why,
- Loc,SendTo) ->
+spawn_fw_call(Mod,IPTC={init_per_testcase,Func},CurrConf,Pid,
+ Why,Loc,SendTo) ->
FwCall =
fun() ->
Skip = {skip,{failed,{Mod,init_per_testcase,Why}}},
%% if init_per_testcase fails, the test case
%% should be skipped
- try do_end_tc_call(Mod,Func, {Pid,Skip,[CurrConf]}, Why) of
+ try begin do_end_tc_call(Mod,IPTC, {Pid,Skip,[CurrConf]}, Why),
+ do_init_tc_call(Mod,{end_per_testcase,Func},
+ [CurrConf],{ok,[CurrConf]}),
+ do_end_tc_call(Mod,{end_per_testcase,Func},
+ {Pid,Skip,[CurrConf]}, Why) end of
_ -> ok
catch
_:FwEndTCErr ->
exit({fw_notify_done,end_tc,FwEndTCErr})
end,
+ Time = case Why of
+ {timetrap_timeout,TVal} -> TVal/1000;
+ _ -> died
+ end,
+ group_leader() ! {printout,12,
+ "ERROR! ~w:init_per_testcase(~w, ~p)"
+ " failed!\n\tReason: ~tp\n",
+ [Mod,Func,CurrConf,Why]},
%% finished, report back
- SendTo ! {self(),fw_notify_done,
- {TVal/1000,Skip,Loc,[],undefined}}
+ SendTo ! {self(),fw_notify_done,{Time,Skip,Loc,[],undefined}}
end,
spawn_link(FwCall);
-spawn_fw_call(Mod,{end_per_testcase,Func},EndConf,Pid,
- {timetrap_timeout,TVal}=Why,_Loc,SendTo) ->
+spawn_fw_call(Mod,EPTC={end_per_testcase,Func},EndConf,Pid,
+ Why,_Loc,SendTo) ->
FwCall =
fun() ->
{RetVal,Report} =
@@ -790,24 +813,38 @@ spawn_fw_call(Mod,{end_per_testcase,Func},EndConf,Pid,
E = {failed,{Mod,end_per_testcase,Why}},
{Result,E}
end,
- group_leader() ! {printout,12,
- "WARNING! ~w:end_per_testcase(~w, ~p)"
- " failed!\n\tReason: timetrap timeout"
- " after ~w ms!\n", [Mod,Func,EndConf,TVal]},
- FailLoc = proplists:get_value(tc_fail_loc, EndConf),
- try do_end_tc_call(Mod,Func,
- {Pid,Report,[EndConf]}, Why) of
+ {Time,Warn} =
+ case Why of
+ {timetrap_timeout,TVal} ->
+ group_leader() !
+ {printout,12,
+ "WARNING! ~w:end_per_testcase(~w, ~p)"
+ " failed!\n\tReason: timetrap timeout"
+ " after ~w ms!\n", [Mod,Func,EndConf,TVal]},
+ W = "<font color=\"red\">"
+ "WARNING: end_per_testcase timed out!</font>",
+ {TVal/1000,W};
+ _ ->
+ group_leader() !
+ {printout,12,
+ "WARNING! ~w:end_per_testcase(~w, ~p)"
+ " failed!\n\tReason: ~tp\n",
+ [Mod,Func,EndConf,Why]},
+ W = "<font color=\"red\">"
+ "WARNING: end_per_testcase failed!</font>",
+ {died,W}
+ end,
+ try do_end_tc_call(Mod,EPTC,{Pid,Report,[EndConf]}, Why) of
_ -> ok
catch
_:FwEndTCErr ->
exit({fw_notify_done,end_tc,FwEndTCErr})
end,
- Warn = "<font color=\"red\">"
- "WARNING: end_per_testcase timed out!</font>",
+ FailLoc = proplists:get_value(tc_fail_loc, EndConf),
%% finished, report back (if end_per_testcase fails, a warning
%% should be printed as part of the comment)
SendTo ! {self(),fw_notify_done,
- {TVal/1000,RetVal,FailLoc,[],Warn}}
+ {Time,RetVal,FailLoc,[],Warn}}
end,
spawn_link(FwCall);
@@ -830,10 +867,12 @@ spawn_fw_call(FwMod,FwFunc,_,_Pid,{framework_error,FwError},_,SendTo) ->
spawn_link(FwCall);
spawn_fw_call(Mod,Func,CurrConf,Pid,Error,Loc,SendTo) ->
- Func1 = case Func of
- {_InitOrEndPerTC,F} -> F;
- F -> F
- end,
+ {Func1,EndTCFunc} = case Func of
+ CF when CF == init_per_suite; CF == end_per_suite;
+ CF == init_per_group; CF == end_per_group ->
+ {CF,CF};
+ TC -> {TC,{end_per_testcase,TC}}
+ end,
FwCall =
fun() ->
try fw_error_notify(Mod,Func1,[],
@@ -845,8 +884,7 @@ spawn_fw_call(Mod,Func,CurrConf,Pid,Error,Loc,SendTo) ->
FwErrorNotifyErr})
end,
Conf = [{tc_status,{failed,Error}}|CurrConf],
- try do_end_tc_call(Mod,Func1,
- {Pid,Error,[Conf]},Error) of
+ try do_end_tc_call(Mod,EndTCFunc,{Pid,Error,[Conf]},Error) of
_ -> ok
catch
_:FwEndTCErr ->
@@ -923,32 +961,38 @@ run_test_case_eval(Mod, Func, Args0, Name, Ref, RunInit,
Where = [{Mod,Func}],
put(test_server_loc, Where),
- FWInitResult = test_server_sup:framework_call(init_tc,[Mod,Func,Args0],
- {ok,Args0}),
+ FWInitFunc = case RunInit of
+ run_init -> {init_per_testcase,Func};
+ _ -> Func
+ end,
+
+ FWInitResult0 = do_init_tc_call(Mod,FWInitFunc,Args0,{ok,Args0}),
+
set_tc_state(running),
{{Time,Value},Loc,Opts} =
- case FWInitResult of
+ case FWInitResult0 of
{ok,Args} ->
run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback);
Error = {error,_Reason} ->
- NewResult = do_end_tc_call(Mod,Func, {Error,Args0},
+ NewResult = do_end_tc_call(Mod,FWInitFunc, {Error,Args0},
{auto_skip,{failed,Error}}),
{{0,NewResult},Where,[]};
{fail,Reason} ->
- Conf = [{tc_status,{failed,Reason}} | hd(Args0)],
+ Conf = [{tc_status,{failed,Reason}} | hd(Args0)],
fw_error_notify(Mod, Func, Conf, Reason),
- NewResult = do_end_tc_call(Mod,Func, {{error,Reason},[Conf]},
+ NewResult = do_end_tc_call(Mod,FWInitFunc,
+ {{error,Reason},[Conf]},
{fail,Reason}),
{{0,NewResult},Where,[]};
Skip = {SkipType,_Reason} when SkipType == skip;
SkipType == skipped ->
- NewResult = do_end_tc_call(Mod,Func,
+ NewResult = do_end_tc_call(Mod,FWInitFunc,
{Skip,Args0}, Skip),
{{0,NewResult},Where,[]};
AutoSkip = {auto_skip,_Reason} ->
%% special case where a conf case "pretends" to be skipped
NewResult =
- do_end_tc_call(Mod,Func, {AutoSkip,Args0}, AutoSkip),
+ do_end_tc_call(Mod,FWInitFunc, {AutoSkip,Args0}, AutoSkip),
{{0,NewResult},Where,[]}
end,
exit({Ref,Time,Value,Loc,Opts}).
@@ -963,31 +1007,41 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) ->
SkipType == skipped ->
Line = get_loc(),
Conf = [{tc_status,{skipped,Reason}}|hd(Args)],
- NewRes = do_end_tc_call(Mod,Func,
+ NewRes = do_end_tc_call(Mod,{init_per_testcase,Func},
{Skip,[Conf]}, Skip),
{{0,NewRes},Line,[]};
{skip_and_save,Reason,SaveCfg} ->
Line = get_loc(),
Conf = [{tc_status,{skipped,Reason}},
{save_config,SaveCfg}|hd(Args)],
- NewRes = do_end_tc_call(Mod,Func, {{skip,Reason},[Conf]},
+ NewRes = do_end_tc_call(Mod,{init_per_testcase,Func},
+ {{skip,Reason},[Conf]},
{skip,Reason}),
{{0,NewRes},Line,[]};
FailTC = {fail,Reason} -> % user fails the testcase
EndConf = [{tc_status,{failed,Reason}} | hd(Args)],
fw_error_notify(Mod, Func, EndConf, Reason),
- NewRes = do_end_tc_call(Mod,Func,
+ NewRes = do_end_tc_call(Mod,{init_per_testcase,Func},
{{error,Reason},[EndConf]},
FailTC),
{{0,NewRes},[{Mod,Func}],[]};
{ok,NewConf} ->
- %% call user callback function if defined
- NewConf1 =
- user_callback(TCCallback, Mod, Func, init, NewConf),
- %% save current state in controller loop
- set_tc_state(tc, NewConf1),
- %% execute the test case
- {{T,Return},Loc} = {ts_tc(Mod,Func,[NewConf1]), get_loc()},
+ IPTCEndRes = do_end_tc_call(Mod,{init_per_testcase,Func},
+ {ok,[NewConf]}, NewConf),
+ {{T,Return},Loc,NewConf1} =
+ if not is_list(IPTCEndRes) ->
+ %% received skip or fail, not config
+ {{0,IPTCEndRes},undefined,NewConf};
+ true ->
+ %% call user callback function if defined
+ NewConfUC =
+ user_callback(TCCallback, Mod, Func,
+ init, IPTCEndRes),
+ %% save current state in controller loop
+ set_tc_state(tc, NewConfUC),
+ %% execute the test case
+ {ts_tc(Mod,Func,[NewConfUC]),get_loc(),NewConfUC}
+ end,
{EndConf,TSReturn,FWReturn} =
case Return of
{E,TCError} when E=='EXIT' ; E==failed ->
@@ -1013,36 +1067,47 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) ->
%% call user callback function if defined
EndConf1 =
user_callback(TCCallback, Mod, Func, 'end', EndConf),
+
+ %% We can't handle fails or skips here
+ EndConf2 =
+ case do_init_tc_call(Mod,{end_per_testcase,Func},
+ [EndConf1],{ok,[EndConf1]}) of
+ {ok,[EPTCInitRes]} when is_list(EPTCInitRes) ->
+ EPTCInitRes;
+ _ ->
+ EndConf1
+ end,
+
%% update current state in controller loop
- {FWReturn1,TSReturn1,EndConf2} =
- case end_per_testcase(Mod, Func, EndConf1) of
+ {FWReturn1,TSReturn1,EndConf3} =
+ case end_per_testcase(Mod, Func, EndConf2) of
SaveCfg1={save_config,_} ->
{FWReturn,TSReturn,
[SaveCfg1|lists:keydelete(save_config,1,
- EndConf1)]};
+ EndConf2)]};
{fail,ReasonToFail} ->
%% user has failed the testcase
- fw_error_notify(Mod, Func, EndConf1,
+ fw_error_notify(Mod, Func, EndConf2,
ReasonToFail),
{{error,ReasonToFail},
{failed,ReasonToFail},
- EndConf1};
+ EndConf2};
{failed,{_,end_per_testcase,_}} = Failure when
FWReturn == ok ->
%% unexpected termination in end_per_testcase
%% report this as the result to the framework
- {Failure,TSReturn,EndConf1};
+ {Failure,TSReturn,EndConf2};
_ ->
%% test case result should be reported to
%% framework no matter the status of
%% end_per_testcase
- {FWReturn,TSReturn,EndConf1}
+ {FWReturn,TSReturn,EndConf2}
end,
%% clear current state in controller loop
- case do_end_tc_call(Mod,Func,
- {FWReturn1,[EndConf2]}, TSReturn1) of
+ case do_end_tc_call(Mod,{end_per_testcase,Func},
+ {FWReturn1,[EndConf3]}, TSReturn1) of
{failed,Reason} = NewReturn ->
- fw_error_notify(Mod,Func,EndConf2, Reason),
+ fw_error_notify(Mod,Func,EndConf3, Reason),
{{T,NewReturn},[{Mod,Func}],[]};
NewReturn ->
{{T,NewReturn},Loc,[]}
@@ -1068,7 +1133,38 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) ->
{{T,Return2},Loc,Opts}
end.
+do_init_tc_call(Mod, Func, Res, Return) ->
+ test_server_sup:framework_call(init_tc,[Mod,Func,Res],Return).
+
+do_end_tc_call(Mod, IPTC={init_per_testcase,Func}, Res, Return) ->
+ case Return of
+ {NOk,_} when NOk == auto_skip; NOk == fail;
+ NOk == skip ; NOk == skipped ->
+ {_,Args} = Res,
+ IPTCEndRes =
+ case do_end_tc_call1(Mod, IPTC, Res, Return) of
+ IPTCEndConfig when is_list(IPTCEndConfig) ->
+ IPTCEndConfig;
+ _ ->
+ Args
+ end,
+ EPTCInitRes =
+ case do_init_tc_call(Mod,{end_per_testcase,Func},
+ IPTCEndRes,Return) of
+ {ok,EPTCInitConfig} when is_list(EPTCInitConfig) ->
+ {Return,EPTCInitConfig};
+ _ ->
+ Return
+ end,
+ do_end_tc_call1(Mod, {end_per_testcase,Func},
+ EPTCInitRes, Return);
+ _Ok ->
+ do_end_tc_call1(Mod, IPTC, Res, Return)
+ end;
do_end_tc_call(Mod, Func, Res, Return) ->
+ do_end_tc_call1(Mod, Func, Res, Return).
+
+do_end_tc_call1(Mod, Func, Res, Return) ->
FwMod = os:getenv("TEST_SERVER_FRAMEWORK"),
Ref = make_ref(),
if FwMod == "ct_framework" ; FwMod == "undefined"; FwMod == false ->
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-10 17:48:39 +0000