aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test/src
diff options
context:
space:
mode:
authorSiri Hansen <[email protected]>2017-03-06 08:51:22 +0100
committerSiri Hansen <[email protected]>2017-03-06 08:51:22 +0100
commit0e0155410734e04556ceecb1222dd2ef956b4177 (patch)
treea714e355d78f4c96f1644ab64783a1a0a32d4da7 /lib/common_test/src
parent0438c6c2c5c4a447ab3a47a443968488a0d835eb (diff)
parent50cf93672b8abb21f1455d41d333481071e6194e (diff)
downloadotp-0e0155410734e04556ceecb1222dd2ef956b4177.tar.gz
otp-0e0155410734e04556ceecb1222dd2ef956b4177.tar.bz2
otp-0e0155410734e04556ceecb1222dd2ef956b4177.zip
Merge branch 'siri/ct_hooks/callbacks-on-skip/OTP-10599' into maint
* siri/ct_hooks/callbacks-on-skip/OTP-10599: [ct] Update built-in ct hooks with new Suite parameter [cth_surefire] Handle skips from test spec [ct] Remove excessive skipped/failed tag in hook function [ct] Send tc_start event on force_stop and failed sequence [ct] Correctly handle process died in init and end_per_testcase [ct] Fix faulty hook callbacks for skipped tests Add dummy end_per_suite/1 [ct] Fix hooks and fail when one of init/end_per_* does not exit [ct] Add Suite argument to hook callback functions [ct] Fix function_clause in ct_framework when hook function crashes
Diffstat (limited to 'lib/common_test/src')
-rw-r--r--lib/common_test/src/ct_framework.erl161
-rw-r--r--lib/common_test/src/ct_groups.erl14
-rw-r--r--lib/common_test/src/ct_hooks.erl141
-rw-r--r--lib/common_test/src/cth_conn_log.erl8
-rw-r--r--lib/common_test/src/cth_log_redirect.erl28
-rw-r--r--lib/common_test/src/cth_surefire.erl54
-rw-r--r--lib/common_test/src/test_server.erl10
-rw-r--r--lib/common_test/src/test_server_ctrl.erl116
8 files changed, 322 insertions, 210 deletions
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index 291a4d716c..43f1c9de0f 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -52,6 +52,10 @@
%%%
%%% @doc Test server framework callback, called by the test_server
%%% when a new test case is started.
+init_tc(_,{end_per_testcase_not_run,_},[Config]) ->
+ %% Testcase is completed (skipped or failed), but end_per_testcase
+ %% is not run - don't call pre-hook.
+ {ok,[Config]};
init_tc(Mod,EPTC={end_per_testcase,_},[Config]) ->
%% in case Mod == ct_framework, lookup the suite name
Suite = get_suite_name(Mod, Config),
@@ -62,7 +66,7 @@ init_tc(Mod,EPTC={end_per_testcase,_},[Config]) ->
Other
end;
-init_tc(Mod,Func0,Args) ->
+init_tc(Mod,Func0,Args) ->
%% in case Mod == ct_framework, lookup the suite name
Suite = get_suite_name(Mod, Args),
{Func,HookFunc} = case Func0 of
@@ -84,12 +88,15 @@ init_tc(Mod,Func0,Args) ->
andalso Func=/=end_per_group
andalso ct_util:get_testdata(skip_rest) of
true ->
+ initialize(false,Mod,Func,Args),
{auto_skip,"Repeated test stopped by force_stop option"};
_ ->
case ct_util:get_testdata(curr_tc) of
{Suite,{suite0_failed,{require,Reason}}} ->
+ initialize(false,Mod,Func,Args),
{auto_skip,{require_failed_in_suite0,Reason}};
{Suite,{suite0_failed,_}=Failure} ->
+ initialize(false,Mod,Func,Args),
{fail,Failure};
_ ->
ct_util:update_testdata(curr_tc,
@@ -118,16 +125,14 @@ init_tc(Mod,Func0,Args) ->
end,
init_tc1(Mod,Suite,Func,HookFunc,Args);
{failed,Seq,BadFunc} ->
- {auto_skip,{sequence_failed,Seq,BadFunc}}
+ initialize(false,Mod,Func,Args),
+ {auto_skip,{sequence_failed,Seq,BadFunc}}
end
end
end.
init_tc1(?MODULE,_,error_in_suite,_,[Config0]) when is_list(Config0) ->
- ct_logs:init_tc(false),
- ct_event:notify(#event{name=tc_start,
- node=node(),
- data={?MODULE,error_in_suite}}),
+ initialize(false,?MODULE,error_in_suite),
_ = ct_suite_init(?MODULE,error_in_suite,[],Config0),
case ?val(error,Config0) of
undefined ->
@@ -177,27 +182,21 @@ init_tc1(Mod,Suite,Func,HookFunc,[Config0]) when is_list(Config0) ->
ct_config:delete_default_config(testcase),
HookFunc
end,
- Initialize = fun() ->
- ct_logs:init_tc(false),
- ct_event:notify(#event{name=tc_start,
- node=node(),
- data={Mod,FuncSpec}})
- end,
case add_defaults(Mod,Func,AllGroups) of
Error = {suite0_failed,_} ->
- Initialize(),
+ initialize(false,Mod,FuncSpec),
ct_util:set_testdata({curr_tc,{Suite,Error}}),
{error,Error};
Error = {group0_failed,_} ->
- Initialize(),
+ initialize(false,Mod,FuncSpec),
{auto_skip,Error};
Error = {testcase0_failed,_} ->
- Initialize(),
+ initialize(false,Mod,FuncSpec),
{auto_skip,Error};
{SuiteInfo,MergeResult} ->
case MergeResult of
{error,Reason} ->
- Initialize(),
+ initialize(false,Mod,FuncSpec),
{fail,Reason};
_ ->
init_tc2(Mod,Suite,Func,HookFunc1,
@@ -236,11 +235,8 @@ init_tc2(Mod,Suite,Func,HookFunc,SuiteInfo,MergeResult,Config) ->
Conns ->
ct_util:silence_connections(Conns)
end,
- ct_logs:init_tc(Func == init_per_suite),
FuncSpec = group_or_func(Func,Config),
- ct_event:notify(#event{name=tc_start,
- node=node(),
- data={Mod,FuncSpec}}),
+ initialize((Func==init_per_suite),Mod,FuncSpec),
case catch configure(MergedInfo,MergedInfo,SuiteInfo,
FuncSpec,[],Config) of
@@ -268,6 +264,18 @@ init_tc2(Mod,Suite,Func,HookFunc,SuiteInfo,MergeResult,Config) ->
end
end.
+initialize(RefreshLogs,Mod,Func,[Config]) when is_list(Config) ->
+ initialize(RefreshLogs,Mod,group_or_func(Func,Config));
+initialize(RefreshLogs,Mod,Func,_) ->
+ initialize(RefreshLogs,Mod,Func).
+
+initialize(RefreshLogs,Mod,FuncSpec) ->
+ ct_logs:init_tc(RefreshLogs),
+ ct_event:notify(#event{name=tc_start,
+ node=node(),
+ data={Mod,FuncSpec}}).
+
+
ct_suite_init(Suite,HookFunc,PostInitHook,Config) when is_list(Config) ->
case ct_hooks:init_tc(Suite,HookFunc,Config) of
NewConfig when is_list(NewConfig) ->
@@ -675,22 +683,35 @@ end_tc(Mod,Func,{Result,[Args]}, Return) ->
end_tc(Mod,Func,self(),Result,Args,Return).
end_tc(Mod,IPTC={init_per_testcase,_Func},_TCPid,Result,Args,Return) ->
- %% in case Mod == ct_framework, lookup the suite name
- Suite = get_suite_name(Mod, Args),
- case ct_hooks:end_tc(Suite,IPTC,Args,Result,Return) of
- '$ct_no_change' ->
- ok;
- HookResult ->
- HookResult
+ case end_hook_func(IPTC,Return,IPTC) of
+ undefined -> ok;
+ _ ->
+ %% in case Mod == ct_framework, lookup the suite name
+ Suite = get_suite_name(Mod, Args),
+ case ct_hooks:end_tc(Suite,IPTC,Args,Result,Return) of
+ '$ct_no_change' ->
+ ok;
+ HookResult ->
+ HookResult
+ end
end;
end_tc(Mod,Func0,TCPid,Result,Args,Return) ->
%% in case Mod == ct_framework, lookup the suite name
Suite = get_suite_name(Mod, Args),
- {EPTC,Func} = case Func0 of
- {end_per_testcase,F} -> {true,F};
- _ -> {false,Func0}
- end,
+ {Func,FuncSpec,HookFunc} =
+ case Func0 of
+ {end_per_testcase_not_run,F} ->
+ %% Testcase is completed (skipped or failed), but
+ %% end_per_testcase is not run - don't call post-hook.
+ {F,F,undefined};
+ {end_per_testcase,F} ->
+ {F,F,Func0};
+ _ ->
+ FS = group_or_func(Func0,Args),
+ HF = end_hook_func(Func0,Return,FS),
+ {Func0,FS,HF}
+ end,
test_server:timetrap_cancel(),
@@ -717,20 +738,18 @@ end_tc(Mod,Func0,TCPid,Result,Args,Return) ->
end,
ct_util:delete_suite_data(last_saved_config),
- {FuncSpec,HookFunc} =
- if not EPTC ->
- FS = group_or_func(Func,Args),
- {FS,FS};
- true ->
- {Func,Func0}
- end,
{Result1,FinalNotify} =
- case ct_hooks:end_tc(Suite,HookFunc,Args,Result,Return) of
- '$ct_no_change' ->
- {ok,Result};
- HookResult ->
- {HookResult,HookResult}
- end,
+ case HookFunc of
+ undefined ->
+ {ok,Result};
+ _ ->
+ case ct_hooks:end_tc(Suite,HookFunc,Args,Result,Return) of
+ '$ct_no_change' ->
+ {ok,Result};
+ HookResult ->
+ {HookResult,HookResult}
+ end
+ end,
FinalResult =
case get('$test_server_framework_test') of
undefined ->
@@ -821,6 +840,34 @@ end_tc(Mod,Func0,TCPid,Result,Args,Return) ->
end,
FinalResult.
+%% This is to make sure that no post_init_per_* is ever called if the
+%% corresponding pre_init_per_* was not called.
+%% The skip or fail reasons are those that can be returned from
+%% init_tc above in situations where we never came to call
+%% ct_hooks:init_tc/3, e.g. if suite/0 fails, then we never call
+%% ct_hooks:init_tc for init_per_suite, and thus we must not call
+%% ct_hooks:end_tc for init_per_suite either.
+end_hook_func({init_per_testcase,_},{auto_skip,{sequence_failed,_,_}},_) ->
+ undefined;
+end_hook_func({init_per_testcase,_},{auto_skip,"Repeated test stopped by force_stop option"},_) ->
+ undefined;
+end_hook_func({init_per_testcase,_},{fail,{config_name_already_in_use,_}},_) ->
+ undefined;
+end_hook_func({init_per_testcase,_},{auto_skip,{InfoFuncError,_}},_)
+ when InfoFuncError==testcase0_failed;
+ InfoFuncError==require_failed ->
+ undefined;
+end_hook_func(init_per_group,{auto_skip,{InfoFuncError,_}},_)
+ when InfoFuncError==group0_failed;
+ InfoFuncError==require_failed ->
+ undefined;
+end_hook_func(init_per_suite,{auto_skip,{require_failed_in_suite0,_}},_) ->
+ undefined;
+end_hook_func(init_per_suite,{auto_skip,{failed,{error,{suite0_failed,_}}}},_) ->
+ undefined;
+end_hook_func(_,_,Default) ->
+ Default.
+
%% {error,Reason} | {skip,Reason} | {timetrap_timeout,TVal} |
%% {testcase_aborted,Reason} | testcase_aborted_or_killed |
%% {'EXIT',Reason} | {fail,Reason} | {failed,Reason} |
@@ -1339,25 +1386,25 @@ report(What,Data) ->
ok;
tc_done ->
{Suite,{Func,GrName},Result} = Data,
- Data1 = if GrName == undefined -> {Suite,Func,Result};
- true -> Data
- end,
+ FuncSpec = if GrName == undefined -> Func;
+ true -> {Func,GrName}
+ end,
%% Register the group leader for the process calling the report
%% function, making it possible for a hook function to print
%% in the test case log file
ReportingPid = self(),
ct_logs:register_groupleader(ReportingPid, group_leader()),
case Result of
- {failed, _} ->
- ct_hooks:on_tc_fail(What, Data1);
- {skipped,{failed,{_,init_per_testcase,_}}} ->
- ct_hooks:on_tc_skip(tc_auto_skip, Data1);
- {skipped,{require_failed,_}} ->
- ct_hooks:on_tc_skip(tc_auto_skip, Data1);
- {skipped,_} ->
- ct_hooks:on_tc_skip(tc_user_skip, Data1);
- {auto_skipped,_} ->
- ct_hooks:on_tc_skip(tc_auto_skip, Data1);
+ {failed, Reason} ->
+ ct_hooks:on_tc_fail(What, {Suite,FuncSpec,Reason});
+ {skipped,{failed,{_,init_per_testcase,_}}=Reason} ->
+ ct_hooks:on_tc_skip(tc_auto_skip, {Suite,FuncSpec,Reason});
+ {skipped,{require_failed,_}=Reason} ->
+ ct_hooks:on_tc_skip(tc_auto_skip, {Suite,FuncSpec,Reason});
+ {skipped,Reason} ->
+ ct_hooks:on_tc_skip(tc_user_skip, {Suite,FuncSpec,Reason});
+ {auto_skipped,Reason} ->
+ ct_hooks:on_tc_skip(tc_auto_skip, {Suite,FuncSpec,Reason});
_Else ->
ok
end,
diff --git a/lib/common_test/src/ct_groups.erl b/lib/common_test/src/ct_groups.erl
index 1375e7dcc7..1c9faf6a70 100644
--- a/lib/common_test/src/ct_groups.erl
+++ b/lib/common_test/src/ct_groups.erl
@@ -442,17 +442,21 @@ make_conf(Mod, Name, Props, TestSpec) ->
ok
end,
{InitConf,EndConf,ExtraProps} =
- case erlang:function_exported(Mod,init_per_group,2) of
- true ->
- {{Mod,init_per_group},{Mod,end_per_group},[]};
- false ->
+ case {erlang:function_exported(Mod,init_per_group,2),
+ erlang:function_exported(Mod,end_per_group,2)} of
+ {false,false} ->
ct_logs:log("TEST INFO", "init_per_group/2 and "
"end_per_group/2 missing for group "
"~w in ~w, using default.",
[Name,Mod]),
{{ct_framework,init_per_group},
{ct_framework,end_per_group},
- [{suite,Mod}]}
+ [{suite,Mod}]};
+ _ ->
+ %% If any of these exist, the other should too
+ %% (required and documented). If it isn't, it will fail
+ %% with reason 'undef'.
+ {{Mod,init_per_group},{Mod,end_per_group},[]}
end,
{conf,[{name,Name}|Props++ExtraProps],InitConf,TestSpec,EndConf}.
diff --git a/lib/common_test/src/ct_hooks.erl b/lib/common_test/src/ct_hooks.erl
index c9a4abb5ee..60d1ea2b1c 100644
--- a/lib/common_test/src/ct_hooks.erl
+++ b/lib/common_test/src/ct_hooks.erl
@@ -92,15 +92,17 @@ init_tc(Mod, end_per_suite, Config) ->
call(fun call_generic/3, Config, [pre_end_per_suite, Mod]);
init_tc(Mod, {init_per_group, GroupName, Properties}, Config) ->
maybe_start_locker(Mod, GroupName, Properties),
- call(fun call_generic/3, Config, [pre_init_per_group, GroupName]);
-init_tc(_Mod, {end_per_group, GroupName, _}, Config) ->
- call(fun call_generic/3, Config, [pre_end_per_group, GroupName]);
-init_tc(_Mod, {init_per_testcase,TC}, Config) ->
- call(fun call_generic/3, Config, [pre_init_per_testcase, TC]);
-init_tc(_Mod, {end_per_testcase,TC}, Config) ->
- call(fun call_generic/3, Config, [pre_end_per_testcase, TC]);
-init_tc(_Mod, TC = error_in_suite, Config) ->
- call(fun call_generic/3, Config, [pre_init_per_testcase, TC]).
+ call(fun call_generic_fallback/3, Config,
+ [pre_init_per_group, Mod, GroupName]);
+init_tc(Mod, {end_per_group, GroupName, _}, Config) ->
+ call(fun call_generic_fallback/3, Config,
+ [pre_end_per_group, Mod, GroupName]);
+init_tc(Mod, {init_per_testcase,TC}, Config) ->
+ call(fun call_generic_fallback/3, Config, [pre_init_per_testcase, Mod, TC]);
+init_tc(Mod, {end_per_testcase,TC}, Config) ->
+ call(fun call_generic_fallback/3, Config, [pre_end_per_testcase, Mod, TC]);
+init_tc(Mod, TC = error_in_suite, Config) ->
+ call(fun call_generic_fallback/3, Config, [pre_init_per_testcase, Mod, TC]).
%% @doc Called as each test case is completed. This includes all configuration
%% tests.
@@ -126,23 +128,23 @@ end_tc(Mod, init_per_suite, Config, _Result, Return) ->
end_tc(Mod, end_per_suite, Config, Result, _Return) ->
call(fun call_generic/3, Result, [post_end_per_suite, Mod, Config],
'$ct_no_change');
-end_tc(_Mod, {init_per_group, GroupName, _}, Config, _Result, Return) ->
- call(fun call_generic/3, Return, [post_init_per_group, GroupName, Config],
- '$ct_no_change');
+end_tc(Mod, {init_per_group, GroupName, _}, Config, _Result, Return) ->
+ call(fun call_generic_fallback/3, Return,
+ [post_init_per_group, Mod, GroupName, Config], '$ct_no_change');
end_tc(Mod, {end_per_group, GroupName, Properties}, Config, Result, _Return) ->
- Res = call(fun call_generic/3, Result,
- [post_end_per_group, GroupName, Config], '$ct_no_change'),
+ Res = call(fun call_generic_fallback/3, Result,
+ [post_end_per_group, Mod, GroupName, Config], '$ct_no_change'),
maybe_stop_locker(Mod, GroupName, Properties),
Res;
-end_tc(_Mod, {init_per_testcase,TC}, Config, Result, _Return) ->
- call(fun call_generic/3, Result, [post_init_per_testcase, TC, Config],
- '$ct_no_change');
-end_tc(_Mod, {end_per_testcase,TC}, Config, Result, _Return) ->
- call(fun call_generic/3, Result, [post_end_per_testcase, TC, Config],
- '$ct_no_change');
-end_tc(_Mod, TC = error_in_suite, Config, Result, _Return) ->
- call(fun call_generic/3, Result, [post_end_per_testcase, TC, Config],
- '$ct_no_change').
+end_tc(Mod, {init_per_testcase,TC}, Config, Result, _Return) ->
+ call(fun call_generic_fallback/3, Result,
+ [post_init_per_testcase, Mod, TC, Config], '$ct_no_change');
+end_tc(Mod, {end_per_testcase,TC}, Config, Result, _Return) ->
+ call(fun call_generic_fallback/3, Result,
+ [post_end_per_testcase, Mod, TC, Config], '$ct_no_change');
+end_tc(Mod, TC = error_in_suite, Config, Result, _Return) ->
+ call(fun call_generic_fallback/3, Result,
+ [post_end_per_testcase, Mod, TC, Config], '$ct_no_change').
%% Case = TestCase | {TestCase,GroupName}
@@ -181,15 +183,21 @@ call_terminate(#ct_hook_config{ module = Mod, state = State} = Hook, _, _) ->
{[],Hook}.
call_cleanup(#ct_hook_config{ module = Mod, state = State} = Hook,
- Reason, [Function, _Suite | Args]) ->
+ Reason, [Function | Args]) ->
NewState = catch_apply(Mod,Function, Args ++ [Reason, State],
- State),
+ State, true),
{Reason, Hook#ct_hook_config{ state = NewState } }.
-call_generic(#ct_hook_config{ module = Mod, state = State} = Hook,
- Value, [Function | Args]) ->
+call_generic(Hook, Value, Meta) ->
+ do_call_generic(Hook, Value, Meta, false).
+
+call_generic_fallback(Hook, Value, Meta) ->
+ do_call_generic(Hook, Value, Meta, true).
+
+do_call_generic(#ct_hook_config{ module = Mod, state = State} = Hook,
+ Value, [Function | Args], Fallback) ->
{NewValue, NewState} = catch_apply(Mod, Function, Args ++ [Value, State],
- {Value,State}),
+ {Value,State}, Fallback),
{NewValue, Hook#ct_hook_config{ state = NewState } }.
%% Generic call function
@@ -257,15 +265,15 @@ remove(Key,List) when is_list(List) ->
remove(_, Else) ->
Else.
-%% Translate scopes, i.e. init_per_group,group1 -> end_per_group,group1 etc
-scope([pre_init_per_testcase, TC|_]) ->
- [post_init_per_testcase, TC];
-scope([pre_end_per_testcase, TC|_]) ->
- [post_end_per_testcase, TC];
-scope([pre_init_per_group, GroupName|_]) ->
- [post_end_per_group, GroupName];
-scope([post_init_per_group, GroupName|_]) ->
- [post_end_per_group, GroupName];
+%% Translate scopes, i.e. is_tuplenit_per_group,group1 -> end_per_group,group1 etc
+scope([pre_init_per_testcase, SuiteName, TC|_]) ->
+ [post_init_per_testcase, SuiteName, TC];
+scope([pre_end_per_testcase, SuiteName, TC|_]) ->
+ [post_end_per_testcase, SuiteName, TC];
+scope([pre_init_per_group, SuiteName, GroupName|_]) ->
+ [post_end_per_group, SuiteName, GroupName];
+scope([post_init_per_group, SuiteName, GroupName|_]) ->
+ [post_end_per_group, SuiteName, GroupName];
scope([pre_init_per_suite, SuiteName|_]) ->
[post_end_per_suite, SuiteName];
scope([post_init_per_suite, SuiteName|_]) ->
@@ -273,14 +281,29 @@ scope([post_init_per_suite, SuiteName|_]) ->
scope(init) ->
none.
-terminate_if_scope_ends(HookId, [on_tc_skip,_Suite,{end_per_group,Name}],
+strip_config([post_init_per_testcase, SuiteName, TC|_]) ->
+ [post_init_per_testcase, SuiteName, TC];
+strip_config([post_end_per_testcase, SuiteName, TC|_]) ->
+ [post_end_per_testcase, SuiteName, TC];
+strip_config([post_init_per_group, SuiteName, GroupName|_]) ->
+ [post_init_per_group, SuiteName, GroupName];
+strip_config([post_end_per_group, SuiteName, GroupName|_]) ->
+ [post_end_per_group, SuiteName, GroupName];
+strip_config([post_init_per_suite, SuiteName|_]) ->
+ [post_init_per_suite, SuiteName];
+strip_config([post_end_per_suite, SuiteName|_]) ->
+ [post_end_per_suite, SuiteName];
+strip_config(Other) ->
+ Other.
+
+
+terminate_if_scope_ends(HookId, [on_tc_skip,Suite,{end_per_group,Name}],
Hooks) ->
- terminate_if_scope_ends(HookId, [post_end_per_group, Name], Hooks);
+ terminate_if_scope_ends(HookId, [post_end_per_group, Suite, Name], Hooks);
terminate_if_scope_ends(HookId, [on_tc_skip,Suite,end_per_suite], Hooks) ->
terminate_if_scope_ends(HookId, [post_end_per_suite, Suite], Hooks);
-terminate_if_scope_ends(HookId, [Function,Tag|T], Hooks) when T =/= [] ->
- terminate_if_scope_ends(HookId,[Function,Tag],Hooks);
-terminate_if_scope_ends(HookId, Function, Hooks) ->
+terminate_if_scope_ends(HookId, Function0, Hooks) ->
+ Function = strip_config(Function0),
case lists:keyfind(HookId, #ct_hook_config.id, Hooks) of
#ct_hook_config{ id = HookId, scope = Function} = Hook ->
terminate([Hook]),
@@ -384,21 +407,29 @@ pos(Id,[_|Rest],Num) ->
catch_apply(M,F,A, Default) ->
+ catch_apply(M,F,A,Default,false).
+catch_apply(M,F,A, Default, Fallback) ->
+ not erlang:module_loaded(M) andalso (catch M:module_info()),
+ case erlang:function_exported(M,F,length(A)) of
+ false when Fallback ->
+ catch_apply(M,F,tl(A),Default,false);
+ false ->
+ Default;
+ true ->
+ catch_apply(M,F,A)
+ end.
+
+catch_apply(M,F,A) ->
try
- erlang:apply(M,F,A)
+ erlang:apply(M,F,A)
catch _:Reason ->
- case erlang:get_stacktrace() of
- %% Return the default if it was the CTH module which did not have the function.
- [{M,F,A,_}|_] when Reason == undef ->
- Default;
- Trace ->
- ct_logs:log("Suite Hook","Call to CTH failed: ~w:~p",
- [error,{Reason,Trace}]),
- throw({error_in_cth_call,
- lists:flatten(
- io_lib:format("~w:~w/~w CTH call failed",
- [M,F,length(A)]))})
- end
+ Trace = erlang:get_stacktrace(),
+ ct_logs:log("Suite Hook","Call to CTH failed: ~w:~p",
+ [error,{Reason,Trace}]),
+ throw({error_in_cth_call,
+ lists:flatten(
+ io_lib:format("~w:~w/~w CTH call failed",
+ [M,F,length(A)]))})
end.
diff --git a/lib/common_test/src/cth_conn_log.erl b/lib/common_test/src/cth_conn_log.erl
index 883da0da0a..ce8852b3ea 100644
--- a/lib/common_test/src/cth_conn_log.erl
+++ b/lib/common_test/src/cth_conn_log.erl
@@ -54,8 +54,8 @@
-include_lib("common_test/include/ct.hrl").
-export([init/2,
- pre_init_per_testcase/3,
- post_end_per_testcase/4]).
+ pre_init_per_testcase/4,
+ post_end_per_testcase/5]).
%%----------------------------------------------------------------------
%% Exported types
@@ -104,7 +104,7 @@ get_log_opts(Mod,Opts) ->
Hosts = proplists:get_value(hosts,Opts,[]),
{LogType,Hosts}.
-pre_init_per_testcase(TestCase,Config,CthState) ->
+pre_init_per_testcase(_Suite,TestCase,Config,CthState) ->
Logs =
lists:map(
fun({ConnMod,{LogType,Hosts}}) ->
@@ -158,7 +158,7 @@ pre_init_per_testcase(TestCase,Config,CthState) ->
ct_util:update_testdata(?MODULE, Update, [create]),
{Config,CthState}.
-post_end_per_testcase(TestCase,_Config,Return,CthState) ->
+post_end_per_testcase(_Suite,TestCase,_Config,Return,CthState) ->
Update =
fun(PrevUsers) ->
case lists:delete(TestCase, PrevUsers) of
diff --git a/lib/common_test/src/cth_log_redirect.erl b/lib/common_test/src/cth_log_redirect.erl
index 6d77d7ee9e..eda090d4f5 100644
--- a/lib/common_test/src/cth_log_redirect.erl
+++ b/lib/common_test/src/cth_log_redirect.erl
@@ -28,10 +28,10 @@
%% CTH Callbacks
-export([id/1, init/2,
pre_init_per_suite/3, pre_end_per_suite/3, post_end_per_suite/4,
- pre_init_per_group/3, post_init_per_group/4,
- pre_end_per_group/3, post_end_per_group/4,
- pre_init_per_testcase/3, post_init_per_testcase/4,
- pre_end_per_testcase/3, post_end_per_testcase/4]).
+ pre_init_per_group/4, post_init_per_group/5,
+ pre_end_per_group/4, post_end_per_group/5,
+ pre_init_per_testcase/4, post_init_per_testcase/5,
+ pre_end_per_testcase/4, post_end_per_testcase/5]).
%% Event handler Callbacks
-export([init/1,
@@ -71,11 +71,11 @@ post_end_per_suite(_Suite, Config, Return, State) ->
set_curr_func(undefined, Config),
{Return, State}.
-pre_init_per_group(Group, Config, State) ->
+pre_init_per_group(_Suite, Group, Config, State) ->
set_curr_func({group,Group,init_per_group}, Config),
{Config, State}.
-post_init_per_group(Group, Config, Result, tc_log_async) when is_list(Config) ->
+post_init_per_group(_Suite, Group, Config, Result, tc_log_async) when is_list(Config) ->
case lists:member(parallel,proplists:get_value(
tc_group_properties,Config,[])) of
true ->
@@ -83,33 +83,33 @@ post_init_per_group(Group, Config, Result, tc_log_async) when is_list(Config) ->
false ->
{Result, tc_log_async}
end;
-post_init_per_group(_Group, _Config, Result, State) ->
+post_init_per_group(_Suite, _Group, _Config, Result, State) ->
{Result, State}.
-pre_init_per_testcase(TC, Config, State) ->
+pre_init_per_testcase(_Suite, TC, Config, State) ->
set_curr_func(TC, Config),
{Config, State}.
-post_init_per_testcase(_TC, _Config, Return, State) ->
+post_init_per_testcase(_Suite, _TC, _Config, Return, State) ->
{Return, State}.
-pre_end_per_testcase(_TC, Config, State) ->
+pre_end_per_testcase(_Suite, _TC, Config, State) ->
{Config, State}.
-post_end_per_testcase(_TC, _Config, Result, State) ->
+post_end_per_testcase(_Suite, _TC, _Config, Result, State) ->
%% Make sure that the event queue is flushed
%% before ending this test case.
gen_event:call(error_logger, ?MODULE, flush, 300000),
{Result, State}.
-pre_end_per_group(Group, Config, {tc_log, Group}) ->
+pre_end_per_group(_Suite, Group, Config, {tc_log, Group}) ->
set_curr_func({group,Group,end_per_group}, Config),
{Config, set_log_func(tc_log_async)};
-pre_end_per_group(Group, Config, State) ->
+pre_end_per_group(_Suite, Group, Config, State) ->
set_curr_func({group,Group,end_per_group}, Config),
{Config, State}.
-post_end_per_group(_Group, Config, Return, State) ->
+post_end_per_group(_Suite, _Group, Config, Return, State) ->
set_curr_func({group,undefined}, Config),
{Return, State}.
diff --git a/lib/common_test/src/cth_surefire.erl b/lib/common_test/src/cth_surefire.erl
index 59b916851e..c4941948cc 100644
--- a/lib/common_test/src/cth_surefire.erl
+++ b/lib/common_test/src/cth_surefire.erl
@@ -33,16 +33,16 @@
-export([pre_end_per_suite/3]).
-export([post_end_per_suite/4]).
--export([pre_init_per_group/3]).
--export([post_init_per_group/4]).
--export([pre_end_per_group/3]).
--export([post_end_per_group/4]).
+-export([pre_init_per_group/4]).
+-export([post_init_per_group/5]).
+-export([pre_end_per_group/4]).
+-export([post_end_per_group/5]).
--export([pre_init_per_testcase/3]).
--export([post_end_per_testcase/4]).
+-export([pre_init_per_testcase/4]).
+-export([post_end_per_testcase/5]).
--export([on_tc_fail/3]).
--export([on_tc_skip/3]).
+-export([on_tc_fail/4]).
+-export([on_tc_skip/4]).
-export([terminate/1]).
@@ -116,29 +116,29 @@ pre_end_per_suite(_Suite,Config,State) ->
post_end_per_suite(_Suite,Config,Result,State) ->
{Result, end_tc(end_per_suite,Config,Result,State)}.
-pre_init_per_group(Group,Config,State) ->
+pre_init_per_group(_Suite,Group,Config,State) ->
{Config, init_tc(State#state{ curr_group = [Group|State#state.curr_group]},
Config)}.
-post_init_per_group(_Group,Config,Result,State) ->
+post_init_per_group(_Suite,_Group,Config,Result,State) ->
{Result, end_tc(init_per_group,Config,Result,State)}.
-pre_end_per_group(_Group,Config,State) ->
+pre_end_per_group(_Suite,_Group,Config,State) ->
{Config, init_tc(State, Config)}.
-post_end_per_group(_Group,Config,Result,State) ->
+post_end_per_group(_Suite,_Group,Config,Result,State) ->
NewState = end_tc(end_per_group, Config, Result, State),
{Result, NewState#state{ curr_group = tl(NewState#state.curr_group)}}.
-pre_init_per_testcase(_TC,Config,State) ->
+pre_init_per_testcase(_Suite,_TC,Config,State) ->
{Config, init_tc(State, Config)}.
-post_end_per_testcase(TC,Config,Result,State) ->
+post_end_per_testcase(_Suite,TC,Config,Result,State) ->
{Result, end_tc(TC,Config, Result,State)}.
-on_tc_fail(_TC, _Res, State = #state{test_cases = []}) ->
+on_tc_fail(_Suite,_TC, _Res, State = #state{test_cases = []}) ->
State;
-on_tc_fail(_TC, Res, State) ->
+on_tc_fail(_Suite,_TC, Res, State) ->
TCs = State#state.test_cases,
TC = hd(TCs),
NewTC = TC#testcase{
@@ -146,10 +146,9 @@ on_tc_fail(_TC, Res, State) ->
{fail,lists:flatten(io_lib:format("~p",[Res]))} },
State#state{ test_cases = [NewTC | tl(TCs)]}.
-on_tc_skip({ConfigFunc,_GrName},{Type,_Reason} = Res, State0)
- when Type == tc_auto_skip; Type == tc_user_skip ->
- on_tc_skip(ConfigFunc, Res, State0);
-on_tc_skip(Tc,{Type,_Reason} = Res, State0) when Type == tc_auto_skip ->
+on_tc_skip(Suite,{ConfigFunc,_GrName}, Res, State) ->
+ on_tc_skip(Suite,ConfigFunc, Res, State);
+on_tc_skip(Suite,Tc, Res, State0) ->
TcStr = atom_to_list(Tc),
State =
case State0#state.test_cases of
@@ -158,11 +157,7 @@ on_tc_skip(Tc,{Type,_Reason} = Res, State0) when Type == tc_auto_skip ->
_ ->
State0
end,
- do_tc_skip(Res, end_tc(Tc,[],Res,init_tc(State,[])));
-on_tc_skip(_Tc, _Res, State = #state{test_cases = []}) ->
- State;
-on_tc_skip(_Tc, Res, State) ->
- do_tc_skip(Res, State).
+ do_tc_skip(Res, end_tc(Tc,[],Res,init_tc(set_suite(Suite,State),[]))).
do_tc_skip(Res, State) ->
TCs = State#state.test_cases,
@@ -209,6 +204,12 @@ end_tc(Name, _Config, _Res, State = #state{ curr_suite = Suite,
result = passed }|
State#state.test_cases],
tc_log = ""}. % so old tc_log is not set if next is on_tc_skip
+
+set_suite(Suite,#state{curr_suite=undefined}=State) ->
+ State#state{curr_suite=Suite, curr_suite_ts=?now};
+set_suite(_,State) ->
+ State.
+
close_suite(#state{ test_cases = [] } = State) ->
State;
close_suite(#state{ test_cases = TCs, url_base = UrlBase } = State) ->
@@ -228,7 +229,8 @@ close_suite(#state{ test_cases = TCs, url_base = UrlBase } = State) ->
testcases = lists:reverse(TCs),
log = SuiteLog,
url = SuiteUrl},
- State#state{ test_cases = [],
+ State#state{ curr_suite = undefined,
+ test_cases = [],
test_suites = [Suite | State#state.test_suites]}.
terminate(State = #state{ test_cases = [] }) ->
diff --git a/lib/common_test/src/test_server.erl b/lib/common_test/src/test_server.erl
index 924086f2bd..be49191f2e 100644
--- a/lib/common_test/src/test_server.erl
+++ b/lib/common_test/src/test_server.erl
@@ -778,9 +778,9 @@ spawn_fw_call(Mod,IPTC={init_per_testcase,Func},CurrConf,Pid,
%% if init_per_testcase fails, the test case
%% should be skipped
try begin do_end_tc_call(Mod,IPTC, {Pid,Skip,[CurrConf]}, Why),
- do_init_tc_call(Mod,{end_per_testcase,Func},
+ do_init_tc_call(Mod,{end_per_testcase_not_run,Func},
[CurrConf],{ok,[CurrConf]}),
- do_end_tc_call(Mod,{end_per_testcase,Func},
+ do_end_tc_call(Mod,{end_per_testcase_not_run,Func},
{Pid,Skip,[CurrConf]}, Why) end of
_ -> ok
catch
@@ -1151,14 +1151,14 @@ do_end_tc_call(Mod, IPTC={init_per_testcase,Func}, Res, Return) ->
Args
end,
EPTCInitRes =
- case do_init_tc_call(Mod,{end_per_testcase,Func},
+ case do_init_tc_call(Mod,{end_per_testcase_not_run,Func},
IPTCEndRes,Return) of
{ok,EPTCInitConfig} when is_list(EPTCInitConfig) ->
{Return,EPTCInitConfig};
_ ->
- Return
+ {Return,IPTCEndRes}
end,
- do_end_tc_call1(Mod, {end_per_testcase,Func},
+ do_end_tc_call1(Mod, {end_per_testcase_not_run,Func},
EPTCInitRes, Return);
_Ok ->
do_end_tc_call1(Mod, IPTC, Res, Return)
diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl
index b52e4bef9b..39c523f8b3 100644
--- a/lib/common_test/src/test_server_ctrl.erl
+++ b/lib/common_test/src/test_server_ctrl.erl
@@ -2051,17 +2051,21 @@ add_init_and_end_per_suite([], _LastMod, skipped_suite, _FwMod) ->
add_init_and_end_per_suite([], LastMod, LastRef, FwMod) ->
%% we'll add end_per_suite here even if it's not exported
%% (and simply let the call fail if it's missing)
- case erlang:function_exported(LastMod, end_per_suite, 1) of
- true ->
- [{conf,LastRef,[],{LastMod,end_per_suite}}];
- false ->
+ case {erlang:function_exported(LastMod, end_per_suite, 1),
+ erlang:function_exported(LastMod, init_per_suite, 1)} of
+ {false,false} ->
%% let's call a "fake" end_per_suite if it exists
case erlang:function_exported(FwMod, end_per_suite, 1) of
true ->
[{conf,LastRef,[{suite,LastMod}],{FwMod,end_per_suite}}];
false ->
[{conf,LastRef,[],{LastMod,end_per_suite}}]
- end
+ end;
+ _ ->
+ %% If any of these exist, the other should too
+ %% (required and documented). If it isn't, it will fail
+ %% with reason 'undef'.
+ [{conf,LastRef,[],{LastMod,end_per_suite}}]
end.
do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) ->
@@ -2070,11 +2074,9 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) ->
_ -> ok
end,
{Init,NextMod,NextRef} =
- case erlang:function_exported(Mod, init_per_suite, 1) of
- true ->
- Ref = make_ref(),
- {[{conf,Ref,[],{Mod,init_per_suite}}],Mod,Ref};
- false ->
+ case {erlang:function_exported(Mod, init_per_suite, 1),
+ erlang:function_exported(Mod, end_per_suite, 1)} of
+ {false,false} ->
%% let's call a "fake" init_per_suite if it exists
case erlang:function_exported(FwMod, init_per_suite, 1) of
true ->
@@ -2083,8 +2085,13 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) ->
{FwMod,init_per_suite}}],Mod,Ref};
false ->
{[],Mod,undefined}
- end
-
+ end;
+ _ ->
+ %% If any of these exist, the other should too
+ %% (required and documented). If it isn't, it will fail
+ %% with reason 'undef'.
+ Ref = make_ref(),
+ {[{conf,Ref,[],{Mod,init_per_suite}}],Mod,Ref}
end,
Cases =
if LastRef==undefined ->
@@ -2094,10 +2101,9 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) ->
true ->
%% we'll add end_per_suite here even if it's not exported
%% (and simply let the call fail if it's missing)
- case erlang:function_exported(LastMod, end_per_suite, 1) of
- true ->
- [{conf,LastRef,[],{LastMod,end_per_suite}}|Init];
- false ->
+ case {erlang:function_exported(LastMod, end_per_suite, 1),
+ erlang:function_exported(LastMod, init_per_suite, 1)} of
+ {false,false} ->
%% let's call a "fake" end_per_suite if it exists
case erlang:function_exported(FwMod, end_per_suite, 1) of
true ->
@@ -2105,8 +2111,13 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) ->
{FwMod,end_per_suite}}|Init];
false ->
[{conf,LastRef,[],{LastMod,end_per_suite}}|Init]
- end
- end
+ end;
+ _ ->
+ %% If any of these exist, the other should too
+ %% (required and documented). If it isn't, it will fail
+ %% with reason 'undef'.
+ [{conf,LastRef,[],{LastMod,end_per_suite}}|Init]
+ end
end,
{Cases,NextMod,NextRef}.
@@ -2115,11 +2126,9 @@ do_add_end_per_suite_and_skip(LastMod, LastRef, Mod, FwMod) ->
No when No==undefined ; No==skipped_suite ->
{[],Mod,skipped_suite};
_Ref ->
- case erlang:function_exported(LastMod, end_per_suite, 1) of
- true ->
- {[{conf,LastRef,[],{LastMod,end_per_suite}}],
- Mod,skipped_suite};
- false ->
+ case {erlang:function_exported(LastMod, end_per_suite, 1),
+ erlang:function_exported(LastMod, init_per_suite, 1)} of
+ {false,false} ->
case erlang:function_exported(FwMod, end_per_suite, 1) of
true ->
%% let's call "fake" end_per_suite if it exists
@@ -2128,7 +2137,13 @@ do_add_end_per_suite_and_skip(LastMod, LastRef, Mod, FwMod) ->
false ->
{[{conf,LastRef,[],{LastMod,end_per_suite}}],
Mod,skipped_suite}
- end
+ end;
+ _ ->
+ %% If any of these exist, the other should too
+ %% (required and documented). If it isn't, it will fail
+ %% with reason 'undef'.
+ {[{conf,LastRef,[],{LastMod,end_per_suite}}],
+ Mod,skipped_suite}
end
end.
@@ -2924,22 +2939,21 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status)
exit(framework_error);
%% sequential execution of test case finished
{Time,RetVal,_} ->
+ RetTag =
+ if is_tuple(RetVal) -> element(1,RetVal);
+ true -> undefined
+ end,
{Failed,Status1} =
- case Time of
- died ->
- {true,update_status(failed, Mod, Func, Status)};
- _ when is_tuple(RetVal) ->
- case element(1, RetVal) of
- R when R=='EXIT'; R==failed ->
- {true,update_status(failed, Mod, Func, Status)};
- R when R==skip; R==skipped ->
- {false,update_status(skipped, Mod, Func, Status)};
- _ ->
- {false,update_status(ok, Mod, Func, Status)}
- end;
- _ ->
- {false,update_status(ok, Mod, Func, Status)}
- end,
+ case RetTag of
+ Skip when Skip==skip; Skip==skipped ->
+ {false,update_status(skipped, Mod, Func, Status)};
+ Fail when Fail=='EXIT'; Fail==failed ->
+ {true,update_status(failed, Mod, Func, Status)};
+ _ when Time==died, RetVal=/=ok ->
+ {true,update_status(failed, Mod, Func, Status)};
+ _ ->
+ {false,update_status(ok, Mod, Func, Status)}
+ end,
case check_prop(sequence, Mode) of
false ->
stop_minor_log_file(),
@@ -3794,7 +3808,15 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit,
{died,{timetrap_timeout,TimetrapTimeout}} ->
progress(failed, Num, Mod, Func, GrName, Loc,
timetrap_timeout, TimetrapTimeout, Comment, Style);
- {died,Reason} ->
+ {died,{Skip,Reason}} when Skip==skip; Skip==skipped ->
+ %% died in init_per_testcase
+ progress(skip, Num, Mod, Func, GrName, Loc, Reason,
+ Time, Comment, Style);
+ {died,Reason} when Reason=/=ok ->
+ %% (If Reason==ok it means that process died in
+ %% end_per_testcase after successfully completing the
+ %% test case itself - then we shall not fail, but a
+ %% warning will be issued in the comment field.)
progress(failed, Num, Mod, Func, GrName, Loc, Reason,
Time, Comment, Style);
{_,{'EXIT',{Skip,Reason}}} when Skip==skip; Skip==skipped;
@@ -3943,6 +3965,9 @@ progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time,
[get_info_str(Mod,Func, CaseNum, get(test_server_cases))]),
test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName},
{ReportTag,Reason1}}]),
+ TimeStr = io_lib:format(if is_float(Time) -> "~.3fs";
+ true -> "~w"
+ end, [Time]),
ReasonStr = escape_chars(reason_to_string(Reason1)),
ReasonStr1 = lists:flatten([string:strip(S,left) ||
S <- string:tokens(ReasonStr,[$\n])]),
@@ -3957,10 +3982,10 @@ progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time,
_ -> xhtml("<br>(","<br />(") ++ to_string(Comment) ++ ")"
end,
print(html,
- "<td>" ++ St0 ++ "~.3fs" ++ St1 ++ "</td>"
+ "<td>" ++ St0 ++ "~ts" ++ St1 ++ "</td>"
"<td><font color=\"~ts\">SKIPPED</font></td>"
"<td>~ts~ts</td></tr>\n",
- [Time,Color,ReasonStr2,Comment1]),
+ [TimeStr,Color,ReasonStr2,Comment1]),
FormatLoc = test_server_sup:format_loc(Loc),
print(minor, "=== Location: ~ts", [FormatLoc]),
print(minor, "=== Reason: ~ts", [ReasonStr1]),
@@ -4098,6 +4123,9 @@ progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time,
Comment0, {St0,St1}) ->
print(minor, "successfully completed test case", []),
test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName},ok}]),
+ TimeStr = io_lib:format(if is_float(Time) -> "~.3fs";
+ true -> "~w"
+ end, [Time]),
Comment =
case RetVal of
{comment,RetComment} ->
@@ -4116,10 +4144,10 @@ progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time,
end,
print(major, "=elapsed ~p", [Time]),
print(html,
- "<td>" ++ St0 ++ "~.3fs" ++ St1 ++ "</td>"
+ "<td>" ++ St0 ++ "~ts" ++ St1 ++ "</td>"
"<td><font color=\"green\">Ok</font></td>"
"~ts</tr>\n",
- [Time,Comment]),
+ [TimeStr,Comment]),
print(minor,
escape_chars(io_lib:format("=== Returned value: ~tp", [RetVal])),
[]),