aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test/src/ct_framework.erl
diff options
context:
space:
mode:
Diffstat (limited to 'lib/common_test/src/ct_framework.erl')
-rw-r--r--lib/common_test/src/ct_framework.erl360
1 files changed, 246 insertions, 114 deletions
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index 141c7f5b0a..0437d6786f 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -312,7 +312,7 @@ add_defaults(Mod,Func, GroupPath) ->
end;
{'EXIT',Reason} ->
ErrStr = io_lib:format("~n*** ERROR *** "
- "~w:suite/0 failed: ~p~n",
+ "~w:suite/0 failed: ~tp~n",
[Suite,Reason]),
io:format(ErrStr, []),
io:format(?def_gl, ErrStr, []),
@@ -335,7 +335,7 @@ add_defaults(Mod,Func, GroupPath) ->
false ->
ErrStr = io_lib:format("~n*** ERROR *** "
"Invalid return value from "
- "~w:suite/0: ~p~n",
+ "~w:suite/0: ~tp~n",
[Suite,SuiteInfo]),
io:format(ErrStr, []),
io:format(?def_gl, ErrStr, []),
@@ -344,7 +344,7 @@ add_defaults(Mod,Func, GroupPath) ->
SuiteInfo ->
ErrStr = io_lib:format("~n*** ERROR *** "
"Invalid return value from "
- "~w:suite/0: ~p~n", [Suite,SuiteInfo]),
+ "~w:suite/0: ~tp~n", [Suite,SuiteInfo]),
io:format(ErrStr, []),
io:format(?def_gl, ErrStr, []),
{suite0_failed,bad_return_value}
@@ -371,7 +371,7 @@ add_defaults1(Mod,Func, GroupPath, SuiteInfo) ->
{value,{error,BadGr0Val,GrName}} ->
Gr0ErrStr = io_lib:format("~n*** ERROR *** "
"Invalid return value from "
- "~w:group(~w): ~p~n",
+ "~w:group(~tw): ~tp~n",
[Mod,GrName,BadGr0Val]),
io:format(Gr0ErrStr, []),
io:format(?def_gl, Gr0ErrStr, []),
@@ -393,7 +393,7 @@ add_defaults1(Mod,Func, GroupPath, SuiteInfo) ->
{error,BadTC0Val} ->
TC0ErrStr = io_lib:format("~n*** ERROR *** "
"Invalid return value from "
- "~w:~w/0: ~p~n",
+ "~w:~tw/0: ~tp~n",
[Mod,Func,BadTC0Val]),
io:format(TC0ErrStr, []),
io:format(?def_gl, TC0ErrStr, []),
@@ -696,9 +696,16 @@ end_tc(Mod,IPTC={init_per_testcase,_Func},_TCPid,Result,Args,Return) ->
end
end;
-end_tc(Mod,Func0,TCPid,Result,Args,Return) ->
+end_tc(Mod,Func00,TCPid,Result,Args,Return) ->
%% in case Mod == ct_framework, lookup the suite name
Suite = get_suite_name(Mod, Args),
+ {OnlyCleanup,Func0} =
+ case Func00 of
+ {cleanup,F0} ->
+ {true,F0};
+ _ ->
+ {false,Func00}
+ end,
{Func,FuncSpec,HookFunc} =
case Func0 of
{end_per_testcase_not_run,F} ->
@@ -742,6 +749,8 @@ end_tc(Mod,Func0,TCPid,Result,Args,Return) ->
case HookFunc of
undefined ->
{ok,Result};
+ _ when OnlyCleanup ->
+ {ok,Result};
_ ->
case ct_hooks:end_tc(Suite,HookFunc,Args,Result,Return) of
'$ct_no_change' ->
@@ -752,6 +761,8 @@ end_tc(Mod,Func0,TCPid,Result,Args,Return) ->
end,
FinalResult =
case get('$test_server_framework_test') of
+ _ when OnlyCleanup ->
+ Result1;
undefined ->
%% send sync notification so that event handlers may print
%% in the log file before it gets closed
@@ -921,7 +932,7 @@ error_notification(Mod,Func,_Args,{Error,Loc}) ->
end,
ErrorStr = case ErrorSpec of
{badmatch,Descr} ->
- Descr1 = lists:flatten(io_lib:format("~P",[Descr,10])),
+ Descr1 = lists:flatten(io_lib:format("~tP",[Descr,10])),
if length(Descr1) > 50 ->
Descr2 = string:substr(Descr1,1,50),
io_lib:format("{badmatch,~ts...}",[Descr2]);
@@ -931,15 +942,15 @@ error_notification(Mod,Func,_Args,{Error,Loc}) ->
{test_case_failed,Reason} ->
case (catch io_lib:format("{test_case_failed,~ts}", [Reason])) of
{'EXIT',_} ->
- io_lib:format("{test_case_failed,~p}", [Reason]);
+ io_lib:format("{test_case_failed,~tp}", [Reason]);
Result -> Result
end;
{'EXIT',_Reason} = EXIT ->
- io_lib:format("~P", [EXIT,5]);
+ io_lib:format("~tP", [EXIT,5]);
{Spec,_Reason} when is_atom(Spec) ->
- io_lib:format("~w", [Spec]);
+ io_lib:format("~tw", [Spec]);
Other ->
- io_lib:format("~P", [Other,5])
+ io_lib:format("~tP", [Other,5])
end,
ErrorHtml =
"<font color=\"brown\">" ++ ct_logs:escape_chars(ErrorStr) ++ "</font>",
@@ -996,16 +1007,16 @@ error_notification(Mod,Func,_Args,{Error,Loc}) ->
%% if a function specified by all/0 does not exist, we
%% pick up undef here
[{LastMod,LastFunc}|_] when ErrorStr == "undef" ->
- PrintError("~w:~w could not be executed~nReason: ~ts",
+ PrintError("~w:~tw could not be executed~nReason: ~ts",
[LastMod,LastFunc,ErrorStr]);
[{LastMod,LastFunc}|_] ->
- PrintError("~w:~w failed~nReason: ~ts", [LastMod,LastFunc,ErrorStr]);
+ PrintError("~w:~tw failed~nReason: ~ts", [LastMod,LastFunc,ErrorStr]);
[{LastMod,LastFunc,LastLine}|_] ->
%% print error to console, we are only
%% interested in the last executed expression
- PrintError("~w:~w failed on line ~w~nReason: ~ts",
+ PrintError("~w:~tw failed on line ~w~nReason: ~ts",
[LastMod,LastFunc,LastLine,ErrorStr]),
case ct_util:read_suite_data({seq,Mod,Func}) of
@@ -1055,21 +1066,41 @@ group_or_func(Func, _Config) ->
%%% should be returned.
get_suite(Mod, all) ->
- case catch apply(Mod, groups, []) of
- {'EXIT',_} ->
- get_all(Mod, []);
- GroupDefs when is_list(GroupDefs) ->
- case catch ct_groups:find_groups(Mod, all, all, GroupDefs) of
- {error,_} = Error ->
- %% this makes test_server call error_in_suite as first
- %% (and only) test case so we can report Error properly
- [{?MODULE,error_in_suite,[[Error]]}];
- ConfTests ->
- get_all(Mod, ConfTests)
- end;
- _ ->
+ case safe_apply_groups_0(Mod,{ok,[]}) of
+ {ok,GroupDefs} ->
+ try ct_groups:find_groups(Mod, all, all, GroupDefs) of
+ ConfTests when is_list(ConfTests) ->
+ get_all(Mod, ConfTests)
+ catch
+ throw:{error,Error} ->
+ [{?MODULE,error_in_suite,[[{error,Error}]]}];
+ _:Error ->
+ S = erlang:get_stacktrace(),
+ [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}]
+ end;
+ {error,{bad_return,_Bad}} ->
E = "Bad return value from "++atom_to_list(Mod)++":groups/0",
- [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}]
+ [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}];
+ {error,{bad_hook_return,Bad}} ->
+ E = "Bad return value from post_groups/2 hook function",
+ [{?MODULE,error_in_suite,[[{error,{list_to_atom(E),Bad}}]]}];
+ {error,{failed,ExitReason}} ->
+ case ct_util:get_testdata({error_in_suite,Mod}) of
+ undefined ->
+ ErrStr = io_lib:format("~n*** ERROR *** "
+ "~w:groups/0 failed: ~p~n",
+ [Mod,ExitReason]),
+ io:format(?def_gl, ErrStr, []),
+ %% save the error info so it doesn't get printed twice
+ ct_util:set_testdata_async({{error_in_suite,Mod},
+ ExitReason});
+ _ExitReason ->
+ ct_util:delete_testdata({error_in_suite,Mod})
+ end,
+ Reason = list_to_atom(atom_to_list(Mod)++":groups/0 failed"),
+ [{?MODULE,error_in_suite,[[{error,Reason}]]}];
+ {error,What} ->
+ [{?MODULE,error_in_suite,[[{error,What}]]}]
end;
%%!============================================================
@@ -1079,54 +1110,75 @@ get_suite(Mod, all) ->
%% group
get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) ->
- Name = ?val(name, Props),
- case catch apply(Mod, groups, []) of
- {'EXIT',_} ->
- [Group];
- GroupDefs when is_list(GroupDefs) ->
- case catch ct_groups:find_groups(Mod, Name, TCs, GroupDefs) of
- {error,_} = Error ->
- %% this makes test_server call error_in_suite as first
- %% (and only) test case so we can report Error properly
- [{?MODULE,error_in_suite,[[Error]]}];
- [] ->
- [];
- ConfTests ->
- case lists:member(skipped, Props) of
- true ->
- %% a *subgroup* specified *only* as skipped (and not
- %% as an explicit test) should not be returned, or
- %% init/end functions for top groups will be executed
- case catch ?val(name, element(2, hd(ConfTests))) of
- Name -> % top group
- ct_groups:delete_subs(ConfTests, ConfTests);
- _ ->
- []
- end;
- false ->
- ConfTests1 = ct_groups:delete_subs(ConfTests,
- ConfTests),
- case ?val(override, Props) of
- undefined ->
- ConfTests1;
- [] ->
- ConfTests1;
- ORSpec ->
- ORSpec1 = if is_tuple(ORSpec) -> [ORSpec];
- true -> ORSpec end,
- ct_groups:search_and_override(ConfTests1,
- ORSpec1, Mod)
- end
- end
- end;
- _ ->
+ case safe_apply_groups_0(Mod,{ok,[Group]}) of
+ {ok,GroupDefs} ->
+ Name = ?val(name, Props),
+ try ct_groups:find_groups(Mod, Name, TCs, GroupDefs) of
+ [] ->
+ [];
+ ConfTests when is_list(ConfTests) ->
+ case lists:member(skipped, Props) of
+ true ->
+ %% a *subgroup* specified *only* as skipped (and not
+ %% as an explicit test) should not be returned, or
+ %% init/end functions for top groups will be executed
+ try ?val(name, element(2, hd(ConfTests))) of
+ Name -> % top group
+ ct_groups:delete_subs(ConfTests, ConfTests);
+ _ -> []
+ catch
+ _:_ -> []
+ end;
+ false ->
+ ConfTests1 = ct_groups:delete_subs(ConfTests,
+ ConfTests),
+ case ?val(override, Props) of
+ undefined ->
+ ConfTests1;
+ [] ->
+ ConfTests1;
+ ORSpec ->
+ ORSpec1 = if is_tuple(ORSpec) -> [ORSpec];
+ true -> ORSpec end,
+ ct_groups:search_and_override(ConfTests1,
+ ORSpec1, Mod)
+ end
+ end
+ catch
+ throw:{error,Error} ->
+ [{?MODULE,error_in_suite,[[{error,Error}]]}];
+ _:Error ->
+ S = erlang:get_stacktrace(),
+ [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}]
+ end;
+ {error,{bad_return,_Bad}} ->
E = "Bad return value from "++atom_to_list(Mod)++":groups/0",
- [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}]
+ [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}];
+ {error,{bad_hook_return,Bad}} ->
+ E = "Bad return value from post_groups/2 hook function",
+ [{?MODULE,error_in_suite,[[{error,{list_to_atom(E),Bad}}]]}];
+ {error,{failed,ExitReason}} ->
+ case ct_util:get_testdata({error_in_suite,Mod}) of
+ undefined ->
+ ErrStr = io_lib:format("~n*** ERROR *** "
+ "~w:groups/0 failed: ~p~n",
+ [Mod,ExitReason]),
+ io:format(?def_gl, ErrStr, []),
+ %% save the error info so it doesn't get printed twice
+ ct_util:set_testdata_async({{error_in_suite,Mod},
+ ExitReason});
+ _ExitReason ->
+ ct_util:delete_testdata({error_in_suite,Mod})
+ end,
+ Reason = list_to_atom(atom_to_list(Mod)++":groups/0 failed"),
+ [{?MODULE,error_in_suite,[[{error,Reason}]]}];
+ {error,What} ->
+ [{?MODULE,error_in_suite,[[{error,What}]]}]
end;
%% testcase
get_suite(Mod, Name) ->
- get_seq(Mod, Name).
+ get_seq(Mod, Name).
%%%-----------------------------------------------------------------
@@ -1160,25 +1212,53 @@ get_all_cases1(_, []) ->
%%%-----------------------------------------------------------------
-get_all(Mod, ConfTests) ->
- case catch apply(Mod, all, []) of
- {'EXIT',{undef,[{Mod,all,[],_} | _]}} ->
+get_all(Mod, ConfTests) ->
+ case safe_apply_all_0(Mod) of
+ {ok,AllTCs} ->
+ %% expand group references using ConfTests
+ try ct_groups:expand_groups(AllTCs, ConfTests, Mod) of
+ {error,_} = Error ->
+ [{?MODULE,error_in_suite,[[Error]]}];
+ Tests0 ->
+ Tests = ct_groups:delete_subs(Tests0, Tests0),
+ expand_tests(Mod, Tests)
+ catch
+ throw:{error,Error} ->
+ [{?MODULE,error_in_suite,[[{error,Error}]]}];
+ _:Error ->
+ S = erlang:get_stacktrace(),
+ [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}]
+ end;
+ Skip = {skip,_Reason} ->
+ Skip;
+ {error,undef} ->
+ Reason =
+ case code:which(Mod) of
+ non_existing ->
+ list_to_atom(
+ atom_to_list(Mod)++
+ " cannot be compiled or loaded");
+ _ ->
+ list_to_atom(
+ atom_to_list(Mod)++":all/0 is missing")
+ end,
+ %% this makes test_server call error_in_suite as first
+ %% (and only) test case so we can report Reason properly
+ [{?MODULE,error_in_suite,[[{error,Reason}]]}];
+ {error,{bad_return,_Bad}} ->
Reason =
- case code:which(Mod) of
- non_existing ->
- list_to_atom(atom_to_list(Mod)++
- " can not be compiled or loaded");
- _ ->
- list_to_atom(atom_to_list(Mod)++":all/0 is missing")
- end,
- %% this makes test_server call error_in_suite as first
- %% (and only) test case so we can report Reason properly
+ list_to_atom("Bad return value from "++
+ atom_to_list(Mod)++":all/0"),
[{?MODULE,error_in_suite,[[{error,Reason}]]}];
- {'EXIT',ExitReason} ->
+ {error,{bad_hook_return,Bad}} ->
+ Reason =
+ list_to_atom("Bad return value from post_all/3 hook function"),
+ [{?MODULE,error_in_suite,[[{error,{Reason,Bad}}]]}];
+ {error,{failed,ExitReason}} ->
case ct_util:get_testdata({error_in_suite,Mod}) of
undefined ->
ErrStr = io_lib:format("~n*** ERROR *** "
- "~w:all/0 failed: ~p~n",
+ "~w:all/0 failed: ~tp~n",
[Mod,ExitReason]),
io:format(?def_gl, ErrStr, []),
%% save the error info so it doesn't get printed twice
@@ -1191,28 +1271,8 @@ get_all(Mod, ConfTests) ->
%% this makes test_server call error_in_suite as first
%% (and only) test case so we can report Reason properly
[{?MODULE,error_in_suite,[[{error,Reason}]]}];
- AllTCs when is_list(AllTCs) ->
- case catch save_seqs(Mod,AllTCs) of
- {error,What} ->
- [{?MODULE,error_in_suite,[[{error,What}]]}];
- SeqsAndTCs ->
- %% expand group references in all() using ConfTests
- case catch ct_groups:expand_groups(SeqsAndTCs,
- ConfTests,
- Mod) of
- {error,_} = Error ->
- [{?MODULE,error_in_suite,[[Error]]}];
- Tests ->
- ct_groups:delete_subs(Tests, Tests)
- end
- end;
- Skip = {skip,_Reason} ->
- Skip;
- _ ->
- Reason =
- list_to_atom("Bad return value from "++
- atom_to_list(Mod)++":all/0"),
- [{?MODULE,error_in_suite,[[{error,Reason}]]}]
+ {error,What} ->
+ [{?MODULE,error_in_suite,[[{error,What}]]}]
end.
%%!============================================================
@@ -1294,8 +1354,8 @@ save_seq(Mod,Seq,SeqTCs,All) ->
check_private(Seq,TCs,All) ->
Bad = lists:filter(fun(TC) -> lists:member(TC,All) end, TCs),
if Bad /= [] ->
- Reason = io_lib:format("regular test cases not allowed in sequence ~p: "
- "~p",[Seq,Bad]),
+ Reason = io_lib:format("regular test cases not allowed in sequence ~tp: "
+ "~tp",[Seq,Bad]),
throw({error,list_to_atom(lists:flatten(Reason))});
true ->
ok
@@ -1312,7 +1372,7 @@ check_multiple(Mod,Seq,TCs) ->
end,TCs),
if Bad /= [] ->
Reason = io_lib:format("test cases found in multiple sequences: "
- "~p",[Bad]),
+ "~tp",[Bad]),
throw({error,list_to_atom(lists:flatten(Reason))});
true ->
ok
@@ -1340,15 +1400,15 @@ end_per_suite(_Config) ->
%% if the group config functions are missing in the suite,
%% use these instead
init_per_group(GroupName, Config) ->
- ct:comment(io_lib:format("start of ~p", [GroupName])),
- ct_logs:log("TEST INFO", "init_per_group/2 for ~w missing "
+ ct:comment(io_lib:format("start of ~tp", [GroupName])),
+ ct_logs:log("TEST INFO", "init_per_group/2 for ~tw missing "
"in suite, using default.",
[GroupName]),
Config.
end_per_group(GroupName, _) ->
- ct:comment(io_lib:format("end of ~p", [GroupName])),
- ct_logs:log("TEST INFO", "end_per_group/2 for ~w missing "
+ ct:comment(io_lib:format("end of ~tp", [GroupName])),
+ ct_logs:log("TEST INFO", "end_per_group/2 for ~tw missing "
"in suite, using default.",
[GroupName]),
ok.
@@ -1570,3 +1630,75 @@ get_html_wrapper(TestName, PrintLabel, Cwd, TableCols, Encoding) ->
%%% @spec get_log_dir() -> {ok,LogDir}
get_log_dir() ->
ct_logs:get_log_dir(true).
+
+%%%-----------------------------------------------------------------
+%%% Call all and group callbacks and post_* hooks with error handling
+safe_apply_all_0(Mod) ->
+ try apply(Mod, all, []) of
+ AllTCs0 when is_list(AllTCs0) ->
+ try save_seqs(Mod,AllTCs0) of
+ SeqsAndTCs when is_list(SeqsAndTCs) ->
+ all_hook(Mod,SeqsAndTCs)
+ catch throw:{error,What} ->
+ {error,What}
+ end;
+ {skip,_}=Skip ->
+ all_hook(Mod,Skip);
+ Bad ->
+ {error,{bad_return,Bad}}
+ catch
+ _:Reason ->
+ handle_callback_crash(Reason,erlang:get_stacktrace(),Mod,all,{error,undef})
+ end.
+
+all_hook(Mod, All) ->
+ case ct_hooks:all(Mod, All) of
+ AllTCs when is_list(AllTCs) ->
+ {ok,AllTCs};
+ {skip,_}=Skip ->
+ Skip;
+ {fail,Reason} ->
+ {error,Reason};
+ Bad ->
+ {error,{bad_hook_return,Bad}}
+ end.
+
+safe_apply_groups_0(Mod,Default) ->
+ try apply(Mod, groups, []) of
+ GroupDefs when is_list(GroupDefs) ->
+ case ct_hooks:groups(Mod, GroupDefs) of
+ GroupDefs1 when is_list(GroupDefs1) ->
+ {ok,GroupDefs1};
+ {fail,Reason} ->
+ {error,Reason};
+ Bad ->
+ {error,{bad_hook_return,Bad}}
+ end;
+ Bad ->
+ {error,{bad_return,Bad}}
+ catch
+ _:Reason ->
+ handle_callback_crash(Reason,erlang:get_stacktrace(),
+ Mod,groups,Default)
+ end.
+
+handle_callback_crash(undef,[{Mod,Func,[],_}|_],Mod,Func,Default) ->
+ case ct_hooks:Func(Mod, []) of
+ [] ->
+ Default;
+ List when is_list(List) ->
+ {ok,List};
+ {fail,Reason} ->
+ {error,Reason};
+ Bad ->
+ {error,{bad_hook_return,Bad}}
+ end;
+handle_callback_crash(Reason,Stacktrace,_Mod,_Func,_Default) ->
+ {error,{failed,{Reason,Stacktrace}}}.
+
+expand_tests(Mod, [{testcase,Case,[Prop]}|Tests]) ->
+ [{repeat,{Mod,Case},Prop}|expand_tests(Mod,Tests)];
+expand_tests(Mod,[Test|Tests]) ->
+ [Test|expand_tests(Mod,Tests)];
+expand_tests(_Mod,[]) ->
+ [].
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-14 17:37:15 +0000