aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test/src/ct_framework.erl
diff options
context:
space:
mode:
Diffstat (limited to 'lib/common_test/src/ct_framework.erl')
-rw-r--r--lib/common_test/src/ct_framework.erl346
1 files changed, 237 insertions, 109 deletions
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index 134ae0e1cc..bce6420042 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -314,8 +314,8 @@ add_defaults(Mod,Func, GroupPath) ->
ErrStr = io_lib:format("~n*** ERROR *** "
"~w:suite/0 failed: ~tp~n",
[Suite,Reason]),
- io:format(ErrStr, []),
- io:format(?def_gl, ErrStr, []),
+ io:format("~ts", [ErrStr]),
+ io:format(?def_gl, "~ts", [ErrStr]),
{suite0_failed,{exited,Reason}};
SuiteInfo when is_list(SuiteInfo) ->
case lists:all(fun(E) when is_tuple(E) -> true;
@@ -337,16 +337,16 @@ add_defaults(Mod,Func, GroupPath) ->
"Invalid return value from "
"~w:suite/0: ~tp~n",
[Suite,SuiteInfo]),
- io:format(ErrStr, []),
- io:format(?def_gl, ErrStr, []),
+ io:format("~ts", [ErrStr]),
+ io:format(?def_gl, "~ts", [ErrStr]),
{suite0_failed,bad_return_value}
end;
SuiteInfo ->
ErrStr = io_lib:format("~n*** ERROR *** "
"Invalid return value from "
"~w:suite/0: ~tp~n", [Suite,SuiteInfo]),
- io:format(ErrStr, []),
- io:format(?def_gl, ErrStr, []),
+ io:format("~ts", [ErrStr]),
+ io:format(?def_gl, "~ts", [ErrStr]),
{suite0_failed,bad_return_value}
end.
@@ -373,8 +373,8 @@ add_defaults1(Mod,Func, GroupPath, SuiteInfo) ->
"Invalid return value from "
"~w:group(~tw): ~tp~n",
[Mod,GrName,BadGr0Val]),
- io:format(Gr0ErrStr, []),
- io:format(?def_gl, Gr0ErrStr, []),
+ io:format("~ts", [Gr0ErrStr]),
+ io:format(?def_gl, "~ts", [Gr0ErrStr]),
{group0_failed,bad_return_value};
_ ->
Args = if Func == init_per_group ; Func == end_per_group ->
@@ -395,8 +395,8 @@ add_defaults1(Mod,Func, GroupPath, SuiteInfo) ->
"Invalid return value from "
"~w:~tw/0: ~tp~n",
[Mod,Func,BadTC0Val]),
- io:format(TC0ErrStr, []),
- io:format(?def_gl, TC0ErrStr, []),
+ io:format("~ts", [TC0ErrStr]),
+ io:format(?def_gl, "~ts", [TC0ErrStr]),
{testcase0_failed,bad_return_value};
_ ->
%% let test case info (also for all config funcs) override
@@ -696,9 +696,16 @@ end_tc(Mod,IPTC={init_per_testcase,_Func},_TCPid,Result,Args,Return) ->
end
end;
-end_tc(Mod,Func0,TCPid,Result,Args,Return) ->
+end_tc(Mod,Func00,TCPid,Result,Args,Return) ->
%% in case Mod == ct_framework, lookup the suite name
Suite = get_suite_name(Mod, Args),
+ {OnlyCleanup,Func0} =
+ case Func00 of
+ {cleanup,F0} ->
+ {true,F0};
+ _ ->
+ {false,Func00}
+ end,
{Func,FuncSpec,HookFunc} =
case Func0 of
{end_per_testcase_not_run,F} ->
@@ -742,6 +749,8 @@ end_tc(Mod,Func0,TCPid,Result,Args,Return) ->
case HookFunc of
undefined ->
{ok,Result};
+ _ when OnlyCleanup ->
+ {ok,Result};
_ ->
case ct_hooks:end_tc(Suite,HookFunc,Args,Result,Return) of
'$ct_no_change' ->
@@ -752,6 +761,8 @@ end_tc(Mod,Func0,TCPid,Result,Args,Return) ->
end,
FinalResult =
case get('$test_server_framework_test') of
+ _ when OnlyCleanup ->
+ Result1;
undefined ->
%% send sync notification so that event handlers may print
%% in the log file before it gets closed
@@ -972,11 +983,10 @@ error_notification(Mod,Func,_Args,{Error,Loc}) ->
end,
PrintError = fun(ErrorFormat, ErrorArgs) ->
- Div = "~n- - - - - - - - - - - - - - - - - - - "
- "- - - - - - - - - - - - - - - - - - - - -~n",
+ Div = "\n- - - - - - - - - - - - - - - - - - - "
+ "- - - - - - - - - - - - - - - - - - - - -\n",
ErrorStr2 = io_lib:format(ErrorFormat, ErrorArgs),
- io:format(?def_gl, lists:concat([Div,ErrorStr2,Div,"~n"]),
- []),
+ io:format(?def_gl, "~ts~n", [lists:concat([Div,ErrorStr2,Div])]),
Link =
"\n\n<a href=\"#end\">"
"Full error description and stacktrace"
@@ -985,7 +995,8 @@ error_notification(Mod,Func,_Args,{Error,Loc}) ->
ct_logs:tc_log(ct_error_notify,
?MAX_IMPORTANCE,
"CT Error Notification",
- ErrorHtml2++Link, [], [])
+ "~ts", [ErrorHtml2++Link],
+ [])
end,
case Loc of
[{?MODULE,error_in_suite}] ->
@@ -1056,21 +1067,40 @@ group_or_func(Func, _Config) ->
%%% should be returned.
get_suite(Mod, all) ->
- case catch apply(Mod, groups, []) of
- {'EXIT',_} ->
- get_all(Mod, []);
- GroupDefs when is_list(GroupDefs) ->
- case catch ct_groups:find_groups(Mod, all, all, GroupDefs) of
- {error,_} = Error ->
- %% this makes test_server call error_in_suite as first
- %% (and only) test case so we can report Error properly
- [{?MODULE,error_in_suite,[[Error]]}];
- ConfTests ->
- get_all(Mod, ConfTests)
- end;
- _ ->
+ case safe_apply_groups_0(Mod,{ok,[]}) of
+ {ok,GroupDefs} ->
+ try ct_groups:find_groups(Mod, all, all, GroupDefs) of
+ ConfTests when is_list(ConfTests) ->
+ get_all(Mod, ConfTests)
+ catch
+ throw:{error,Error} ->
+ [{?MODULE,error_in_suite,[[{error,Error}]]}];
+ _:Error:S ->
+ [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}]
+ end;
+ {error,{bad_return,_Bad}} ->
E = "Bad return value from "++atom_to_list(Mod)++":groups/0",
- [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}]
+ [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}];
+ {error,{bad_hook_return,Bad}} ->
+ E = "Bad return value from post_groups/2 hook function",
+ [{?MODULE,error_in_suite,[[{error,{list_to_atom(E),Bad}}]]}];
+ {error,{failed,ExitReason}} ->
+ case ct_util:get_testdata({error_in_suite,Mod}) of
+ undefined ->
+ ErrStr = io_lib:format("~n*** ERROR *** "
+ "~w:groups/0 failed: ~p~n",
+ [Mod,ExitReason]),
+ io:format(?def_gl, ErrStr, []),
+ %% save the error info so it doesn't get printed twice
+ ct_util:set_testdata_async({{error_in_suite,Mod},
+ ExitReason});
+ _ExitReason ->
+ ct_util:delete_testdata({error_in_suite,Mod})
+ end,
+ Reason = list_to_atom(atom_to_list(Mod)++":groups/0 failed"),
+ [{?MODULE,error_in_suite,[[{error,Reason}]]}];
+ {error,What} ->
+ [{?MODULE,error_in_suite,[[{error,What}]]}]
end;
%%!============================================================
@@ -1080,54 +1110,74 @@ get_suite(Mod, all) ->
%% group
get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) ->
- Name = ?val(name, Props),
- case catch apply(Mod, groups, []) of
- {'EXIT',_} ->
- [Group];
- GroupDefs when is_list(GroupDefs) ->
- case catch ct_groups:find_groups(Mod, Name, TCs, GroupDefs) of
- {error,_} = Error ->
- %% this makes test_server call error_in_suite as first
- %% (and only) test case so we can report Error properly
- [{?MODULE,error_in_suite,[[Error]]}];
- [] ->
- [];
- ConfTests ->
- case lists:member(skipped, Props) of
- true ->
- %% a *subgroup* specified *only* as skipped (and not
- %% as an explicit test) should not be returned, or
- %% init/end functions for top groups will be executed
- case catch ?val(name, element(2, hd(ConfTests))) of
- Name -> % top group
- ct_groups:delete_subs(ConfTests, ConfTests);
- _ ->
- []
- end;
- false ->
- ConfTests1 = ct_groups:delete_subs(ConfTests,
- ConfTests),
- case ?val(override, Props) of
- undefined ->
- ConfTests1;
- [] ->
- ConfTests1;
- ORSpec ->
- ORSpec1 = if is_tuple(ORSpec) -> [ORSpec];
- true -> ORSpec end,
- ct_groups:search_and_override(ConfTests1,
- ORSpec1, Mod)
- end
- end
- end;
- _ ->
+ case safe_apply_groups_0(Mod,{ok,[Group]}) of
+ {ok,GroupDefs} ->
+ Name = ?val(name, Props),
+ try ct_groups:find_groups(Mod, Name, TCs, GroupDefs) of
+ [] ->
+ [];
+ ConfTests when is_list(ConfTests) ->
+ case lists:member(skipped, Props) of
+ true ->
+ %% a *subgroup* specified *only* as skipped (and not
+ %% as an explicit test) should not be returned, or
+ %% init/end functions for top groups will be executed
+ try ?val(name, element(2, hd(ConfTests))) of
+ Name -> % top group
+ ct_groups:delete_subs(ConfTests, ConfTests);
+ _ -> []
+ catch
+ _:_ -> []
+ end;
+ false ->
+ ConfTests1 = ct_groups:delete_subs(ConfTests,
+ ConfTests),
+ case ?val(override, Props) of
+ undefined ->
+ ConfTests1;
+ [] ->
+ ConfTests1;
+ ORSpec ->
+ ORSpec1 = if is_tuple(ORSpec) -> [ORSpec];
+ true -> ORSpec end,
+ ct_groups:search_and_override(ConfTests1,
+ ORSpec1, Mod)
+ end
+ end
+ catch
+ throw:{error,Error} ->
+ [{?MODULE,error_in_suite,[[{error,Error}]]}];
+ _:Error:S ->
+ [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}]
+ end;
+ {error,{bad_return,_Bad}} ->
E = "Bad return value from "++atom_to_list(Mod)++":groups/0",
- [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}]
+ [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}];
+ {error,{bad_hook_return,Bad}} ->
+ E = "Bad return value from post_groups/2 hook function",
+ [{?MODULE,error_in_suite,[[{error,{list_to_atom(E),Bad}}]]}];
+ {error,{failed,ExitReason}} ->
+ case ct_util:get_testdata({error_in_suite,Mod}) of
+ undefined ->
+ ErrStr = io_lib:format("~n*** ERROR *** "
+ "~w:groups/0 failed: ~p~n",
+ [Mod,ExitReason]),
+ io:format(?def_gl, ErrStr, []),
+ %% save the error info so it doesn't get printed twice
+ ct_util:set_testdata_async({{error_in_suite,Mod},
+ ExitReason});
+ _ExitReason ->
+ ct_util:delete_testdata({error_in_suite,Mod})
+ end,
+ Reason = list_to_atom(atom_to_list(Mod)++":groups/0 failed"),
+ [{?MODULE,error_in_suite,[[{error,Reason}]]}];
+ {error,What} ->
+ [{?MODULE,error_in_suite,[[{error,What}]]}]
end;
%% testcase
get_suite(Mod, Name) ->
- get_seq(Mod, Name).
+ get_seq(Mod, Name).
%%%-----------------------------------------------------------------
@@ -1161,27 +1211,54 @@ get_all_cases1(_, []) ->
%%%-----------------------------------------------------------------
-get_all(Mod, ConfTests) ->
- case catch apply(Mod, all, []) of
- {'EXIT',{undef,[{Mod,all,[],_} | _]}} ->
+get_all(Mod, ConfTests) ->
+ case safe_apply_all_0(Mod) of
+ {ok,AllTCs} ->
+ %% expand group references using ConfTests
+ try ct_groups:expand_groups(AllTCs, ConfTests, Mod) of
+ {error,_} = Error ->
+ [{?MODULE,error_in_suite,[[Error]]}];
+ Tests0 ->
+ Tests = ct_groups:delete_subs(Tests0, Tests0),
+ expand_tests(Mod, Tests)
+ catch
+ throw:{error,Error} ->
+ [{?MODULE,error_in_suite,[[{error,Error}]]}];
+ _:Error:S ->
+ [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}]
+ end;
+ Skip = {skip,_Reason} ->
+ Skip;
+ {error,undef} ->
+ Reason =
+ case code:which(Mod) of
+ non_existing ->
+ list_to_atom(
+ atom_to_list(Mod)++
+ " cannot be compiled or loaded");
+ _ ->
+ list_to_atom(
+ atom_to_list(Mod)++":all/0 is missing")
+ end,
+ %% this makes test_server call error_in_suite as first
+ %% (and only) test case so we can report Reason properly
+ [{?MODULE,error_in_suite,[[{error,Reason}]]}];
+ {error,{bad_return,_Bad}} ->
Reason =
- case code:which(Mod) of
- non_existing ->
- list_to_atom(atom_to_list(Mod)++
- " can not be compiled or loaded");
- _ ->
- list_to_atom(atom_to_list(Mod)++":all/0 is missing")
- end,
- %% this makes test_server call error_in_suite as first
- %% (and only) test case so we can report Reason properly
+ list_to_atom("Bad return value from "++
+ atom_to_list(Mod)++":all/0"),
[{?MODULE,error_in_suite,[[{error,Reason}]]}];
- {'EXIT',ExitReason} ->
+ {error,{bad_hook_return,Bad}} ->
+ Reason =
+ list_to_atom("Bad return value from post_all/3 hook function"),
+ [{?MODULE,error_in_suite,[[{error,{Reason,Bad}}]]}];
+ {error,{failed,ExitReason}} ->
case ct_util:get_testdata({error_in_suite,Mod}) of
undefined ->
ErrStr = io_lib:format("~n*** ERROR *** "
"~w:all/0 failed: ~tp~n",
[Mod,ExitReason]),
- io:format(?def_gl, ErrStr, []),
+ io:format(?def_gl, "~ts", [ErrStr]),
%% save the error info so it doesn't get printed twice
ct_util:set_testdata_async({{error_in_suite,Mod},
ExitReason});
@@ -1192,28 +1269,8 @@ get_all(Mod, ConfTests) ->
%% this makes test_server call error_in_suite as first
%% (and only) test case so we can report Reason properly
[{?MODULE,error_in_suite,[[{error,Reason}]]}];
- AllTCs when is_list(AllTCs) ->
- case catch save_seqs(Mod,AllTCs) of
- {error,What} ->
- [{?MODULE,error_in_suite,[[{error,What}]]}];
- SeqsAndTCs ->
- %% expand group references in all() using ConfTests
- case catch ct_groups:expand_groups(SeqsAndTCs,
- ConfTests,
- Mod) of
- {error,_} = Error ->
- [{?MODULE,error_in_suite,[[Error]]}];
- Tests ->
- ct_groups:delete_subs(Tests, Tests)
- end
- end;
- Skip = {skip,_Reason} ->
- Skip;
- _ ->
- Reason =
- list_to_atom("Bad return value from "++
- atom_to_list(Mod)++":all/0"),
- [{?MODULE,error_in_suite,[[{error,Reason}]]}]
+ {error,What} ->
+ [{?MODULE,error_in_suite,[[{error,What}]]}]
end.
%%!============================================================
@@ -1571,3 +1628,74 @@ get_html_wrapper(TestName, PrintLabel, Cwd, TableCols, Encoding) ->
%%% -spec get_log_dir() -> {ok,LogDir}
get_log_dir() ->
ct_logs:get_log_dir(true).
+
+%%%-----------------------------------------------------------------
+%%% Call all and group callbacks and post_* hooks with error handling
+safe_apply_all_0(Mod) ->
+ try apply(Mod, all, []) of
+ AllTCs0 when is_list(AllTCs0) ->
+ try save_seqs(Mod,AllTCs0) of
+ SeqsAndTCs when is_list(SeqsAndTCs) ->
+ all_hook(Mod,SeqsAndTCs)
+ catch throw:{error,What} ->
+ {error,What}
+ end;
+ {skip,_}=Skip ->
+ all_hook(Mod,Skip);
+ Bad ->
+ {error,{bad_return,Bad}}
+ catch
+ _:Reason:Stacktrace ->
+ handle_callback_crash(Reason,Stacktrace,Mod,all,{error,undef})
+ end.
+
+all_hook(Mod, All) ->
+ case ct_hooks:all(Mod, All) of
+ AllTCs when is_list(AllTCs) ->
+ {ok,AllTCs};
+ {skip,_}=Skip ->
+ Skip;
+ {fail,Reason} ->
+ {error,Reason};
+ Bad ->
+ {error,{bad_hook_return,Bad}}
+ end.
+
+safe_apply_groups_0(Mod,Default) ->
+ try apply(Mod, groups, []) of
+ GroupDefs when is_list(GroupDefs) ->
+ case ct_hooks:groups(Mod, GroupDefs) of
+ GroupDefs1 when is_list(GroupDefs1) ->
+ {ok,GroupDefs1};
+ {fail,Reason} ->
+ {error,Reason};
+ Bad ->
+ {error,{bad_hook_return,Bad}}
+ end;
+ Bad ->
+ {error,{bad_return,Bad}}
+ catch
+ _:Reason:Stacktrace ->
+ handle_callback_crash(Reason,Stacktrace,Mod,groups,Default)
+ end.
+
+handle_callback_crash(undef,[{Mod,Func,[],_}|_],Mod,Func,Default) ->
+ case ct_hooks:Func(Mod, []) of
+ [] ->
+ Default;
+ List when is_list(List) ->
+ {ok,List};
+ {fail,Reason} ->
+ {error,Reason};
+ Bad ->
+ {error,{bad_hook_return,Bad}}
+ end;
+handle_callback_crash(Reason,Stacktrace,_Mod,_Func,_Default) ->
+ {error,{failed,{Reason,Stacktrace}}}.
+
+expand_tests(Mod, [{testcase,Case,[Prop]}|Tests]) ->
+ [{repeat,{Mod,Case},Prop}|expand_tests(Mod,Tests)];
+expand_tests(Mod,[Test|Tests]) ->
+ [Test|expand_tests(Mod,Tests)];
+expand_tests(_Mod,[]) ->
+ [].
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-01 04:13:46 +0000