From acd87425acf7705328b01aa8f0a804207ffe4790 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Wed, 10 Jan 2018 16:13:52 +0100 Subject: [ct] Add post_groups/2 and post_all/3 hook functions --- lib/common_test/doc/src/ct_hooks.xml | 123 ++++++++ lib/common_test/src/ct_framework.erl | 297 ++++++++++++------ lib/common_test/src/ct_hooks.erl | 76 ++++- lib/common_test/test/ct_hooks_SUITE.erl | 345 ++++++++++++++++++++- .../cth/tests/all_and_groups_SUITE.erl | 47 +++ .../cth/tests/all_and_groups_cth.erl | 100 ++++++ .../cth/tests/ct_match_state_cth.erl | 58 ++++ .../tests/ct_scope_suite_group_only_cth_SUITE.erl | 54 ++++ .../ct_hooks_SUITE_data/cth/tests/empty_cth.erl | 28 ++ 9 files changed, 1030 insertions(+), 98 deletions(-) create mode 100644 lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_SUITE.erl create mode 100644 lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_cth.erl create mode 100644 lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_match_state_cth.erl create mode 100644 lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_group_only_cth_SUITE.erl diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml index 954be0ffba..613b694796 100644 --- a/lib/common_test/doc/src/ct_hooks.xml +++ b/lib/common_test/doc/src/ct_hooks.xml @@ -108,6 +108,129 @@ + + Module:post_groups(SuiteName, GroupDefs) -> NewGroupDefs + Called after groups/0. + + SuiteName = atom() + GroupDefs = NewGroupDefs = [Group] + Group = {GroupName,Properties,GroupsAndTestCases} + GroupName = atom() + Properties = [parallel | sequence | Shuffle | {RepeatType,N}] + GroupsAndTestCases = [Group | {group,GroupName} | TestCase] + TestCase = atom() + Shuffle = shuffle | {shuffle,Seed} + Seed = {integer(),integer(),integer()} + RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail + N = integer() | forever + + +

OPTIONAL

+ +

This function is called after + groups/0. + It is used to modify the test group definitions, for + instance to add or remove groups or change group properties.

+ +

GroupDefs is what + groups/0 + returned, that is, a list of group definitions.

+ +

NewGroupDefs is the possibly modified version of this list.

+ +

This function is called only if the CTH is added before + init_per_suite is run. For details, see section + CTH Scope + in the User's Guide.

+ +

Notice that for CTHs that are installed by means of the + suite/0 + function, post_groups/2 is called before + the init/2 + hook function. However, for CTHs that are installed by means + of the CT start flag, + the init/2 + function is called first.

+ + +

Prior to each test execution, Common Test does a + simulated test run in order to count test suites, groups + and cases for logging purposes. This causes + the post_groups/2 hook function to always be called + twice. For this reason, side effects are best avoided in + this callback.

+ + + + + + Module:post_all(SuiteName, Return, GroupDefs) -> NewReturn + Called after all/0. + + SuiteName = atom() + Return = NewReturn = Tests | {skip,Reason} + Tests = [TestCase | {group,GroupName} | {group,GroupName,Properties} | {group,GroupName,Properties,SubGroups}] + TestCase = atom() + GroupName = atom() + Properties = GroupProperties | default + SubGroups = [{GroupName,Properties} | {GroupName,Properties,SubGroups}] + Shuffle = shuffle | {shuffle,Seed} + Seed = {integer(),integer(),integer()} + RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail + N = integer() | forever + GroupDefs = NewGroupDefs = [Group] + Group = {GroupName,GroupProperties,GroupsAndTestCases} + GroupProperties = [parallel | sequence | Shuffle | {RepeatType,N}] + GroupsAndTestCases = [Group | {group,GroupName} | TestCase] + Reason = term() + + +

OPTIONAL

+ +

This function is called after + all/0. + It is used to modify the set of test cases and test group to + be executed, for instance to add or remove test cases and + groups, change group properties, or even skip all tests in + the suite.

+ +

Return is what + all/0 + returned, that is, a list of test cases and groups to be + executed, or a tuple {skip,Reason}.

+ +

GroupDefs is what + groups/0 + or the post_groups/2 hook returned, that is, a list + of group definitions.

+ +

NewReturn is the possibly modified version of Return.

+ +

This function is called only if the CTH is added before + init_per_suite is run. For details, see section + CTH Scope + in the User's Guide.

+ +

Notice that for CTHs that are installed by means of the + suite/0 + function, post_all/2 is called before + the init/2 + hook function. However, for CTHs that are installed by means + of the CT start flag, + the init/2 + function is called first.

+ + +

Prior to each test execution, Common Test does a + simulated test run in order to count test suites, groups + and cases for logging purposes. This causes + the post_all/3 hook function to always be called + twice. For this reason, side effects are best avoided in + this callback.

+ + + + Module:pre_init_per_suite(SuiteName, InitData, CTHState) -> Result Called before init_per_suite. diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 6066470233..f2d063254a 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -1055,21 +1055,41 @@ group_or_func(Func, _Config) -> %%% should be returned. get_suite(Mod, all) -> - case catch apply(Mod, groups, []) of - {'EXIT',_} -> - get_all(Mod, []); - GroupDefs when is_list(GroupDefs) -> - case catch ct_groups:find_groups(Mod, all, all, GroupDefs) of - {error,_} = Error -> - %% this makes test_server call error_in_suite as first - %% (and only) test case so we can report Error properly - [{?MODULE,error_in_suite,[[Error]]}]; - ConfTests -> - get_all(Mod, ConfTests) - end; - _ -> + case safe_apply_groups_0(Mod,{ok,[]}) of + {ok,GroupDefs} -> + try ct_groups:find_groups(Mod, all, all, GroupDefs) of + ConfTests when is_list(ConfTests) -> + get_all(Mod, ConfTests) + catch + throw:{error,Error} -> + [{?MODULE,error_in_suite,[[{error,Error}]]}]; + _:Error -> + S = erlang:get_stacktrace(), + [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}] + end; + {error,{bad_return,_Bad}} -> E = "Bad return value from "++atom_to_list(Mod)++":groups/0", - [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}] + [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}]; + {error,{bad_hook_return,Bad}} -> + E = "Bad return value from post_groups/2 hook function", + [{?MODULE,error_in_suite,[[{error,{list_to_atom(E),Bad}}]]}]; + {error,{failed,ExitReason}} -> + case ct_util:get_testdata({error_in_suite,Mod}) of + undefined -> + ErrStr = io_lib:format("~n*** ERROR *** " + "~w:groups/0 failed: ~p~n", + [Mod,ExitReason]), + io:format(?def_gl, ErrStr, []), + %% save the error info so it doesn't get printed twice + ct_util:set_testdata_async({{error_in_suite,Mod}, + ExitReason}); + _ExitReason -> + ct_util:delete_testdata({error_in_suite,Mod}) + end, + Reason = list_to_atom(atom_to_list(Mod)++":groups/0 failed"), + [{?MODULE,error_in_suite,[[{error,Reason}]]}]; + {error,What} -> + [{?MODULE,error_in_suite,[[{error,What}]]}] end; %%!============================================================ @@ -1079,54 +1099,75 @@ get_suite(Mod, all) -> %% group get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) -> - Name = ?val(name, Props), - case catch apply(Mod, groups, []) of - {'EXIT',_} -> - [Group]; - GroupDefs when is_list(GroupDefs) -> - case catch ct_groups:find_groups(Mod, Name, TCs, GroupDefs) of - {error,_} = Error -> - %% this makes test_server call error_in_suite as first - %% (and only) test case so we can report Error properly - [{?MODULE,error_in_suite,[[Error]]}]; - [] -> - []; - ConfTests -> - case lists:member(skipped, Props) of - true -> - %% a *subgroup* specified *only* as skipped (and not - %% as an explicit test) should not be returned, or - %% init/end functions for top groups will be executed - case catch ?val(name, element(2, hd(ConfTests))) of - Name -> % top group - ct_groups:delete_subs(ConfTests, ConfTests); - _ -> - [] - end; - false -> - ConfTests1 = ct_groups:delete_subs(ConfTests, - ConfTests), - case ?val(override, Props) of - undefined -> - ConfTests1; - [] -> - ConfTests1; - ORSpec -> - ORSpec1 = if is_tuple(ORSpec) -> [ORSpec]; - true -> ORSpec end, - ct_groups:search_and_override(ConfTests1, - ORSpec1, Mod) - end - end - end; - _ -> + case safe_apply_groups_0(Mod,{ok,[Group]}) of + {ok,GroupDefs} -> + Name = ?val(name, Props), + try ct_groups:find_groups(Mod, Name, TCs, GroupDefs) of + [] -> + []; + ConfTests when is_list(ConfTests) -> + case lists:member(skipped, Props) of + true -> + %% a *subgroup* specified *only* as skipped (and not + %% as an explicit test) should not be returned, or + %% init/end functions for top groups will be executed + try ?val(name, element(2, hd(ConfTests))) of + Name -> % top group + ct_groups:delete_subs(ConfTests, ConfTests); + _ -> [] + catch + _:_ -> [] + end; + false -> + ConfTests1 = ct_groups:delete_subs(ConfTests, + ConfTests), + case ?val(override, Props) of + undefined -> + ConfTests1; + [] -> + ConfTests1; + ORSpec -> + ORSpec1 = if is_tuple(ORSpec) -> [ORSpec]; + true -> ORSpec end, + ct_groups:search_and_override(ConfTests1, + ORSpec1, Mod) + end + end + catch + throw:{error,Error} -> + [{?MODULE,error_in_suite,[[{error,Error}]]}]; + _:Error -> + S = erlang:get_stacktrace(), + [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}] + end; + {error,{bad_return,_Bad}} -> E = "Bad return value from "++atom_to_list(Mod)++":groups/0", - [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}] + [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}]; + {error,{bad_hook_return,Bad}} -> + E = "Bad return value from post_groups/2 hook function", + [{?MODULE,error_in_suite,[[{error,{list_to_atom(E),Bad}}]]}]; + {error,{failed,ExitReason}} -> + case ct_util:get_testdata({error_in_suite,Mod}) of + undefined -> + ErrStr = io_lib:format("~n*** ERROR *** " + "~w:groups/0 failed: ~p~n", + [Mod,ExitReason]), + io:format(?def_gl, ErrStr, []), + %% save the error info so it doesn't get printed twice + ct_util:set_testdata_async({{error_in_suite,Mod}, + ExitReason}); + _ExitReason -> + ct_util:delete_testdata({error_in_suite,Mod}) + end, + Reason = list_to_atom(atom_to_list(Mod)++":groups/0 failed"), + [{?MODULE,error_in_suite,[[{error,Reason}]]}]; + {error,What} -> + [{?MODULE,error_in_suite,[[{error,What}]]}] end; %% testcase get_suite(Mod, Name) -> - get_seq(Mod, Name). + get_seq(Mod, Name). %%%----------------------------------------------------------------- @@ -1160,21 +1201,48 @@ get_all_cases1(_, []) -> %%%----------------------------------------------------------------- -get_all(Mod, ConfTests) -> - case catch apply(Mod, all, []) of - {'EXIT',{undef,[{Mod,all,[],_} | _]}} -> +get_all(Mod, ConfTests) -> + case safe_apply_all_0(Mod) of + {ok,AllTCs} -> + %% expand group references using ConfTests + try ct_groups:expand_groups(AllTCs, ConfTests, Mod) of + {error,_} = Error -> + [{?MODULE,error_in_suite,[[Error]]}]; + Tests -> + ct_groups:delete_subs(Tests, Tests) + catch + throw:{error,Error} -> + [{?MODULE,error_in_suite,[[{error,Error}]]}]; + _:Error -> + S = erlang:get_stacktrace(), + [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}] + end; + Skip = {skip,_Reason} -> + Skip; + {error,undef} -> + Reason = + case code:which(Mod) of + non_existing -> + list_to_atom( + atom_to_list(Mod)++ + " cannot be compiled or loaded"); + _ -> + list_to_atom( + atom_to_list(Mod)++":all/0 is missing") + end, + %% this makes test_server call error_in_suite as first + %% (and only) test case so we can report Reason properly + [{?MODULE,error_in_suite,[[{error,Reason}]]}]; + {error,{bad_return,_Bad}} -> Reason = - case code:which(Mod) of - non_existing -> - list_to_atom(atom_to_list(Mod)++ - " can not be compiled or loaded"); - _ -> - list_to_atom(atom_to_list(Mod)++":all/0 is missing") - end, - %% this makes test_server call error_in_suite as first - %% (and only) test case so we can report Reason properly + list_to_atom("Bad return value from "++ + atom_to_list(Mod)++":all/0"), [{?MODULE,error_in_suite,[[{error,Reason}]]}]; - {'EXIT',ExitReason} -> + {error,{bad_hook_return,Bad}} -> + Reason = + list_to_atom("Bad return value from post_all/3 hook function"), + [{?MODULE,error_in_suite,[[{error,{Reason,Bad}}]]}]; + {error,{failed,ExitReason}} -> case ct_util:get_testdata({error_in_suite,Mod}) of undefined -> ErrStr = io_lib:format("~n*** ERROR *** " @@ -1191,28 +1259,8 @@ get_all(Mod, ConfTests) -> %% this makes test_server call error_in_suite as first %% (and only) test case so we can report Reason properly [{?MODULE,error_in_suite,[[{error,Reason}]]}]; - AllTCs when is_list(AllTCs) -> - case catch save_seqs(Mod,AllTCs) of - {error,What} -> - [{?MODULE,error_in_suite,[[{error,What}]]}]; - SeqsAndTCs -> - %% expand group references in all() using ConfTests - case catch ct_groups:expand_groups(SeqsAndTCs, - ConfTests, - Mod) of - {error,_} = Error -> - [{?MODULE,error_in_suite,[[Error]]}]; - Tests -> - ct_groups:delete_subs(Tests, Tests) - end - end; - Skip = {skip,_Reason} -> - Skip; - _ -> - Reason = - list_to_atom("Bad return value from "++ - atom_to_list(Mod)++":all/0"), - [{?MODULE,error_in_suite,[[{error,Reason}]]}] + {error,What} -> + [{?MODULE,error_in_suite,[[{error,What}]]}] end. %%!============================================================ @@ -1570,3 +1618,68 @@ get_html_wrapper(TestName, PrintLabel, Cwd, TableCols, Encoding) -> %%% @spec get_log_dir() -> {ok,LogDir} get_log_dir() -> ct_logs:get_log_dir(true). + +%%%----------------------------------------------------------------- +%%% Call all and group callbacks and post_* hooks with error handling +safe_apply_all_0(Mod) -> + try apply(Mod, all, []) of + AllTCs0 when is_list(AllTCs0) -> + try save_seqs(Mod,AllTCs0) of + SeqsAndTCs when is_list(SeqsAndTCs) -> + all_hook(Mod,SeqsAndTCs) + catch throw:{error,What} -> + {error,What} + end; + {skip,_}=Skip -> + all_hook(Mod,Skip); + Bad -> + {error,{bad_return,Bad}} + catch + _:Reason -> + handle_callback_crash(Reason,erlang:get_stacktrace(),Mod,all,{error,undef}) + end. + +all_hook(Mod, All) -> + case ct_hooks:all(Mod, All) of + AllTCs when is_list(AllTCs) -> + {ok,AllTCs}; + {skip,_}=Skip -> + Skip; + {fail,Reason} -> + {error,Reason}; + Bad -> + {error,{bad_hook_return,Bad}} + end. + +safe_apply_groups_0(Mod,Default) -> + try apply(Mod, groups, []) of + GroupDefs when is_list(GroupDefs) -> + case ct_hooks:groups(Mod, GroupDefs) of + GroupDefs1 when is_list(GroupDefs1) -> + {ok,GroupDefs1}; + {fail,Reason} -> + {error,Reason}; + Bad -> + {error,{bad_hook_return,Bad}} + end; + Bad -> + {error,{bad_return,Bad}} + catch + _:Reason -> + handle_callback_crash(Reason,erlang:get_stacktrace(), + Mod,groups,Default) + end. + +handle_callback_crash(undef,[{Mod,Func,[],_}|_],Mod,Func,Default) -> + case ct_hooks:Func(Mod, []) of + [] -> + Default; + List when is_list(List) -> + {ok,List}; + {fail,Reason} -> + {error,Reason}; + Bad -> + {error,{bad_hook_return,Bad}} + end; +handle_callback_crash(Reason,Stacktrace,_Mod,_Func,_Default) -> + {error,{failed,{Reason,Stacktrace}}}. diff --git a/lib/common_test/src/ct_hooks.erl b/lib/common_test/src/ct_hooks.erl index f0592a40be..903710963c 100644 --- a/lib/common_test/src/ct_hooks.erl +++ b/lib/common_test/src/ct_hooks.erl @@ -26,6 +26,8 @@ %% API Exports -export([init/1]). +-export([groups/2]). +-export([all/2]). -export([init_tc/3]). -export([end_tc/5]). -export([terminate/1]). @@ -41,7 +43,8 @@ opts = [], prio = ctfirst }]). --record(ct_hook_config, {id, module, prio, scope, opts = [], state = []}). +-record(ct_hook_config, {id, module, prio, scope, opts = [], + state = [], groups = []}). %% ------------------------------------------------------------------------- %% API Functions @@ -54,6 +57,47 @@ init(Opts) -> call(get_builtin_hooks(Opts) ++ get_new_hooks(Opts, undefined), ok, init, []). +%% Call the post_groups/2 hook callback +groups(Mod, Groups) -> + Info = try proplists:get_value(ct_hooks, Mod:suite(), []) of + CTHooks when is_list(CTHooks) -> + [{?config_name,CTHooks}]; + CTHook when is_atom(CTHook) -> + [{?config_name,[CTHook]}] + catch _:_ -> + %% since this might be the first time Mod:suite() + %% is called, and it might just fail or return + %% something bad, we allow any failure here - it + %% will be catched later if there is something + %% really wrong. + [{?config_name,[]}] + end, + case call(fun call_generic/3, Info ++ [{'$ct_groups',Groups}], [post_groups, Mod]) of + [{'$ct_groups',NewGroups}] -> + NewGroups; + Other -> + Other + end. + +%% Call the post_all/3 hook callback +all(Mod, Tests) -> + Info = try proplists:get_value(ct_hooks, Mod:suite(), []) of + CTHooks when is_list(CTHooks) -> + [{?config_name,CTHooks}]; + CTHook when is_atom(CTHook) -> + [{?config_name,[CTHook]}] + catch _:_ -> + %% just allow any failure here - it will be catched + %% later if there is something really wrong. + [{?config_name,[]}] + end, + case call(fun call_generic/3, Info ++ [{'$ct_all',Tests}], [post_all, Mod]) of + [{'$ct_all',NewTests}] -> + NewTests; + Other -> + Other + end. + %% @doc Called after all suites are done. -spec terminate(Hooks :: term()) -> ok. @@ -88,6 +132,7 @@ init_tc(Mod, init_per_suite, Config) -> [{?config_name,[]}] end, call(fun call_generic/3, Config ++ Info, [pre_init_per_suite, Mod]); + init_tc(Mod, end_per_suite, Config) -> call(fun call_generic/3, Config, [pre_end_per_suite, Mod]); init_tc(Mod, {init_per_group, GroupName, Properties}, Config) -> @@ -163,7 +208,7 @@ call_id(#ct_hook_config{ module = Mod, opts = Opts} = Hook, Config, Scope) -> {Config, Hook#ct_hook_config{ id = Id, scope = scope(Scope)}}. call_init(#ct_hook_config{ module = Mod, opts = Opts, id = Id, prio = P} = Hook, - Config,_Meta) -> + Config, _Meta) -> case Mod:init(Id, Opts) of {ok, NewState} when P =:= undefined -> {Config, Hook#ct_hook_config{ state = NewState, prio = 0 } }; @@ -194,6 +239,18 @@ call_generic(Hook, Value, Meta) -> call_generic_fallback(Hook, Value, Meta) -> do_call_generic(Hook, Value, Meta, true). +do_call_generic(#ct_hook_config{ module = Mod} = Hook, + [{'$ct_groups',Groups}], [post_groups | Args], Fallback) -> + NewGroups = catch_apply(Mod, post_groups, Args ++ [Groups], + Groups, Fallback), + {[{'$ct_groups',NewGroups}], Hook#ct_hook_config{ groups = NewGroups } }; + +do_call_generic(#ct_hook_config{ module = Mod, groups = Groups} = Hook, + [{'$ct_all',Tests}], [post_all | Args], Fallback) -> + NewTests = catch_apply(Mod, post_all, Args ++ [Tests, Groups], + Tests, Fallback), + {[{'$ct_all',NewTests}], Hook}; + do_call_generic(#ct_hook_config{ module = Mod, state = State} = Hook, Value, [Function | Args], Fallback) -> {NewValue, NewState} = catch_apply(Mod, Function, Args ++ [Value, State], @@ -228,6 +285,12 @@ call([{Hook, call_id, NextFun} | Rest], Config, Meta, Hooks) -> Rest ++ [{NewId, call_init}]}; ExistingHook when is_tuple(ExistingHook) -> {Hooks, Rest}; + _ when hd(Meta)=:=post_groups; hd(Meta)=:=post_all -> + %% If CTH is started because of a call from + %% groups/2 or all/2, CTH:init/1 must not be + %% called (the suite scope should be used). + {Hooks ++ [NewHook], + Rest ++ [{NewId,NextFun}]}; _ -> {Hooks ++ [NewHook], Rest ++ [{NewId, call_init}, {NewId,NextFun}]} @@ -237,8 +300,8 @@ call([{Hook, call_id, NextFun} | Rest], Config, Meta, Hooks) -> Trace = erlang:get_stacktrace(), ct_logs:log("Suite Hook","Failed to start a CTH: ~tp:~tp", [Error,{Reason,Trace}]), - call([], {fail,"Failed to start CTH" - ", see the CT Log for details"}, Meta, Hooks) + call([], {fail,"Failed to start CTH, " + "see the CT Log for details"}, Meta, Hooks) end; call([{HookId, call_init} | Rest], Config, Meta, Hooks) -> call([{HookId, fun call_init/3} | Rest], Config, Meta, Hooks); @@ -278,6 +341,10 @@ scope([pre_init_per_suite, SuiteName|_]) -> [post_end_per_suite, SuiteName]; scope([post_init_per_suite, SuiteName|_]) -> [post_end_per_suite, SuiteName]; +scope([post_groups, SuiteName|_]) -> + [post_groups, SuiteName]; +scope([post_all, SuiteName|_]) -> + [post_all, SuiteName]; scope(init) -> none. @@ -364,6 +431,7 @@ resort(Calls,Hooks,[F|_R]) when F == pre_end_per_testcase; F == pre_end_per_suite; F == post_end_per_suite -> lists:reverse(resort(Calls,Hooks)); + resort(Calls,Hooks,_Meta) -> resort(Calls,Hooks). diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl index 7900fcd5bc..edcfec4fb5 100644 --- a/lib/common_test/test/ct_hooks_SUITE.erl +++ b/lib/common_test/test/ct_hooks_SUITE.erl @@ -73,11 +73,15 @@ all() -> all(suite) -> lists:reverse( [ + crash_groups, crash_all, bad_return_groups, bad_return_all, + illegal_values_groups, illegal_values_all, alter_groups, alter_all, + alter_all_to_skip, alter_all_from_skip, one_cth, two_cth, faulty_cth_no_init, faulty_cth_id_no_init, faulty_cth_exit_in_init, faulty_cth_exit_in_id, faulty_cth_exit_in_init_scope_suite, minimal_cth, minimal_and_maximal_cth, faulty_cth_undef, scope_per_suite_cth, scope_per_group_cth, scope_suite_cth, + scope_suite_group_only_cth, scope_per_suite_state_cth, scope_per_group_state_cth, scope_suite_state_cth, fail_pre_suite_cth, double_fail_pre_suite_cth, @@ -152,6 +156,11 @@ scope_suite_cth(Config) when is_list(Config) -> do_test(scope_suite_cth, "ct_scope_suite_cth_SUITE.erl", [],Config). +scope_suite_group_only_cth(Config) when is_list(Config) -> + do_test(scope_suite_group_only_cth, + "ct_scope_suite_group_only_cth_SUITE.erl", + [],Config,ok,2,[{group,g1}]). + scope_per_group_cth(Config) when is_list(Config) -> do_test(scope_per_group_cth, "ct_scope_per_group_cth_SUITE.erl", [],Config). @@ -299,10 +308,74 @@ repeat_force_stop(Config) -> [{force_stop,skip_rest},{duration,"000009"}]). %% Test that expected callbacks, and only those, are called when a test -%% are fails due to clash in config alias names +%% fails due to clash in config alias names config_clash(Config) -> do_test(config_clash, "config_clash_SUITE.erl", [skip_cth], Config). +%% Test post_groups and post_all hook callbacks, introduced by OTP-14746 +alter_groups(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_groups_return,[{new_group,[tc1,tc2]}]}, + {post_all_return,[{group,new_group}]}],Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +alter_all(Config) -> + CfgFile = gen_config(?FUNCTION_NAME,[{post_all_return,[tc2]}],Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +alter_all_from_skip(Config) -> + CfgFile = gen_config(?FUNCTION_NAME,[{all_return,{skip,"skipped by all/0"}}, + {post_all_return,[tc2]}],Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +alter_all_to_skip(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_all_return,{skip,"skipped by post_all/3"}}], + Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +bad_return_groups(Config) -> + CfgFile = gen_config(?FUNCTION_NAME,[{post_groups_return,not_a_list}], + Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +bad_return_all(Config) -> + CfgFile = gen_config(?FUNCTION_NAME,[{post_all_return,not_a_list}], + Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +illegal_values_groups(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_groups_return,[{new_group,[this_test_does_not_exist]}, + this_is_not_a_group_def]}], + Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +illegal_values_all(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_all_return,[{group,this_group_does_not_exist}, + {this_is_not_a_valid_term}]}], + Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +crash_groups(Config) -> + CfgFile = gen_config(?FUNCTION_NAME,[{post_groups_return,crash}],Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +crash_all(Config) -> + CfgFile = gen_config(?FUNCTION_NAME,[{post_all_return,crash}],Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + %%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- @@ -322,6 +395,7 @@ do_test(Tag, {WhatTag,Wildcard}, CTHs, Config, Res, EC, ExtraOpts) -> filename:join([DataDir,"cth/tests",Wildcard])), {Opts,ERPid} = setup([{WhatTag,Files},{ct_hooks,CTHs},{label,Tag}|ExtraOpts], Config), + Res = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), @@ -347,6 +421,13 @@ reformat(Events, EH) -> %reformat(Events, _EH) -> % Events. +gen_config(Name,KeyVals,Config) -> + PrivDir = ?config(priv_dir,Config), + File = filename:join(PrivDir,atom_to_list(Name)++".cfg"), + ok = file:write_file(File,[io_lib:format("~p.~n",[{Key,Value}]) + || {Key,Value} <- KeyVals]), + File. + %%%----------------------------------------------------------------- %%% TEST EVENTS %%%----------------------------------------------------------------- @@ -365,13 +446,16 @@ test_events(one_empty_cth) -> {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, {?eh,cth,{empty_cth,id,[[]]}}, {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + %% check that post_groups and post_all comes after init when hook + %% is installed with start flag/option. + {?eh,cth,{empty_cth,post_groups,[ct_cth_empty_SUITE,[]]}}, + {?eh,cth,{empty_cth,post_all,[ct_cth_empty_SUITE,[test_case],[]]}}, {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, {?eh,cth,{empty_cth,pre_init_per_suite, [ct_cth_empty_SUITE,'$proplist',[]]}}, {?eh,cth,{empty_cth,post_init_per_suite, [ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, - {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, {?eh,cth,{empty_cth,pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, {?eh,cth,{empty_cth,post_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist','_',[]]}}, @@ -580,6 +664,10 @@ test_events(scope_suite_cth) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + %% check that post_groups and post_all comes before init when hook + %% is installed in suite/0 + {?eh,cth,{'_',post_groups,['_',[]]}}, + {?eh,cth,{'_',post_all,['_','_',[]]}}, {?eh,tc_start,{ct_scope_suite_cth_SUITE,init_per_suite}}, {?eh,cth,{'_',id,[[]]}}, {?eh,cth,{'_',init,['_',[]]}}, @@ -601,6 +689,34 @@ test_events(scope_suite_cth) -> {?eh,stop_logging,[]} ]; +test_events(scope_suite_group_only_cth) -> + Suite = ct_scope_suite_group_only_cth_SUITE, + CTH = empty_cth, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,1}}, + %% check that post_groups and post_all comes before init when hook + %% is installed in suite/0 + {?eh,cth,{CTH,post_groups,['_',['_']]}}, + {negative, + {?eh,cth,{CTH,post_all,['_','_','_']}}, + {?eh,tc_start,{Suite,init_per_suite}}}, + {?eh,cth,{CTH,id,[[]]}}, + {?eh,cth,{CTH,init,['_',[]]}}, + {?eh,cth,{CTH,pre_init_per_suite,[Suite,'$proplist',mystate]}}, + {?eh,cth,{CTH,post_init_per_suite,[Suite,'$proplist','$proplist',mystate]}}, + {?eh,tc_done,{Suite,init_per_suite,ok}}, + + {?eh,tc_start,{Suite,end_per_suite}}, + {?eh,cth,{CTH,pre_end_per_suite,[Suite,'$proplist',mystate]}}, + {?eh,cth,{CTH,post_end_per_suite,[Suite,'$proplist','_',mystate]}}, + {?eh,cth,{CTH,terminate,[mystate]}}, + {?eh,tc_done,{Suite,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + test_events(scope_per_group_cth) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, @@ -660,6 +776,8 @@ test_events(scope_suite_state_cth) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{'_',post_groups,['_',[]]}}, + {?eh,cth,{'_',post_all,['_','_',[]]}}, {?eh,tc_start,{ct_scope_suite_state_cth_SUITE,init_per_suite}}, {?eh,cth,{'_',id,[[test]]}}, {?eh,cth,{'_',init,['_',[test]]}}, @@ -2308,6 +2426,229 @@ test_events(config_clash) -> %% Make sure no 'cth_error' events are received! [{negative,{?eh,cth_error,'_'},E} || E <- Events]; +test_events(alter_groups) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE, + [{new_group,[tc1,tc2]}]]}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[{group,new_group}], + [{new_group,[tc1,tc2]}]]}}, + {?eh,start_info,{1,1,2}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE, + [{new_group,[tc1,tc2]}]]}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[{group,new_group}], + [{new_group,[tc1,tc2]}]]}}, + {?eh,tc_start,{all_and_groups_SUITE,{init_per_group,new_group,[]}}}, + {?eh,tc_done,{all_and_groups_SUITE, + {init_per_group,new_group,'$proplist'},ok}}, + {?eh,tc_start,{all_and_groups_SUITE,tc1}}, + {?eh,tc_done,{all_and_groups_SUITE,tc1,ok}}, + {?eh,tc_start,{all_and_groups_SUITE,tc2}}, + {?eh,tc_done,{all_and_groups_SUITE,tc2,ok}}, + {?eh,tc_start,{all_and_groups_SUITE,{end_per_group,new_group,[]}}}, + {?eh,tc_done,{all_and_groups_SUITE, + {end_per_group,new_group,'$proplist'},ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(alter_all) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE, + [{test_group,[tc1]}]]}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[tc2], + [{test_group,[tc1]}]]}}, + {?eh,start_info,{1,1,1}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[tc2],'_']}}, + {?eh,tc_start,{all_and_groups_SUITE,tc2}}, + {?eh,tc_done,{all_and_groups_SUITE,tc2,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(alter_all_from_skip) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE, + [{test_group,[tc1]}]]}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[tc2], + [{test_group,[tc1]}]]}}, + {?eh,start_info,{1,1,1}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[tc2],'_']}}, + {?eh,tc_start,{all_and_groups_SUITE,tc2}}, + {?eh,tc_done,{all_and_groups_SUITE,tc2,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(alter_all_to_skip) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE, + [{test_group,[tc1]}]]}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE, + {skip,"skipped by post_all/3"}, + [{test_group,[tc1]}]]}}, + {?eh,start_info,{1,1,0}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE, + {skip,"skipped by post_all/3"}, + '_']}}, + {?eh,tc_user_skip,{all_and_groups_SUITE,all,"skipped by post_all/3"}}, + {?eh,cth,{'_',on_tc_skip,[all_and_groups_SUITE,all, + {tc_user_skip,"skipped by post_all/3"}, + []]}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(illegal_values_groups) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups, + [all_and_groups_SUITE, + [{new_group,[this_test_does_not_exist]}, + this_is_not_a_group_def]]}}, + {?eh,start_info,{1,0,0}}, + {?eh,cth,{empty_cth,post_groups, + [all_and_groups_SUITE, + [{new_group,[this_test_does_not_exist]}, + this_is_not_a_group_def]]}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, + {?eh,tc_done,{ct_framework,error_in_suite,{failed,{error,'_'}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(illegal_values_all) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all, + [all_and_groups_SUITE, + [{group,this_group_does_not_exist}, + {this_is_not_a_valid_term}],'_']}}, + {?eh,start_info,{1,0,0}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all, + [all_and_groups_SUITE, + [{group,this_group_does_not_exist}, + {this_is_not_a_valid_term}],'_']}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, + {?eh,tc_done, + {ct_framework,error_in_suite, + {failed, + {error,'Invalid reference to group this_group_does_not_exist in all_and_groups_SUITE:all/0'}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(bad_return_groups) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,not_a_list]}}, + {?eh,start_info,{1,0,0}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,not_a_list]}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, + {?eh,tc_done, + {ct_framework,error_in_suite, + {failed, + {error, + {'Bad return value from post_groups/2 hook function',not_a_list}}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(bad_return_all) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,not_a_list,'_']}}, + {?eh,start_info,{1,0,0}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,not_a_list,'_']}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, + {?eh,tc_done, + {ct_framework,error_in_suite, + {failed, + {error,{'Bad return value from post_all/3 hook function',not_a_list}}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(crash_groups) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,crash]}}, + {?eh,start_info,{1,0,0}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,crash]}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, + {?eh,tc_done,{ct_framework,error_in_suite, + {failed, + {error,"all_and_groups_cth:post_groups/2 CTH call failed"}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(crash_all) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,crash,'_']}}, + {?eh,start_info,{1,0,0}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,crash,'_']}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, + {?eh,tc_done,{ct_framework,error_in_suite, + {failed, + {error,"all_and_groups_cth:post_all/3 CTH call failed"}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + test_events(ok) -> ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_SUITE.erl new file mode 100644 index 0000000000..adc86005f9 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_SUITE.erl @@ -0,0 +1,47 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(all_and_groups_SUITE). + +-suite_defaults([{timetrap, {minutes, 10}}]). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include("ct.hrl"). + +init_per_group(_Group,Config) -> + Config. + +end_per_group(_Group,Config) -> + ok. + +all() -> + ct:get_config(all_return,[{group,test_group}]). + +groups() -> + [{test_group,[tc1]}]. + +%% Test cases starts here. +tc1(Config) -> + ok. + +tc2(Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_cth.erl new file mode 100644 index 0000000000..9ebc00e9de --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_cth.erl @@ -0,0 +1,100 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +-module(all_and_groups_cth). + + +-include_lib("common_test/src/ct_util.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +%% Send a cth_error event if a callback is called with unexpected arguments +-define(fail(Info), + gen_event:notify( + ?CT_EVMGR_REF, + #event{ name = cth_error, + node = node(), + data = {illegal_hook_callback,{?MODULE,?FUNCTION_NAME,Info}}})). + +%% CT Hooks +-compile(export_all). + +id(Opts) -> + empty_cth:id(Opts). + +post_groups(Suite,Groups) -> + case empty_cth:post_groups(Suite,ct:get_config(post_groups_return,Groups)) of + crash -> error(crash_in_post_groups); + R -> R + end. + +post_all(Suite,Tests,Groups) -> + case empty_cth:post_all(Suite,ct:get_config(post_all_return,Tests),Groups) of + crash -> error(crash_in_post_all); + R -> R + end. + +init(Id, Opts) -> + empty_cth:init(Id, Opts). + +pre_init_per_suite(Suite, Config, State) -> + empty_cth:pre_init_per_suite(Suite,Config,State). + +post_init_per_suite(Suite,Config,Return,State) -> + empty_cth:post_init_per_suite(Suite,Config,Return,State). + +pre_end_per_suite(Suite,Config,State) -> + empty_cth:pre_end_per_suite(Suite,Config,State). + +post_end_per_suite(Suite,Config,Return,State) -> + empty_cth:post_end_per_suite(Suite,Config,Return,State). + +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). + +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). + +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). + +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). + +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). + +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). + +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). + +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). + +on_tc_fail(Suite,TC,Reason,State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). + +on_tc_skip(Suite,TC,Reason,State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). + +terminate(State) -> + empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_match_state_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_match_state_cth.erl new file mode 100644 index 0000000000..38c9da903d --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_match_state_cth.erl @@ -0,0 +1,58 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +-module(ct_match_state_cth). + + +-include_lib("common_test/src/ct_util.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-compile(export_all). + +id(Opts) -> + empty_cth:id(Opts). + +post_groups(Suite, Groups) -> + empty_cth:post_groups(Suite, Groups). + +post_all(Suite, Tests, Groups) -> + empty_cth:post_all(Suite, Tests, Groups). + +init(Id, Opts) -> + empty_cth:init(Id, Opts), + {ok,mystate}. + +%% In the following, always match against the state value, to ensure +%% that init has indeed been called before the rest of the hooks. +pre_init_per_suite(Suite,Config,mystate) -> + empty_cth:pre_init_per_suite(Suite,Config,mystate). + +post_init_per_suite(Suite,Config,Return,mystate) -> + empty_cth:post_init_per_suite(Suite,Config,Return,mystate). + +pre_end_per_suite(Suite,Config,mystate) -> + empty_cth:pre_end_per_suite(Suite,Config,mystate). + +post_end_per_suite(Suite,Config,Return,mystate) -> + empty_cth:post_end_per_suite(Suite,Config,Return,mystate). + +terminate(mystate) -> + empty_cth:terminate(mystate). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_group_only_cth_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_group_only_cth_SUITE.erl new file mode 100644 index 0000000000..537c97d3f0 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_group_only_cth_SUITE.erl @@ -0,0 +1,54 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(ct_scope_suite_group_only_cth_SUITE). + +-suite_defaults([{timetrap, {minutes, 10}}]). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include("ct.hrl"). + +%% Test server callback functions +suite() -> + [{ct_hooks,[ct_match_state_cth]}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +all() -> + [test_case]. + +groups() -> + [{g1,[test_case]}]. + +%% Test cases starts here. +test_case(Config) when is_list(Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl index 961ea68d2d..324f1dc80a 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl @@ -39,6 +39,9 @@ -export([id/1]). -export([init/2]). +-export([post_all/3]). +-export([post_groups/2]). + -export([pre_init_per_suite/3]). -export([post_init_per_suite/4]). -export([pre_end_per_suite/3]). @@ -71,6 +74,31 @@ -record(state, { id = ?MODULE :: term()}). +%% @doc Called after groups/0. +%% you can change the return value in this function. +-spec post_groups(Suite :: atom(), Groups :: list()) -> list(). +post_groups(Suite,Groups) -> + gen_event:notify( + ?CT_EVMGR_REF, #event{ name = cth, node = node(), + data = {?MODULE, post_groups, + [Suite,Groups]}}), + ct:log("~w:post_groups(~w) called", [?MODULE,Suite]), + Groups. + +%% @doc Called after all/0. +%% you can change the return value in this function. +-spec post_all(Suite :: atom(), + Tests :: list(), + Groups :: term()) -> + list(). +post_all(Suite,Tests,Groups) -> + gen_event:notify( + ?CT_EVMGR_REF, #event{ name = cth, node = node(), + data = {?MODULE, post_all, + [Suite,Tests,Groups]}}), + ct:log("~w:post_all(~w) called", [?MODULE,Suite]), + Tests. + %% @doc Always called before any other callback function. Use this to initiate %% any common state. It should return an state for this CTH. -spec init(Id :: term(), Opts :: proplists:proplist()) -> -- cgit v1.2.3