aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test/test
diff options
context:
space:
mode:
authorSiri Hansen <siri@erlang.org>2017-02-16 15:38:05 +0100
committerSiri Hansen <siri@erlang.org>2017-02-20 12:40:08 +0100
commit6b7463094db2ec3c2c0e4f8784dcd64ef186c96b (patch)
treedaac80c3c2155c7c98aa7edfa2d0762cff07d972 /lib/common_test/test
parent2031e6ceb5588bbecd480f1fb9ca5026f189d7d2 (diff)
downloadotp-6b7463094db2ec3c2c0e4f8784dcd64ef186c96b.tar.gz
otp-6b7463094db2ec3c2c0e4f8784dcd64ef186c96b.tar.bz2
otp-6b7463094db2ec3c2c0e4f8784dcd64ef186c96b.zip
[ct] Fix faulty hook callbacks for skipped tests
The following bugs are corrected: * Tests that were skipped before calling pre_init_per_* got faulty calls to the corresponding post_init_per_*. E.g. if a test was skipped because suite/0 returned a 'require' statement that was not fulfilled, then post_init_per_suite would be called, even though pre_init_per_suite and init_per_suite were not called. * Tests that were skipped before or in init_per_testcase got faulty calls to pre_end_per_testcase and post_end_per_testcase (end_per_testcase is not called in these situations). Test are added to make sure that the expected callbacks, and only those, are called when tests are skipped in different ways. Conflicts: lib/common_test/test/ct_hooks_SUITE.erl
Diffstat (limited to 'lib/common_test/test')
-rw-r--r--lib/common_test/test/ct_hooks_SUITE.erl589
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/config_clash_SUITE.erl43
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/repeat_SUITE.erl42
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/seq_SUITE.erl45
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip.spec8
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl87
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl171
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_fail_SUITE.erl53
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_group_SUITE.erl64
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_init_SUITE.erl53
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_init_tc_cth.erl79
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_req_SUITE.erl53
-rw-r--r--lib/common_test/test/ct_test_support.erl24
13 files changed, 1282 insertions, 29 deletions
diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl
index 835e3d4805..4624324295 100644
--- a/lib/common_test/test/ct_hooks_SUITE.erl
+++ b/lib/common_test/test/ct_hooks_SUITE.erl
@@ -82,11 +82,13 @@ all(suite) ->
scope_suite_state_cth,
fail_pre_suite_cth, double_fail_pre_suite_cth,
fail_post_suite_cth, skip_pre_suite_cth, skip_pre_end_cth,
+ skip_pre_init_tc_cth,
skip_post_suite_cth, recover_post_suite_cth, update_config_cth,
state_update_cth, options_cth, same_id_cth,
fail_n_skip_with_minimal_cth, prio_cth, no_config,
no_init_suite_config, no_init_config, no_end_config,
- fallback, data_dir, cth_log
+ failed_sequence, repeat_force_stop, config_clash,
+ callbacks_on_skip, fallback, data_dir, cth_log
]
).
@@ -191,6 +193,10 @@ skip_post_suite_cth(Config) when is_list(Config) ->
do_test(skip_post_suite_cth, "ct_cth_empty_SUITE.erl",
[skip_post_suite_cth],Config).
+skip_pre_init_tc_cth(Config) ->
+ do_test(skip_pre_init_tc_cth, "ct_cth_empty_SUITE.erl",
+ [skip_pre_init_tc_cth],Config).
+
recover_post_suite_cth(Config) when is_list(Config) ->
do_test(recover_post_suite_cth, "ct_cth_fail_per_suite_SUITE.erl",
[recover_post_suite_cth],Config).
@@ -272,23 +278,46 @@ cth_log(Config) when is_list(Config) ->
fallback(Config) ->
do_test(fallback, "all_hook_callbacks_SUITE.erl",[fallback_cth], Config).
+%% Test that expected callbacks, and only those, are called when tests
+%% are skipped in different ways
+callbacks_on_skip(Config) ->
+ do_test(callbacks_on_skip, {spec,"skip.spec"},[skip_cth], Config).
+
+%% Test that expected callbacks, and only those, are called when tests
+%% are skipped due to failed sequence
+failed_sequence(Config) ->
+ do_test(failed_sequence, "seq_SUITE.erl", [skip_cth], Config).
+
+%% Test that expected callbacks, and only those, are called when tests
+%% are skipped due to {force_stop,skip_rest} option
+repeat_force_stop(Config) ->
+ do_test(repeat_force_stop, "repeat_SUITE.erl", [skip_cth], Config, ok, 2,
+ [{force_stop,skip_rest},{duration,"000009"}]).
+
+%% Test that expected callbacks, and only those, are called when a test
+%% are fails due to clash in config alias names
+config_clash(Config) ->
+ do_test(config_clash, "config_clash_SUITE.erl", [skip_cth], Config).
+
%%%-----------------------------------------------------------------
%%% HELP FUNCTIONS
%%%-----------------------------------------------------------------
-do_test(Tag, SWC, CTHs, Config) ->
- do_test(Tag, SWC, CTHs, Config, ok).
-do_test(Tag, SWC, CTHs, Config, {error,_} = Res) ->
- do_test(Tag, SWC, CTHs, Config, Res, 1);
-do_test(Tag, SWC, CTHs, Config, Res) ->
- do_test(Tag, SWC, CTHs, Config, Res, 2).
+do_test(Tag, WTT, CTHs, Config) ->
+ do_test(Tag, WTT, CTHs, Config, ok).
+do_test(Tag, WTT, CTHs, Config, {error,_} = Res) ->
+ do_test(Tag, WTT, CTHs, Config, Res, 1,[]);
+do_test(Tag, WTT, CTHs, Config, Res) ->
+ do_test(Tag, WTT, CTHs, Config, Res, 2,[]).
-do_test(Tag, SuiteWildCard, CTHs, Config, Res, EC) ->
+do_test(Tag, WhatToTest, CTHs, Config, Res, EC, ExtraOpts) when is_list(WhatToTest) ->
+ do_test(Tag, {suite,WhatToTest}, CTHs, Config, Res, EC, ExtraOpts);
+do_test(Tag, {WhatTag,Wildcard}, CTHs, Config, Res, EC, ExtraOpts) ->
DataDir = ?config(data_dir, Config),
- Suites = filelib:wildcard(
- filename:join([DataDir,"cth/tests",SuiteWildCard])),
- {Opts,ERPid} = setup([{suite,Suites},
- {ct_hooks,CTHs},{label,Tag}], Config),
+ Files = filelib:wildcard(
+ filename:join([DataDir,"cth/tests",Wildcard])),
+ {Opts,ERPid} =
+ setup([{WhatTag,Files},{ct_hooks,CTHs},{label,Tag}|ExtraOpts], Config),
Res = ct_test_support:run(Opts, Config),
Events = ct_test_support:get_events(ERPid, Config),
@@ -845,6 +874,41 @@ test_events(skip_post_suite_cth) ->
{?eh,stop_logging,[]}
];
+test_events(skip_pre_init_tc_cth) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,init,['_',[]]}},
+ {?eh,start_info,{1,1,1}},
+ {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}},
+ {?eh,cth,{empty_cth,pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_suite,
+ [ct_cth_empty_SUITE,'$proplist','$proplist',[]]}},
+ {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}},
+ {?eh,tc_start,{ct_cth_empty_SUITE,test_case}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [ct_cth_empty_SUITE,test_case,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [ct_cth_empty_SUITE,test_case,'$proplist',
+ {skip,"Skipped in pre_init_per_testcase"},
+ []]}},
+ {?eh,tc_done,{ct_cth_empty_SUITE,test_case,
+ {skipped,"Skipped in pre_init_per_testcase"}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [ct_cth_empty_SUITE,test_case,
+ {tc_user_skip,{skipped,"Skipped in pre_init_per_testcase"}},
+ []]}},
+ {?eh,test_stats,{0,0,{1,0}}},
+ {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}},
+ {?eh,cth,{empty_cth,pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_suite,
+ [ct_cth_empty_SUITE,'$proplist',ok,[]]}},
+ {?eh,tc_done,{ct_cth_empty_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ];
+
test_events(recover_post_suite_cth) ->
Suite = ct_cth_fail_per_suite_SUITE,
[
@@ -1551,11 +1615,6 @@ test_events(fallback) ->
{?eh,cth,{empty_cth,post_init_per_testcase,
[fallback_nosuite,skip_case,'$proplist',
{skip,"Skipped in init_per_testcase/2"},[]]}},
- {?eh,cth,{empty_cth,pre_end_per_testcase,
- [fallback_nosuite,skip_case,'$proplist',[]]}},
- {?eh,cth,{empty_cth,post_end_per_testcase,
- [fallback_nosuite,skip_case,'$proplist',
- {skip,"Skipped in init_per_testcase/2"},[]]}},
{?eh,tc_done,{all_hook_callbacks_SUITE,skip_case,
{skipped,"Skipped in init_per_testcase/2"}}},
{?eh,cth,{empty_cth,on_tc_skip,
@@ -1574,6 +1633,502 @@ test_events(fallback) ->
{?eh,stop_logging,[]}
];
+test_events(callbacks_on_skip) ->
+ %% skip_cth.erl will send a 'cth_error' event if a hook is
+ %% erroneously called. Therefore, all Events are changed to
+ %% {negative,{?eh,cth_error,'_'},Event}
+ %% at the end of this function.
+ Events =
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,id,[[]]}},
+ {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ {?eh,start_info,{6,6,12}},
+
+ %% all_hook_callbacks_SUITE is skipped in spec
+ %% Only the on_tc_skip callback shall be called
+ {?eh,tc_user_skip,{all_hook_callbacks_SUITE,all,"Skipped in spec"}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [all_hook_callbacks_SUITE,all,
+ {tc_user_skip,"Skipped in spec"},
+ []]}},
+ {?eh,test_stats,{0,0,{1,0}}},
+
+ %% skip_init_SUITE is skipped in its init_per_suite function
+ %% No group- or testcase-functions shall be called.
+ {?eh,tc_start,{skip_init_SUITE,init_per_suite}},
+ {?eh,cth,{empty_cth,pre_init_per_suite,
+ [skip_init_SUITE,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_init_per_suite,
+ [skip_init_SUITE,
+ '$proplist',
+ {skip,"Skipped in init_per_suite/1"},
+ []]}},
+ {?eh,tc_done,{skip_init_SUITE,init_per_suite,
+ {skipped,"Skipped in init_per_suite/1"}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_init_SUITE,init_per_suite,
+ {tc_user_skip,{skipped,"Skipped in init_per_suite/1"}},
+ []]}},
+ {?eh,tc_user_skip,{skip_init_SUITE,test_case,"Skipped in init_per_suite/1"}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_init_SUITE,test_case,
+ {tc_user_skip,"Skipped in init_per_suite/1"},
+ []]}},
+ {?eh,test_stats,{0,0,{2,0}}},
+ {?eh,tc_user_skip,{skip_init_SUITE,end_per_suite,
+ "Skipped in init_per_suite/1"}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_init_SUITE,end_per_suite,
+ {tc_user_skip,"Skipped in init_per_suite/1"},
+ []]}},
+
+ %% skip_req_SUITE is auto-skipped since a 'require' statement
+ %% returned by suite/0 is not fulfilled.
+ %% No group- or testcase-functions shall be called.
+ {?eh,tc_start,{skip_req_SUITE,init_per_suite}},
+ {?eh,tc_done,{skip_req_SUITE,init_per_suite,
+ {auto_skipped,{require_failed_in_suite0,
+ {not_available,whatever}}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_req_SUITE,init_per_suite,
+ {tc_auto_skip,{auto_skipped,{require_failed_in_suite0,
+ {not_available,whatever}}}},
+ []]}},
+ {?eh,tc_auto_skip,{skip_req_SUITE,test_case,{require_failed_in_suite0,
+ {not_available,whatever}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_req_SUITE,test_case,
+ {tc_auto_skip,{require_failed_in_suite0,
+ {not_available,whatever}}},
+ []]}},
+ {?eh,test_stats,{0,0,{2,1}}},
+ {?eh,tc_auto_skip,{skip_req_SUITE,end_per_suite,
+ {require_failed_in_suite0,
+ {not_available,whatever}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_req_SUITE,end_per_suite,
+ {tc_auto_skip,{require_failed_in_suite0,
+ {not_available,whatever}}},
+ []]}},
+
+ %% skip_fail_SUITE is auto-skipped since the suite/0 function
+ %% retuns a faluty format.
+ %% No group- or testcase-functions shall be called.
+ {?eh,tc_start,{skip_fail_SUITE,init_per_suite}},
+ {?eh,tc_done,{skip_fail_SUITE,init_per_suite,
+ {failed,{error,{suite0_failed,bad_return_value}}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_fail_SUITE,init_per_suite,
+ {tc_auto_skip,
+ {auto_skipped,
+ {failed,{error,{suite0_failed,bad_return_value}}}}},
+ []]}},
+ {?eh,tc_auto_skip,{skip_fail_SUITE,test_case,
+ {failed,{error,{suite0_failed,bad_return_value}}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_fail_SUITE,test_case,
+ {tc_auto_skip,
+ {failed,{error,{suite0_failed,bad_return_value}}}},
+ []]}},
+ {?eh,test_stats,{0,0,{2,2}}},
+ {?eh,tc_auto_skip,{skip_fail_SUITE,end_per_suite,
+ {failed,{error,{suite0_failed,bad_return_value}}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_fail_SUITE,end_per_suite,
+ {tc_auto_skip,
+ {failed,{error,{suite0_failed,bad_return_value}}}},
+ []]}},
+
+ %% skip_group_SUITE
+ {?eh,tc_start,{skip_group_SUITE,init_per_suite}},
+ {?eh,cth,{empty_cth,pre_init_per_suite,
+ [skip_group_SUITE,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_init_per_suite,
+ [skip_group_SUITE,
+ '$proplist',
+ '_',
+ []]}},
+ {?eh,tc_done,{skip_group_SUITE,init_per_suite,ok}},
+
+ %% test_group_1 - auto_skip due to require failed
+ [{?eh,tc_start,{skip_group_SUITE,{init_per_group,test_group_1,[]}}},
+ {?eh,tc_done,
+ {skip_group_SUITE,{init_per_group,test_group_1,[]},
+ {auto_skipped,{require_failed,{not_available,whatever}}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_group_SUITE,
+ {init_per_group,test_group_1},
+ {tc_auto_skip,
+ {auto_skipped,{require_failed,{not_available,whatever}}}},
+ []]}},
+ {?eh,tc_auto_skip,{skip_group_SUITE,{test_case,test_group_1},
+ {require_failed,{not_available,whatever}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_group_SUITE,
+ {test_case,test_group_1},
+ {tc_auto_skip,{require_failed,{not_available,whatever}}},
+ []]}},
+ {?eh,test_stats,{0,0,{2,3}}},
+ {?eh,tc_auto_skip,{skip_group_SUITE,{end_per_group,test_group_1},
+ {require_failed,{not_available,whatever}}}}],
+ %% The following appears to be outside of the group, but
+ %% that's only an implementation detail in
+ %% ct_test_support.erl - it does not know about events from
+ %% test suite specific hooks and regards the group ended with
+ %% the above tc_auto_skip-event for end_per_group.
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_group_SUITE,
+ {end_per_group,test_group_1},
+ {tc_auto_skip,{require_failed,{not_available,whatever}}},
+ []]}},
+
+ %% test_group_2 - auto_skip due to failed return from group/1
+ [{?eh,tc_start,{skip_group_SUITE,{init_per_group,test_group_2,[]}}},
+ {?eh,tc_done,
+ {skip_group_SUITE,{init_per_group,test_group_2,[]},
+ {auto_skipped,{group0_failed,bad_return_value}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_group_SUITE,
+ {init_per_group,test_group_2},
+ {tc_auto_skip,
+ {auto_skipped,{group0_failed,bad_return_value}}},
+ []]}},
+ {?eh,tc_auto_skip,{skip_group_SUITE,{test_case,test_group_2},
+ {group0_failed,bad_return_value}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_group_SUITE,
+ {test_case,test_group_2},
+ {tc_auto_skip,{group0_failed,bad_return_value}},
+ []]}},
+ {?eh,test_stats,{0,0,{2,4}}},
+ {?eh,tc_auto_skip,{skip_group_SUITE,{end_per_group,test_group_2},
+ {group0_failed,bad_return_value}}}],
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_group_SUITE,
+ {end_per_group,test_group_2},
+ {tc_auto_skip,{group0_failed,bad_return_value}},
+ []]}},
+ %% test_group_3 - user_skip in init_per_group/2
+ [{?eh,tc_start,
+ {skip_group_SUITE,{init_per_group,test_group_3,[]}}},
+ {?eh,cth,{empty_cth,pre_init_per_group,
+ [skip_group_SUITE,test_group_3,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_group,
+ [skip_group_SUITE,test_group_3,'$proplist',
+ {skip,"Skipped in init_per_group/2"},
+ []]}},
+ {?eh,tc_done,{skip_group_SUITE,
+ {init_per_group,test_group_3,[]},
+ {skipped,"Skipped in init_per_group/2"}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_group_SUITE,
+ {init_per_group,test_group_3},
+ {tc_user_skip,{skipped,"Skipped in init_per_group/2"}},
+ []]}},
+ {?eh,tc_user_skip,{skip_group_SUITE,
+ {test_case,test_group_3},
+ "Skipped in init_per_group/2"}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_group_SUITE,
+ {test_case,test_group_3},
+ {tc_user_skip,"Skipped in init_per_group/2"},
+ []]}},
+ {?eh,test_stats,{0,0,{3,4}}},
+ {?eh,tc_user_skip,{skip_group_SUITE,
+ {end_per_group,test_group_3},
+ "Skipped in init_per_group/2"}}],
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_group_SUITE,
+ {end_per_group,test_group_3},
+ {tc_user_skip,"Skipped in init_per_group/2"},
+ []]}},
+
+ {?eh,tc_start,{skip_group_SUITE,end_per_suite}},
+ {?eh,cth,{empty_cth,pre_end_per_suite,
+ [skip_group_SUITE,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_end_per_suite,
+ [skip_group_SUITE,
+ '$proplist',
+ ok,[]]}},
+ {?eh,tc_done,{skip_group_SUITE,end_per_suite,ok}},
+
+
+ %% skip_case_SUITE has 4 test cases which are all skipped in
+ %% different ways
+ {?eh,tc_start,{skip_case_SUITE,init_per_suite}},
+ {?eh,cth,{empty_cth,pre_init_per_suite,
+ [skip_case_SUITE,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_init_per_suite,
+ [skip_case_SUITE,
+ '$proplist',
+ '_',
+ []]}},
+ {?eh,tc_done,{skip_case_SUITE,init_per_suite,ok}},
+
+ %% Skip in spec -> only on_tc_skip shall be called
+ {?eh,tc_user_skip,{skip_case_SUITE,skip_in_spec,"Skipped in spec"}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_case_SUITE,skip_in_spec,
+ {tc_user_skip,"Skipped in spec"},
+ []]}},
+ {?eh,test_stats,{0,0,{4,4}}},
+
+ %% Skip in init_per_testcase -> pre/post_end_per_testcase
+ %% shall not be called
+ {?eh,tc_start,{skip_case_SUITE,skip_in_init}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [skip_case_SUITE,skip_in_init,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [skip_case_SUITE,skip_in_init,
+ '$proplist',
+ {skip,"Skipped in init_per_testcase/2"},
+ []]}},
+ {?eh,tc_done,{skip_case_SUITE,skip_in_init,
+ {skipped,"Skipped in init_per_testcase/2"}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_case_SUITE,skip_in_init,
+ {tc_user_skip,{skipped,"Skipped in init_per_testcase/2"}},
+ []]}},
+ {?eh,test_stats,{0,0,{5,4}}},
+
+ %% Fail in init_per_testcase -> pre/post_end_per_testcase
+ %% shall not be called
+ {?eh,tc_start,{skip_case_SUITE,fail_in_init}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [skip_case_SUITE,fail_in_init,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [skip_case_SUITE,fail_in_init,
+ '$proplist',
+ {skip,{failed,'_'}},
+ []]}},
+ {?eh,tc_done,{skip_case_SUITE,fail_in_init,
+ {auto_skipped,{failed,'_'}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_case_SUITE,fail_in_init,
+ {tc_auto_skip,{auto_skipped,{failed,'_'}}},
+ []]}},
+ {?eh,test_stats,{0,0,{5,5}}},
+
+ %% Skip in testcase function -> all callbacks shall be called
+ {?eh,tc_start,{skip_case_SUITE,skip_in_case}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [skip_case_SUITE,skip_in_case,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [skip_case_SUITE,skip_in_case,
+ '$proplist',
+ ok,[]]}},
+ {?eh,cth,{empty_cth,pre_end_per_testcase,
+ [skip_case_SUITE,skip_in_case,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_end_per_testcase,
+ [skip_case_SUITE,skip_in_case,
+ '$proplist',
+ {skip,"Skipped in test case function"},
+ []]}},
+ {?eh,tc_done,{skip_case_SUITE,skip_in_case,
+ {skipped,"Skipped in test case function"}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_case_SUITE,skip_in_case,
+ {tc_user_skip,{skipped,"Skipped in test case function"}},
+ []]}},
+ {?eh,test_stats,{0,0,{6,5}}},
+
+ %% Auto skip due to failed 'require' -> only the on_tc_skip
+ %% callback shall be called
+ {?eh,tc_start,{skip_case_SUITE,req_auto_skip}},
+ {?eh,tc_done,{skip_case_SUITE,req_auto_skip,
+ {auto_skipped,{require_failed,{not_available,whatever}}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_case_SUITE,req_auto_skip,
+ {tc_auto_skip,
+ {auto_skipped,{require_failed,{not_available,whatever}}}},
+ []]}},
+ {?eh,test_stats,{0,0,{6,6}}},
+
+ %% Auto skip due to failed testcase/0 function -> only the
+ %% on_tc_skip callback shall be called
+ {?eh,tc_start,{skip_case_SUITE,fail_auto_skip}},
+ {?eh,tc_done,{skip_case_SUITE,fail_auto_skip,
+ {auto_skipped,{testcase0_failed,bad_return_value}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_case_SUITE,fail_auto_skip,
+ {tc_auto_skip,
+ {auto_skipped,{testcase0_failed,bad_return_value}}},
+ []]}},
+ {?eh,test_stats,{0,0,{6,7}}},
+
+ {?eh,tc_start,{skip_case_SUITE,end_per_suite}},
+ {?eh,cth,{empty_cth,pre_end_per_suite,
+ [skip_case_SUITE,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_end_per_suite,
+ [skip_case_SUITE,
+ '$proplist',
+ ok,[]]}},
+ {?eh,tc_done,{skip_case_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ],
+ %% Make sure no 'cth_error' events are received!
+ [{negative,{?eh,cth_error,'_'},E} || E <- Events];
+
+test_events(failed_sequence) ->
+ %% skip_cth.erl will send a 'cth_error' event if a hook is
+ %% erroneously called. Therefore, all Events are changed to
+ %% {negative,{?eh,cth_error,'_'},Event}
+ %% at the end of this function.
+ Events =
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,id,[[]]}},
+ {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ {?eh,start_info,{1,1,2}},
+ {?eh,tc_start,{ct_framework,init_per_suite}},
+ {?eh,cth,{empty_cth,pre_init_per_suite,[seq_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_suite,
+ [seq_SUITE,'$proplist','$proplist',[]]}},
+ {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+ {?eh,tc_start,{seq_SUITE,test_case_1}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [seq_SUITE,test_case_1,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [seq_SUITE,test_case_1,'$proplist',ok,[]]}},
+ {?eh,cth,{empty_cth,pre_end_per_testcase,
+ [seq_SUITE,test_case_1,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_testcase,
+ [seq_SUITE,test_case_1,'$proplist',
+ {error,failed_on_purpose},[]]}},
+ {?eh,tc_done,{seq_SUITE,test_case_1,{failed,{error,failed_on_purpose}}}},
+ {?eh,cth,{empty_cth,on_tc_fail,
+ [seq_SUITE,test_case_1,{failed,failed_on_purpose},[]]}},
+ {?eh,test_stats,{0,1,{0,0}}},
+ {?eh,tc_done,{seq_SUITE,test_case_2,
+ {auto_skipped,{sequence_failed,seq1,test_case_1}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [seq_SUITE,test_case_2,
+ {tc_auto_skip,
+ {auto_skipped,{sequence_failed,seq1,test_case_1}}},
+ []]}},
+ {?eh,test_stats,{0,1,{0,1}}},
+ {?eh,tc_start,{ct_framework,end_per_suite}},
+ {?eh,cth,{empty_cth,pre_end_per_suite,[seq_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_suite,[seq_SUITE,'$proplist',ok,[]]}},
+ {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ],
+ %% Make sure no 'cth_error' events are received!
+ [{negative,{?eh,cth_error,'_'},E} || E <- Events];
+
+test_events(repeat_force_stop) ->
+ %% skip_cth.erl will send a 'cth_error' event if a hook is
+ %% erroneously called. Therefore, all Events are changed to
+ %% {negative,{?eh,cth_error,'_'},Event}
+ %% at the end of this function.
+ Events=
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,id,[[]]}},
+ {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ {?eh,start_info,{1,1,2}},
+ {?eh,tc_start,{ct_framework,init_per_suite}},
+ {?eh,cth,{empty_cth,pre_init_per_suite,[repeat_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_suite,
+ [repeat_SUITE,'$proplist','$proplist',[]]}},
+ {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+ {?eh,tc_start,{repeat_SUITE,test_case_1}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [repeat_SUITE,test_case_1,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [repeat_SUITE,test_case_1,'$proplist',ok,[]]}},
+ {?eh,cth,{empty_cth,pre_end_per_testcase,
+ [repeat_SUITE,test_case_1,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_testcase,
+ [repeat_SUITE,test_case_1,'$proplist',ok,[]]}},
+ {?eh,tc_done,{repeat_SUITE,test_case_1,ok}},
+ {?eh,test_stats,{1,0,{0,0}}},
+ {?eh,tc_done,{repeat_SUITE,test_case_2,
+ {auto_skipped,
+ "Repeated test stopped by force_stop option"}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [repeat_SUITE,test_case_2,
+ {tc_auto_skip,
+ {auto_skipped,"Repeated test stopped by force_stop option"}},
+ []]}},
+ {?eh,test_stats,{1,0,{0,1}}},
+ {?eh,tc_start,{ct_framework,end_per_suite}},
+ {?eh,cth,{empty_cth,pre_end_per_suite,[repeat_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_suite,
+ [repeat_SUITE,'$proplist',ok,[]]}},
+ {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ],
+ %% Make sure no 'cth_error' events are received!
+ [{negative,{?eh,cth_error,'_'},E} || E <- Events];
+
+test_events(config_clash) ->
+ %% skip_cth.erl will send a 'cth_error' event if a hook is
+ %% erroneously called. Therefore, all Events are changed to
+ %% {negative,{?eh,cth_error,'_'},Event}
+ %% at the end of this function.
+ Events =
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,id,[[]]}},
+ {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ {?eh,start_info,{1,1,1}},
+ {?eh,tc_start,{ct_framework,init_per_suite}},
+ {?eh,cth,{empty_cth,pre_init_per_suite,
+ [config_clash_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_suite,
+ [config_clash_SUITE,'$proplist','$proplist',[]]}},
+ {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+ {?eh,tc_start,{config_clash_SUITE,test_case_1}},
+ {?eh,tc_done,{config_clash_SUITE,test_case_1,
+ {failed,{error,{config_name_already_in_use,[aa]}}}}},
+ {?eh,cth,{empty_cth,on_tc_fail,
+ [config_clash_SUITE,test_case_1,
+ {failed,{config_name_already_in_use,[aa]}},
+ []]}},
+ {?eh,test_stats,{0,1,{0,0}}},
+ {?eh,tc_start,{ct_framework,end_per_suite}},
+ {?eh,cth,{empty_cth,pre_end_per_suite,
+ [config_clash_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_suite,
+ [config_clash_SUITE,'$proplist',ok,[]]}},
+ {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ],
+ %% Make sure no 'cth_error' events are received!
+ [{negative,{?eh,cth_error,'_'},E} || E <- Events];
+
test_events(ok) ->
ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/config_clash_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/config_clash_SUITE.erl
new file mode 100644
index 0000000000..f74c757cc1
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/config_clash_SUITE.erl
@@ -0,0 +1,43 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(config_clash_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+suite() ->
+ [{require,aa,yy},{default_config,yy,"this is a default value"}].
+
+init_per_testcase(_,Config) ->
+ Config.
+
+end_per_testcase(_,_) ->
+ ok.
+
+all() ->
+ [test_case_1].
+
+%% Test cases starts here.
+test_case_1() ->
+ [{require,aa,xx}].
+test_case_1(_Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/repeat_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/repeat_SUITE.erl
new file mode 100644
index 0000000000..fded4c02ab
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/repeat_SUITE.erl
@@ -0,0 +1,42 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(repeat_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+init_per_testcase(_,Config) ->
+ Config.
+
+end_per_testcase(_,_) ->
+ ok.
+
+all() ->
+ [test_case_1, test_case_2].
+
+%% Test cases starts here.
+test_case_1(_Config) ->
+ timer:sleep(10000),
+ ok.
+
+test_case_2(_Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/seq_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/seq_SUITE.erl
new file mode 100644
index 0000000000..6d1302fd35
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/seq_SUITE.erl
@@ -0,0 +1,45 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(seq_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+init_per_testcase(_,Config) ->
+ Config.
+
+end_per_testcase(_,_) ->
+ ok.
+
+all() ->
+ [{sequence,seq1}].
+
+sequences() ->
+ [{seq1,[test_case_1,test_case_2]}].
+
+%% Test cases starts here.
+test_case_1(_Config) ->
+ exit(failed_on_purpose).
+
+test_case_2(_Config) ->
+ ct:fail("This test shall never be run since test_case_1 fails "
+ "and they are run in sequence").
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip.spec b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip.spec
new file mode 100644
index 0000000000..a271c5e8b2
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip.spec
@@ -0,0 +1,8 @@
+{suites,".",[all_hook_callbacks_SUITE,
+ skip_init_SUITE,
+ skip_req_SUITE,
+ skip_fail_SUITE,
+ skip_group_SUITE,
+ skip_case_SUITE]}.
+{skip_suites,".",all_hook_callbacks_SUITE,"Skipped in spec"}.
+{skip_cases,".",skip_case_SUITE,skip_in_spec,"Skipped in spec"}.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl
new file mode 100644
index 0000000000..436d496c32
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl
@@ -0,0 +1,87 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(skip_case_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+suite() ->
+ [].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(Config) ->
+ ok.
+
+init_per_group(_,Config) ->
+ Config.
+
+end_per_group(_,_) ->
+ ok.
+
+init_per_testcase(skip_in_init,Config) ->
+ {skip,"Skipped in init_per_testcase/2"};
+init_per_testcase(fail_in_init,Config) ->
+ ct:fail("Failed in init_per_testcase/2");
+init_per_testcase(_,Config) ->
+ Config.
+
+end_per_testcase(_,_) ->
+ ok.
+
+all() ->
+ [skip_in_spec,
+ skip_in_init,
+ fail_in_init,
+ skip_in_case,
+ req_auto_skip,
+ fail_auto_skip
+ ].
+
+%% Test cases starts here.
+skip_in_spec(Config) ->
+ ct:fail("This test shall never be run. "
+ "It shall be skipped in the test spec.").
+
+skip_in_init(Config) ->
+ ct:fail("This test shall never be run. "
+ "It shall be skipped in init_per_testcase/2.").
+
+fail_in_init(Config) ->
+ ct:fail("This test shall never be run. "
+ "It shall fail in init_per_testcase/2.").
+
+skip_in_case(Config) ->
+ {skip,"Skipped in test case function"}.
+
+req_auto_skip() ->
+ [{require,whatever}].
+req_auto_skip(Config) ->
+ ct:fail("This test shall never be run due to "
+ "failed require").
+
+fail_auto_skip() ->
+ faulty_return_value.
+fail_auto_skip(Config) ->
+ ct:fail("This test shall never be run due to "
+ "faulty return from info function").
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl
new file mode 100644
index 0000000000..7d8f406763
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl
@@ -0,0 +1,171 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(skip_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+%% Send a cth_error event if a callback is called with unexpected arguments
+-define(fail(Info),
+ gen_event:notify(
+ ?CT_EVMGR_REF,
+ #event{ name = cth_error,
+ node = node(),
+ data = {illegal_hook_callback,{?MODULE,?FUNCTION_NAME,Info}}})).
+
+%% CT Hooks
+-compile(export_all).
+
+id(Opts) ->
+ empty_cth:id(Opts).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+ Suite==skip_init_SUITE
+ orelse Suite==skip_group_SUITE
+ orelse Suite==skip_case_SUITE
+ orelse Suite==seq_SUITE
+ orelse Suite==repeat_SUITE
+ orelse Suite==config_clash_SUITE
+ orelse ?fail(Suite),
+ empty_cth:pre_init_per_suite(Suite,Config,State).
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ Suite==skip_init_SUITE
+ orelse Suite==skip_group_SUITE
+ orelse Suite==skip_case_SUITE
+ orelse Suite==seq_SUITE
+ orelse Suite==repeat_SUITE
+ orelse Suite==config_clash_SUITE
+ orelse ?fail(Suite),
+ empty_cth:post_init_per_suite(Suite,Config,Return,State).
+
+pre_end_per_suite(Suite,Config,State) ->
+ Suite==skip_case_SUITE
+ orelse Suite==skip_group_SUITE
+ orelse Suite==seq_SUITE
+ orelse Suite==repeat_SUITE
+ orelse Suite==config_clash_SUITE
+ orelse ?fail(Suite),
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ Suite==skip_case_SUITE
+ orelse Suite==skip_group_SUITE
+ orelse Suite==seq_SUITE
+ orelse Suite==repeat_SUITE
+ orelse Suite==config_clash_SUITE
+ orelse ?fail(Suite),
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Suite,Group,Config,State) ->
+ (Suite==skip_group_SUITE andalso Group==test_group_3)
+ orelse ?fail({Suite,Group}),
+ empty_cth:pre_init_per_group(Suite,Group,Config,State).
+
+post_init_per_group(Suite,Group,Config,Return,State) ->
+ (Suite==skip_group_SUITE andalso Group==test_group_3)
+ orelse ?fail({Suite,Group}),
+ empty_cth:post_init_per_group(Suite,Group,Config,Return,State).
+
+pre_end_per_group(Suite,Group,Config,State) ->
+ ?fail({Suite,Group}),
+ empty_cth:pre_end_per_group(Suite,Group,Config,State).
+
+post_end_per_group(Suite,Group,Config,Return,State) ->
+ ?fail({Suite,Group}),
+ empty_cth:post_end_per_group(Suite,Group,Config,Return,State).
+
+pre_init_per_testcase(Suite,TC,Config,State) ->
+ (Suite==skip_case_SUITE andalso (TC==skip_in_init
+ orelse TC==fail_in_init
+ orelse TC==skip_in_case))
+ orelse (Suite==seq_SUITE andalso TC==test_case_1)
+ orelse (Suite==repeat_SUITE andalso TC==test_case_1)
+ orelse ?fail({Suite,TC}),
+ empty_cth:pre_init_per_testcase(Suite,TC,Config,State).
+
+post_init_per_testcase(Suite,TC,Config,Return,State) ->
+ (Suite==skip_case_SUITE andalso (TC==skip_in_init
+ orelse TC==fail_in_init
+ orelse TC==skip_in_case))
+ orelse (Suite==seq_SUITE andalso TC==test_case_1)
+ orelse (Suite==repeat_SUITE andalso TC==test_case_1)
+ orelse ?fail({Suite,TC}),
+ empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State).
+
+pre_end_per_testcase(Suite,TC,Config,State) ->
+ (Suite==skip_case_SUITE andalso TC==skip_in_case)
+ orelse (Suite==seq_SUITE andalso TC==test_case_1)
+ orelse (Suite==repeat_SUITE andalso TC==test_case_1)
+ orelse ?fail({Suite,TC}),
+ empty_cth:pre_end_per_testcase(Suite,TC,Config,State).
+
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
+ (Suite==skip_case_SUITE andalso TC==skip_in_case)
+ orelse (Suite==seq_SUITE andalso TC==test_case_1)
+ orelse (Suite==repeat_SUITE andalso TC==test_case_1)
+ orelse ?fail({Suite,TC}),
+ empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State).
+
+on_tc_fail(Suite,TC,Reason,State) ->
+ (Suite==seq_SUITE andalso TC==test_case_1)
+ orelse (Suite==config_clash_SUITE andalso TC==test_case_1)
+ orelse ?fail({Suite,TC}),
+ empty_cth:on_tc_fail(Suite,TC,Reason,State).
+
+on_tc_skip(all_hook_callbacks_SUITE=Suite,all=TC, Reason, State) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State);
+on_tc_skip(Suite,TC,Reason,State)
+ when (Suite==skip_init_SUITE
+ orelse Suite==skip_req_SUITE
+ orelse Suite==skip_fail_SUITE)
+ andalso
+ (TC==init_per_suite
+ orelse TC==test_case
+ orelse TC==end_per_suite) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State);
+on_tc_skip(skip_group_SUITE=Suite,TC={C,G},Reason,State)
+ when (C==init_per_group orelse C==test_case orelse C==end_per_group) andalso
+ (G==test_group_1 orelse G==test_group_2 orelse G==test_group_3) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State);
+on_tc_skip(skip_case_SUITE=Suite,TC,Reason,State)
+ when TC==skip_in_spec;
+ TC==skip_in_init;
+ TC==fail_in_init;
+ TC==skip_in_case;
+ TC==req_auto_skip;
+ TC==fail_auto_skip ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State);
+on_tc_skip(Suite,TC,Reason,State)
+ when (Suite==seq_SUITE andalso TC==test_case_2)
+ orelse (Suite==repeat_SUITE andalso TC==test_case_2) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State);
+on_tc_skip(Suite,TC,Reason,State) ->
+ ?fail({Suite,TC}),
+ empty_cth:on_tc_skip(Suite,TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_fail_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_fail_SUITE.erl
new file mode 100644
index 0000000000..9f5dfee6b9
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_fail_SUITE.erl
@@ -0,0 +1,53 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(skip_fail_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+suite() ->
+ faulty_return_value.
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(Config) ->
+ ok.
+
+init_per_group(_,Config) ->
+ Config.
+
+end_per_group(_,_) ->
+ ok.
+
+init_per_testcase(_,Config) ->
+ Config.
+
+end_per_testcase(_,_) ->
+ ok.
+
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+test_case(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_group_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_group_SUITE.erl
new file mode 100644
index 0000000000..d3b848bfbd
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_group_SUITE.erl
@@ -0,0 +1,64 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(skip_group_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+suite() ->
+ [].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(Config) ->
+ ok.
+
+group(test_group_1) ->
+ [{require,whatever}];
+group(test_group_2) ->
+ faulty_return_value;
+group(_) ->
+ [].
+
+init_per_group(test_group_3,Config) ->
+ {skip,"Skipped in init_per_group/2"};
+init_per_group(_,Config) ->
+ ct:fail("This shall never be run due to auto_skip from group/1").
+
+end_per_group(_,_) ->
+ ct:fail("This shall never be run").
+
+all() ->
+ [{group,test_group_1},
+ {group,test_group_2},
+ {group,test_group_3}].
+
+groups() ->
+ [{test_group_1,[test_case]},
+ {test_group_2,[test_case]},
+ {test_group_3,[test_case]}].
+
+%% Test cases starts here.
+test_case(_Config) ->
+ ct:fail("This test case shall never be run due to skip on group level").
+
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_init_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_init_SUITE.erl
new file mode 100644
index 0000000000..70305421ac
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_init_SUITE.erl
@@ -0,0 +1,53 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(skip_init_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+suite() ->
+ [].
+
+init_per_suite(Config) ->
+ {skip,"Skipped in init_per_suite/1"}.
+
+end_per_suite(Config) ->
+ ok.
+
+init_per_group(_,Config) ->
+ Config.
+
+end_per_group(_,_) ->
+ ok.
+
+init_per_testcase(_,Config) ->
+ Config.
+
+end_per_testcase(_,_) ->
+ ok.
+
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+test_case(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_init_tc_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_init_tc_cth.erl
new file mode 100644
index 0000000000..e1d261d59a
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_init_tc_cth.erl
@@ -0,0 +1,79 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(skip_pre_init_tc_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State).
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State).
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_init_per_group(Suite,Group,Config,State).
+
+post_init_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Suite,Group,Config,Return,State).
+
+pre_end_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_end_per_group(Suite,Group,Config,State).
+
+post_end_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Suite,Group,Config,Return,State).
+
+pre_init_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(Suite,TC,Config,State),
+ {{skip, "Skipped in pre_init_per_testcase"}, State}.
+
+post_init_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State).
+
+pre_end_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_end_per_testcase(Suite,TC,Config,State).
+
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State).
+
+on_tc_fail(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_fail(Suite,TC,Reason,State).
+
+on_tc_skip(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_req_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_req_SUITE.erl
new file mode 100644
index 0000000000..bc69dd5ea4
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_req_SUITE.erl
@@ -0,0 +1,53 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(skip_req_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+suite() ->
+ [{require,whatever}].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(Config) ->
+ ok.
+
+init_per_group(_,Config) ->
+ Config.
+
+end_per_group(_,_) ->
+ ok.
+
+init_per_testcase(_,Config) ->
+ Config.
+
+end_per_testcase(_,_) ->
+ ok.
+
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+test_case(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl
index e926abd885..05a452b99d 100644
--- a/lib/common_test/test/ct_test_support.erl
+++ b/lib/common_test/test/ct_test_support.erl
@@ -765,23 +765,23 @@ locate({parallel,TEvs}, Node, Evs, Config) ->
{Done,RemEvs2,length(RemEvs2)}
end;
%% end_per_group auto- or user skipped
- (TEv={TEH,AutoOrUserSkip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize})
+ (TEv={TEH,AutoOrUserSkip,{M,{end_per_group,G},R}}, {Done,RemEvs,_RemSize})
when AutoOrUserSkip == tc_auto_skip;
AutoOrUserSkip == tc_user_skip ->
RemEvs1 =
lists:dropwhile(
fun({EH,#event{name=tc_auto_skip,
node=EvNode,
- data={Mod,end_per_group,Reason}}}) when
- EH == TEH, EvNode == Node, Mod == M ->
+ data={Mod,{end_per_group,EvGroupName},Reason}}}) when
+ EH == TEH, EvNode == Node, Mod == M, EvGroupName == G ->
case match_data(R, Reason) of
match -> false;
_ -> true
end;
({EH,#event{name=tc_user_skip,
node=EvNode,
- data={Mod,end_per_group,Reason}}}) when
- EH == TEH, EvNode == Node, Mod == M ->
+ data={Mod,{end_per_group,EvGroupName},Reason}}}) when
+ EH == TEH, EvNode == Node, Mod == M, EvGroupName == G ->
case match_data(R, Reason) of
match -> false;
_ -> true
@@ -1008,20 +1008,20 @@ locate({shuffle,TEvs}, Node, Evs, Config) ->
{Done,RemEvs2,length(RemEvs2)}
end;
%% end_per_group auto-or user skipped
- (TEv={TEH,AutoOrUserSkip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize})
+ (TEv={TEH,AutoOrUserSkip,{M,{end_per_group,G},R}}, {Done,RemEvs,_RemSize})
when AutoOrUserSkip == tc_auto_skip;
AutoOrUserSkip == tc_user_skip ->
RemEvs1 =
lists:dropwhile(
fun({EH,#event{name=tc_auto_skip,
node=EvNode,
- data={Mod,end_per_group,Reason}}}) when
- EH == TEH, EvNode == Node, Mod == M, Reason == R ->
+ data={Mod,{end_per_group,EvGroupName},Reason}}}) when
+ EH == TEH, EvNode == Node, Mod == M, EvGroupName == G, Reason == R ->
false;
({EH,#event{name=tc_user_skip,
node=EvNode,
- data={Mod,end_per_group,Reason}}}) when
- EH == TEH, EvNode == Node, Mod == M, Reason == R ->
+ data={Mod,{end_per_group,EvGroupName},Reason}}}) when
+ EH == TEH, EvNode == Node, Mod == M, EvGroupName == G, Reason == R ->
false;
({EH,#event{name=stop_logging,
node=EvNode,data=_}}) when
@@ -1264,10 +1264,10 @@ log_events1([E={_EH,tc_done,{_M,{end_per_group,_GrName,Props},_R}} | Evs], Dev,
io:format(Dev, "~s~p]},~n", [Ind,E]),
log_events1(Evs, Dev, Ind--" ")
end;
-log_events1([E={_EH,tc_auto_skip,{_M,end_per_group,_Reason}} | Evs], Dev, Ind) ->
+log_events1([E={_EH,tc_auto_skip,{_M,{end_per_group,_GrName},_Reason}} | Evs], Dev, Ind) ->
io:format(Dev, "~s~p],~n", [Ind,E]),
log_events1(Evs, Dev, Ind--" ");
-log_events1([E={_EH,tc_user_skip,{_M,end_per_group,_Reason}} | Evs], Dev, Ind) ->
+log_events1([E={_EH,tc_user_skip,{_M,{end_per_group,_GrName},_Reason}} | Evs], Dev, Ind) ->
io:format(Dev, "~s~p],~n", [Ind,E]),
log_events1(Evs, Dev, Ind--" ");
log_events1([E], Dev, Ind) ->