aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test/test
diff options
context:
space:
mode:
authorSiri Hansen <[email protected]>2019-02-15 14:52:40 +0100
committerSiri Hansen <[email protected]>2019-02-15 14:52:40 +0100
commit6fab97b926241abb4980a746b23b3e3f0feac485 (patch)
treecf87be0039a756faad83d44222fcb0bf5f3c2e88 /lib/common_test/test
parent9b93a26e8bda8b5e5bb9cd0a1925d3a2d435cb6c (diff)
parentd35233878fd0b65c563374bd4383a122b51a48ec (diff)
downloadotp-6fab97b926241abb4980a746b23b3e3f0feac485.tar.gz
otp-6fab97b926241abb4980a746b23b3e3f0feac485.tar.bz2
otp-6fab97b926241abb4980a746b23b3e3f0feac485.zip
Merge branch 'siri/ct/post_end_per_testcase-change-status/ERIERL-282/OTP-15584' into maint
* siri/ct/post_end_per_testcase-change-status/ERIERL-282/OTP-15584: [ct] Allow post_end_per_testcase hook to change tc status after timetrap
Diffstat (limited to 'lib/common_test/test')
-rw-r--r--lib/common_test/test/ct_hooks_SUITE.erl106
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_update_result_post_end_tc_SUITE.erl101
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_result_post_end_tc_cth.erl98
3 files changed, 304 insertions, 1 deletions
diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl
index 0f5636a789..44b86b1dfe 100644
--- a/lib/common_test/test/ct_hooks_SUITE.erl
+++ b/lib/common_test/test/ct_hooks_SUITE.erl
@@ -84,7 +84,7 @@ all(suite) ->
fail_post_suite_cth, skip_pre_suite_cth, skip_pre_end_cth,
skip_pre_init_tc_cth,
skip_post_suite_cth, recover_post_suite_cth, update_config_cth,
- state_update_cth, options_cth, same_id_cth,
+ state_update_cth, update_result_cth, options_cth, same_id_cth,
fail_n_skip_with_minimal_cth, prio_cth, no_config,
no_init_suite_config, no_init_config, no_end_config,
failed_sequence, repeat_force_stop, config_clash,
@@ -209,6 +209,10 @@ state_update_cth(Config) when is_list(Config) ->
do_test(state_update_cth, "ct_cth_fail_one_skip_one_SUITE.erl",
[state_update_cth,state_update_cth],Config).
+update_result_cth(Config) ->
+ do_test(update_result_cth, "ct_cth_update_result_post_end_tc_SUITE.erl",
+ [update_result_post_end_tc_cth],Config).
+
options_cth(Config) when is_list(Config) ->
do_test(options_cth, "ct_cth_empty_SUITE.erl",
[{empty_cth,[test]}],Config).
@@ -1099,6 +1103,106 @@ test_events(state_update_cth) ->
{?eh,stop_logging,[]}
];
+test_events(update_result_cth) ->
+ Suite = ct_cth_update_result_post_end_tc_SUITE,
+ [
+ {?eh,start_logging,'_'},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{'_',init,['_',[]]}},
+ {?eh,tc_start,{Suite,init_per_suite}},
+ {?eh,tc_done,{Suite,init_per_suite,ok}},
+
+ {?eh,tc_start,{Suite,tc_ok_to_fail}},
+ {?eh,cth,{'_',post_end_per_testcase,[Suite,tc_ok_to_fail,'_',ok,[]]}},
+ {?eh,tc_done,{Suite,tc_ok_to_fail,{failed,{error,"Test failure"}}}},
+ {?eh,cth,{'_',on_tc_fail,'_'}},
+ {?eh,test_stats,{0,1,{0,0}}},
+
+ {?eh,tc_start,{Suite,tc_ok_to_skip}},
+ {?eh,cth,{'_',post_end_per_testcase,[Suite,tc_ok_to_skip,'_',ok,[]]}},
+ {?eh,tc_done,{Suite,tc_ok_to_skip,{skipped,"Test skipped"}}},
+ {?eh,cth,{'_',on_tc_skip,'_'}},
+ {?eh,test_stats,{0,1,{1,0}}},
+
+ {?eh,tc_start,{Suite,tc_fail_to_ok}},
+ {?eh,cth,{'_',post_end_per_testcase,
+ [Suite,tc_fail_to_ok,'_',
+ {error,{test_case_failed,"should be changed to ok"}},[]]}},
+ {?eh,tc_done,{Suite,tc_fail_to_ok,ok}},
+ {?eh,test_stats,{1,1,{1,0}}},
+
+ {?eh,tc_start,{Suite,tc_fail_to_skip}},
+ {?eh,cth,{'_',post_end_per_testcase,
+ [Suite,tc_fail_to_skip,'_',
+ {error,{test_case_failed,"should be changed to skip"}},[]]}},
+ {?eh,tc_done,{Suite,tc_fail_to_skip,{skipped,"Test skipped"}}},
+ {?eh,cth,{'_',on_tc_skip,'_'}},
+ {?eh,test_stats,{1,1,{2,0}}},
+
+ {?eh,tc_start,{Suite,tc_timetrap_to_ok}},
+ {?eh,cth,{'_',post_end_per_testcase,
+ [Suite,tc_timetrap_to_ok,'_',{timetrap_timeout,3000},[]]}},
+ {?eh,tc_done,{Suite,tc_timetrap_to_ok,ok}},
+ {?eh,test_stats,{2,1,{2,0}}},
+
+ {?eh,tc_start,{Suite,tc_timetrap_to_skip}},
+ {?eh,cth,{'_',post_end_per_testcase,
+ [Suite,tc_timetrap_to_skip,'_',{timetrap_timeout,3000},[]]}},
+ {?eh,tc_done,{Suite,tc_timetrap_to_skip,{skipped,"Test skipped"}}},
+ {?eh,cth,{'_',on_tc_skip,'_'}},
+ {?eh,test_stats,{2,1,{3,0}}},
+
+ {?eh,tc_start,{Suite,tc_skip_to_fail}},
+ {?eh,cth,{'_',post_end_per_testcase,
+ [Suite,tc_skip_to_fail,'_',
+ {skip,"should be changed to fail"},[]]}},
+ {?eh,tc_done,{Suite,tc_skip_to_fail,{failed,{error,"Test failure"}}}},
+ {?eh,cth,{'_',on_tc_fail,'_'}},
+ {?eh,test_stats,{2,2,{3,0}}},
+
+ {?eh,tc_start,{Suite,end_fail_to_fail}},
+ {?eh,cth,{'_',post_end_per_testcase,
+ [Suite,end_fail_to_fail,'_',
+ {failed,
+ {Suite,end_per_testcase,
+ {'EXIT',{test_case_failed,"change result when end fails"}}}},[]]}},
+ {?eh,tc_done,{Suite,end_fail_to_fail,{failed,{error,"Test failure"}}}},
+ {?eh,cth,{'_',on_tc_fail,'_'}},
+ {?eh,test_stats,{2,3,{3,0}}},
+
+ {?eh,tc_start,{Suite,end_fail_to_skip}},
+ {?eh,cth,{'_',post_end_per_testcase,
+ [Suite,end_fail_to_skip,'_',
+ {failed,
+ {Suite,end_per_testcase,
+ {'EXIT',{test_case_failed,"change result when end fails"}}}},[]]}},
+ {?eh,tc_done,{Suite,end_fail_to_skip,{skipped,"Test skipped"}}},
+ {?eh,cth,{'_',on_tc_skip,'_'}},
+ {?eh,test_stats,{2,3,{4,0}}},
+
+ {?eh,tc_start,{Suite,end_timetrap_to_fail}},
+ {?eh,cth,{'_',post_end_per_testcase,
+ [Suite,end_timetrap_to_fail,'_',
+ {failed,{Suite,end_per_testcase,{timetrap_timeout,3000}}},[]]}},
+ {?eh,tc_done,{Suite,end_timetrap_to_fail,{failed,{error,"Test failure"}}}},
+ {?eh,cth,{'_',on_tc_fail,'_'}},
+ {?eh,test_stats,{2,4,{4,0}}},
+
+ {?eh,tc_start,{Suite,end_timetrap_to_skip}},
+ {?eh,cth,{'_',post_end_per_testcase,
+ [Suite,end_timetrap_to_skip,'_',
+ {failed,{Suite,end_per_testcase,{timetrap_timeout,3000}}},[]]}},
+ {?eh,tc_done,{Suite,end_timetrap_to_skip,{skipped,"Test skipped"}}},
+ {?eh,cth,{'_',on_tc_skip,'_'}},
+ {?eh,test_stats,{2,4,{5,0}}},
+
+ {?eh,tc_start,{Suite,end_per_suite}},
+ {?eh,tc_done,{Suite,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{'_',terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ];
+
test_events(options_cth) ->
[
{?eh,start_logging,{'DEF','RUNDIR'}},
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_update_result_post_end_tc_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_update_result_post_end_tc_SUITE.erl
new file mode 100644
index 0000000000..a16138ce6f
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_update_result_post_end_tc_SUITE.erl
@@ -0,0 +1,101 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2017. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_cth_update_result_post_end_tc_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+suite() ->
+ [{timetrap,{seconds,3}}].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(Config) ->
+ ok.
+
+init_per_group(_,Config) ->
+ Config.
+
+end_per_group(_,_) ->
+ ok.
+
+init_per_testcase(_,Config) ->
+ Config.
+
+end_per_testcase(EndTimetrap,_) when EndTimetrap==end_timetrap_to_fail;
+ EndTimetrap==end_timetrap_to_skip->
+ timer:sleep(10000);
+end_per_testcase(EndFail,_) when EndFail==end_fail_to_fail;
+ EndFail==end_fail_to_skip->
+ ct:fail("change result when end fails");
+end_per_testcase(_,_) ->
+ ok.
+
+all() ->
+ [tc_ok_to_fail,
+ tc_ok_to_skip,
+ tc_fail_to_ok,
+ tc_fail_to_skip,
+ tc_timetrap_to_ok,
+ tc_timetrap_to_skip,
+ tc_skip_to_fail,
+ end_fail_to_fail,
+ end_fail_to_skip,
+ end_timetrap_to_fail,
+ end_timetrap_to_skip].
+
+%% Test cases starts here.
+tc_ok_to_fail(_Config) ->
+ ok.
+
+tc_ok_to_skip(_Config) ->
+ ok.
+
+tc_fail_to_ok(_Config) ->
+ ct:fail("should be changed to ok").
+
+tc_fail_to_skip(_Config) ->
+ ct:fail("should be changed to skip").
+
+tc_timetrap_to_ok(_Config) ->
+ timer:sleep(10000), % will time out after 3 sek
+ ok.
+
+tc_timetrap_to_skip(_Config) ->
+ timer:sleep(10000), % will time out after 3 sek
+ ok.
+
+tc_skip_to_fail(_Config) ->
+ {skip,"should be changed to fail"}.
+
+end_fail_to_fail(_Config) ->
+ ok.
+
+end_fail_to_skip(_Config) ->
+ ok.
+
+end_timetrap_to_fail(_Config) ->
+ ok.
+
+end_timetrap_to_skip(_Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_result_post_end_tc_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_result_post_end_tc_cth.erl
new file mode 100644
index 0000000000..7afb3d8781
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_result_post_end_tc_cth.erl
@@ -0,0 +1,98 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2017. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(update_result_post_end_tc_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State).
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State).
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_init_per_group(Suite,Group,Config,State).
+
+post_init_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Suite,Group,Config,Return,State).
+
+pre_end_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_end_per_group(Suite,Group,Config,State).
+
+post_end_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Suite,Group,Config,Return,State).
+
+pre_init_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(Suite,TC,Config,State).
+
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State),
+ change_result(TC,Config,State).
+
+on_tc_fail(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_fail(Suite,TC,Reason,State).
+
+on_tc_skip(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
+
+%%%-----------------------------------------------------------------
+%%%
+change_result(tc_ok_to_fail,_Config,State) ->
+ {{fail, "Test failure"}, State};
+change_result(tc_ok_to_skip,_Config,State) ->
+ {{skip, "Test skipped"}, State};
+change_result(tc_fail_to_ok,Config,State) ->
+ {lists:keydelete(tc_status,1,Config),State};
+change_result(tc_fail_to_skip,Config,State) ->
+ {{skip,"Test skipped"},State};
+change_result(tc_timetrap_to_ok,Config,State) ->
+ {lists:keydelete(tc_status,1,Config),State};
+change_result(tc_timetrap_to_skip,Config,State) ->
+ {{skip,"Test skipped"},State};
+change_result(tc_skip_to_fail,_Config,State) ->
+ {{fail, "Test failure"}, State};
+change_result(end_fail_to_fail,_Config,State) ->
+ {{fail, "Test failure"}, State};
+change_result(end_fail_to_skip,_Config,State) ->
+ {{skip, "Test skipped"}, State};
+change_result(end_timetrap_to_fail,_Config,State) ->
+ {{fail, "Test failure"}, State};
+change_result(end_timetrap_to_skip,_Config,State) ->
+ {{skip, "Test skipped"}, State}.