Merge branch 'siri/ct/post_end_per_testcase-change-status/ERIERL-282/OTP-15584' into maint

* siri/ct/post_end_per_testcase-change-status/ERIERL-282/OTP-15584:
  [ct] Allow post_end_per_testcase hook to change tc status after timetrap

3 files changed, 304 insertions, 1 deletions

diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl
index 0f5636a789..44b86b1dfe 100644
--- a/lib/common_test/test/ct_hooks_SUITE.erl
+++ b/lib/common_test/test/ct_hooks_SUITE.erl
@@ -84,7 +84,7 @@ all(suite) ->
        fail_post_suite_cth, skip_pre_suite_cth, skip_pre_end_cth,
        skip_pre_init_tc_cth,
        skip_post_suite_cth, recover_post_suite_cth, update_config_cth,
-       state_update_cth, options_cth, same_id_cth,
+       state_update_cth, update_result_cth, options_cth, same_id_cth,
        fail_n_skip_with_minimal_cth, prio_cth, no_config,
        no_init_suite_config, no_init_config, no_end_config,
        failed_sequence, repeat_force_stop, config_clash,
@@ -209,6 +209,10 @@ state_update_cth(Config) when is_list(Config) ->
     do_test(state_update_cth, "ct_cth_fail_one_skip_one_SUITE.erl",
 	    [state_update_cth,state_update_cth],Config).
 
+update_result_cth(Config) ->
+    do_test(update_result_cth, "ct_cth_update_result_post_end_tc_SUITE.erl",
+            [update_result_post_end_tc_cth],Config).
+
 options_cth(Config) when is_list(Config) ->
     do_test(options_cth, "ct_cth_empty_SUITE.erl",
 	    [{empty_cth,[test]}],Config).
@@ -1099,6 +1103,106 @@ test_events(state_update_cth) ->
      {?eh,stop_logging,[]}
     ];
 
+test_events(update_result_cth) ->
+    Suite = ct_cth_update_result_post_end_tc_SUITE,
+    [
+     {?eh,start_logging,'_'},
+     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+     {?eh,cth,{'_',init,['_',[]]}},
+     {?eh,tc_start,{Suite,init_per_suite}},
+     {?eh,tc_done,{Suite,init_per_suite,ok}},
+
+     {?eh,tc_start,{Suite,tc_ok_to_fail}},
+     {?eh,cth,{'_',post_end_per_testcase,[Suite,tc_ok_to_fail,'_',ok,[]]}},
+     {?eh,tc_done,{Suite,tc_ok_to_fail,{failed,{error,"Test failure"}}}},
+     {?eh,cth,{'_',on_tc_fail,'_'}},
+     {?eh,test_stats,{0,1,{0,0}}},
+
+     {?eh,tc_start,{Suite,tc_ok_to_skip}},
+     {?eh,cth,{'_',post_end_per_testcase,[Suite,tc_ok_to_skip,'_',ok,[]]}},
+     {?eh,tc_done,{Suite,tc_ok_to_skip,{skipped,"Test skipped"}}},
+     {?eh,cth,{'_',on_tc_skip,'_'}},
+     {?eh,test_stats,{0,1,{1,0}}},
+
+     {?eh,tc_start,{Suite,tc_fail_to_ok}},
+     {?eh,cth,{'_',post_end_per_testcase,
+               [Suite,tc_fail_to_ok,'_',
+                {error,{test_case_failed,"should be changed to ok"}},[]]}},
+     {?eh,tc_done,{Suite,tc_fail_to_ok,ok}},
+     {?eh,test_stats,{1,1,{1,0}}},
+
+     {?eh,tc_start,{Suite,tc_fail_to_skip}},
+     {?eh,cth,{'_',post_end_per_testcase,
+               [Suite,tc_fail_to_skip,'_',
+                {error,{test_case_failed,"should be changed to skip"}},[]]}},
+     {?eh,tc_done,{Suite,tc_fail_to_skip,{skipped,"Test skipped"}}},
+     {?eh,cth,{'_',on_tc_skip,'_'}},
+     {?eh,test_stats,{1,1,{2,0}}},
+
+     {?eh,tc_start,{Suite,tc_timetrap_to_ok}},
+     {?eh,cth,{'_',post_end_per_testcase,
+               [Suite,tc_timetrap_to_ok,'_',{timetrap_timeout,3000},[]]}},
+     {?eh,tc_done,{Suite,tc_timetrap_to_ok,ok}},
+     {?eh,test_stats,{2,1,{2,0}}},
+
+     {?eh,tc_start,{Suite,tc_timetrap_to_skip}},
+     {?eh,cth,{'_',post_end_per_testcase,
+               [Suite,tc_timetrap_to_skip,'_',{timetrap_timeout,3000},[]]}},
+     {?eh,tc_done,{Suite,tc_timetrap_to_skip,{skipped,"Test skipped"}}},
+     {?eh,cth,{'_',on_tc_skip,'_'}},
+     {?eh,test_stats,{2,1,{3,0}}},
+
+     {?eh,tc_start,{Suite,tc_skip_to_fail}},
+     {?eh,cth,{'_',post_end_per_testcase,
+               [Suite,tc_skip_to_fail,'_',
+                {skip,"should be changed to fail"},[]]}},
+     {?eh,tc_done,{Suite,tc_skip_to_fail,{failed,{error,"Test failure"}}}},
+     {?eh,cth,{'_',on_tc_fail,'_'}},
+     {?eh,test_stats,{2,2,{3,0}}},
+
+     {?eh,tc_start,{Suite,end_fail_to_fail}},
+     {?eh,cth,{'_',post_end_per_testcase,
+               [Suite,end_fail_to_fail,'_',
+                {failed,
+                 {Suite,end_per_testcase,
+                  {'EXIT',{test_case_failed,"change result when end fails"}}}},[]]}},
+     {?eh,tc_done,{Suite,end_fail_to_fail,{failed,{error,"Test failure"}}}},
+     {?eh,cth,{'_',on_tc_fail,'_'}},
+     {?eh,test_stats,{2,3,{3,0}}},
+
+     {?eh,tc_start,{Suite,end_fail_to_skip}},
+     {?eh,cth,{'_',post_end_per_testcase,
+               [Suite,end_fail_to_skip,'_',
+                {failed,
+                 {Suite,end_per_testcase,
+                  {'EXIT',{test_case_failed,"change result when end fails"}}}},[]]}},
+     {?eh,tc_done,{Suite,end_fail_to_skip,{skipped,"Test skipped"}}},
+     {?eh,cth,{'_',on_tc_skip,'_'}},
+     {?eh,test_stats,{2,3,{4,0}}},
+
+     {?eh,tc_start,{Suite,end_timetrap_to_fail}},
+     {?eh,cth,{'_',post_end_per_testcase,
+               [Suite,end_timetrap_to_fail,'_',
+                {failed,{Suite,end_per_testcase,{timetrap_timeout,3000}}},[]]}},
+     {?eh,tc_done,{Suite,end_timetrap_to_fail,{failed,{error,"Test failure"}}}},
+     {?eh,cth,{'_',on_tc_fail,'_'}},
+     {?eh,test_stats,{2,4,{4,0}}},
+
+     {?eh,tc_start,{Suite,end_timetrap_to_skip}},
+     {?eh,cth,{'_',post_end_per_testcase,
+               [Suite,end_timetrap_to_skip,'_',
+                {failed,{Suite,end_per_testcase,{timetrap_timeout,3000}}},[]]}},
+     {?eh,tc_done,{Suite,end_timetrap_to_skip,{skipped,"Test skipped"}}},
+     {?eh,cth,{'_',on_tc_skip,'_'}},
+     {?eh,test_stats,{2,4,{5,0}}},
+
+     {?eh,tc_start,{Suite,end_per_suite}},
+     {?eh,tc_done,{Suite,end_per_suite,ok}},
+     {?eh,test_done,{'DEF','STOP_TIME'}},
+     {?eh,cth,{'_',terminate,[[]]}},
+     {?eh,stop_logging,[]}
+    ];
+
 test_events(options_cth) ->
     [
      {?eh,start_logging,{'DEF','RUNDIR'}},
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_update_result_post_end_tc_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_update_result_post_end_tc_SUITE.erl
new file mode 100644
index 0000000000..a16138ce6f
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_update_result_post_end_tc_SUITE.erl
@@ -0,0 +1,101 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2017. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_cth_update_result_post_end_tc_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+suite() ->
+    [{timetrap,{seconds,3}}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(Config) ->
+    ok.
+
+init_per_group(_,Config) ->
+    Config.
+
+end_per_group(_,_) ->
+    ok.
+
+init_per_testcase(_,Config) ->
+    Config.
+
+end_per_testcase(EndTimetrap,_) when EndTimetrap==end_timetrap_to_fail;
+                                     EndTimetrap==end_timetrap_to_skip->
+    timer:sleep(10000);
+end_per_testcase(EndFail,_) when EndFail==end_fail_to_fail;
+                                 EndFail==end_fail_to_skip->
+    ct:fail("change result when end fails");
+end_per_testcase(_,_) ->
+    ok.
+
+all() ->
+    [tc_ok_to_fail,
+     tc_ok_to_skip,
+     tc_fail_to_ok,
+     tc_fail_to_skip,
+     tc_timetrap_to_ok,
+     tc_timetrap_to_skip,
+     tc_skip_to_fail,
+     end_fail_to_fail,
+     end_fail_to_skip,
+     end_timetrap_to_fail,
+     end_timetrap_to_skip].
+
+%% Test cases starts here.
+tc_ok_to_fail(_Config) ->
+    ok.
+
+tc_ok_to_skip(_Config) ->
+    ok.
+
+tc_fail_to_ok(_Config) ->
+    ct:fail("should be changed to ok").
+
+tc_fail_to_skip(_Config) ->
+    ct:fail("should be changed to skip").
+
+tc_timetrap_to_ok(_Config) ->
+    timer:sleep(10000), % will time out after 3 sek
+    ok.
+
+tc_timetrap_to_skip(_Config) ->
+    timer:sleep(10000), % will time out after 3 sek
+    ok.
+
+tc_skip_to_fail(_Config) ->
+    {skip,"should be changed to fail"}.
+
+end_fail_to_fail(_Config) ->
+    ok.
+
+end_fail_to_skip(_Config) ->
+    ok.
+
+end_timetrap_to_fail(_Config) ->
+    ok.
+
+end_timetrap_to_skip(_Config) ->
+    ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_result_post_end_tc_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_result_post_end_tc_cth.erl
new file mode 100644
index 0000000000..7afb3d8781
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_result_post_end_tc_cth.erl
@@ -0,0 +1,98 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2017. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(update_result_post_end_tc_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+    empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+    empty_cth:pre_init_per_suite(Suite,Config,State).
+
+post_init_per_suite(Suite,Config,Return,State) ->
+    empty_cth:post_init_per_suite(Suite,Config,Return,State).
+
+pre_end_per_suite(Suite,Config,State) ->
+    empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+    empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Suite,Group,Config,State) ->
+    empty_cth:pre_init_per_group(Suite,Group,Config,State).
+
+post_init_per_group(Suite,Group,Config,Return,State) ->
+    empty_cth:post_init_per_group(Suite,Group,Config,Return,State).
+
+pre_end_per_group(Suite,Group,Config,State) ->
+    empty_cth:pre_end_per_group(Suite,Group,Config,State).
+
+post_end_per_group(Suite,Group,Config,Return,State) ->
+    empty_cth:post_end_per_group(Suite,Group,Config,Return,State).
+
+pre_init_per_testcase(Suite,TC,Config,State) ->
+    empty_cth:pre_init_per_testcase(Suite,TC,Config,State).
+
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
+    empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State),
+    change_result(TC,Config,State).
+
+on_tc_fail(Suite,TC, Reason, State) ->
+    empty_cth:on_tc_fail(Suite,TC,Reason,State).
+
+on_tc_skip(Suite,TC, Reason, State) ->
+    empty_cth:on_tc_skip(Suite,TC,Reason,State).
+
+terminate(State) ->
+    empty_cth:terminate(State).
+
+%%%-----------------------------------------------------------------
+%%% 
+change_result(tc_ok_to_fail,_Config,State) ->
+    {{fail, "Test failure"}, State};
+change_result(tc_ok_to_skip,_Config,State) ->
+    {{skip, "Test skipped"}, State};
+change_result(tc_fail_to_ok,Config,State) ->
+    {lists:keydelete(tc_status,1,Config),State};
+change_result(tc_fail_to_skip,Config,State) ->
+    {{skip,"Test skipped"},State};
+change_result(tc_timetrap_to_ok,Config,State) ->
+    {lists:keydelete(tc_status,1,Config),State};
+change_result(tc_timetrap_to_skip,Config,State) ->
+    {{skip,"Test skipped"},State};
+change_result(tc_skip_to_fail,_Config,State) ->
+    {{fail, "Test failure"}, State};
+change_result(end_fail_to_fail,_Config,State) ->
+    {{fail, "Test failure"}, State};
+change_result(end_fail_to_skip,_Config,State) ->
+    {{skip, "Test skipped"}, State};
+change_result(end_timetrap_to_fail,_Config,State) ->
+    {{fail, "Test failure"}, State};
+change_result(end_timetrap_to_skip,_Config,State) ->
+    {{skip, "Test skipped"}, State}.