Merge branch 'siri/ct/fail-after-failed-init_per_testcase/maint-21/ERIERL-350/OTP-15869' into maint-21

* siri/ct/fail-after-failed-init_per_testcase/maint-21/ERIERL-350/OTP-15869:
  [ct] Allow post_init_per_testcase to change skip to fail

4 files changed, 186 insertions, 8 deletions

diff --git a/lib/common_test/src/test_server.erl b/lib/common_test/src/test_server.erl
index 756cd4d692..588396f101 100644
--- a/lib/common_test/src/test_server.erl
+++ b/lib/common_test/src/test_server.erl
@@ -1364,23 +1364,29 @@ do_end_tc_call(Mod, IPTC={init_per_testcase,Func}, Res, Return) ->
 	 {NOk,_} when NOk == auto_skip; NOk == fail;
 		      NOk == skip ; NOk == skipped ->
 	     {_,Args} = Res,
-	     IPTCEndRes =
+	     {NewConfig,IPTCEndRes} =
 		 case do_end_tc_call1(Mod, IPTC, Res, Return) of
 		     IPTCEndConfig when is_list(IPTCEndConfig) ->
-			 IPTCEndConfig;
+			 {IPTCEndConfig,IPTCEndConfig};
+                     {failed,RetReason} when Return=:={fail,RetReason} ->
+                         %% Fail reason not changed by framework or hook
+                         {Args,Return};
+                     {SF,_} = IPTCEndResult when SF=:=skip; SF=:=skipped;
+                                                 SF=:=fail; SF=:=failed ->
+                         {Args,IPTCEndResult};
 		     _ ->
-			 Args
+			 {Args,Return}
 		 end,
 	     EPTCInitRes =
 		 case do_init_tc_call(Mod,{end_per_testcase_not_run,Func},
-				      IPTCEndRes,Return) of
+				      NewConfig,IPTCEndRes) of
 		     {ok,EPTCInitConfig} when is_list(EPTCInitConfig) ->
-			 {Return,EPTCInitConfig};
+			 {IPTCEndRes,EPTCInitConfig};
 		     _ ->
-                         {Return,IPTCEndRes}
+                         {IPTCEndRes,NewConfig}
 		 end,
 	     do_end_tc_call1(Mod, {end_per_testcase_not_run,Func},
-			     EPTCInitRes, Return);
+			     EPTCInitRes, IPTCEndRes);
 	 _Ok ->
 	     do_end_tc_call1(Mod, IPTC, Res, Return)
      end;
diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl
index 08c18d91e2..b87464f5e4 100644
--- a/lib/common_test/test/ct_hooks_SUITE.erl
+++ b/lib/common_test/test/ct_hooks_SUITE.erl
@@ -86,7 +86,7 @@ all(suite) ->
        scope_suite_state_cth,
        fail_pre_suite_cth, double_fail_pre_suite_cth,
        fail_post_suite_cth, skip_pre_suite_cth, skip_pre_end_cth,
-       skip_pre_init_tc_cth,
+       skip_pre_init_tc_cth, fail_post_init_tc_cth,
        skip_post_suite_cth, recover_post_suite_cth, update_config_cth,
        state_update_cth, update_result_cth, options_cth, same_id_cth,
        fail_n_skip_with_minimal_cth, prio_cth, no_config,
@@ -206,6 +206,10 @@ skip_pre_init_tc_cth(Config) ->
     do_test(skip_pre_init_tc_cth, "ct_cth_empty_SUITE.erl",
 	    [skip_pre_init_tc_cth],Config).
 
+fail_post_init_tc_cth(Config) ->
+    do_test(fail_post_init_tc_cth, "ct_fail_init_tc_SUITE.erl",
+	    [fail_post_init_tc_cth],Config).
+
 recover_post_suite_cth(Config) when is_list(Config) ->
     do_test(recover_post_suite_cth, "ct_cth_fail_per_suite_SUITE.erl",
 	    [recover_post_suite_cth],Config).
@@ -1042,6 +1046,44 @@ test_events(skip_pre_init_tc_cth) ->
      {?eh,stop_logging,[]}
     ];
 
+test_events(fail_post_init_tc_cth) ->
+    [
+     {?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+     {?eh,cth,{empty_cth,init,['_',[]]}},
+     {?eh,start_info,{1,1,1}},
+     {?eh,tc_start,{ct_fail_init_tc_SUITE,init_per_suite}},
+     {?eh,cth,{empty_cth,pre_init_per_suite,[ct_fail_init_tc_SUITE,'$proplist',[]]}},
+     {?eh,cth,{empty_cth,post_init_per_suite,
+               [ct_fail_init_tc_SUITE,'$proplist','$proplist',[]]}},
+     {?eh,tc_done,{ct_fail_init_tc_SUITE,init_per_suite,ok}},
+     {?eh,tc_start,{ct_fail_init_tc_SUITE,test_case}},
+     {?eh,cth,{empty_cth,pre_init_per_testcase,
+               [ct_fail_init_tc_SUITE,test_case,'$proplist',[]]}},
+     {?eh,cth,{empty_cth,post_init_per_testcase,
+               [ct_fail_init_tc_SUITE,test_case,'$proplist',
+                {skip,
+                 {failed,
+                  {ct_fail_init_tc_SUITE,init_per_testcase,
+                   {{test_case_failed,"Failed in init_per_testcase"},'_'}}}},
+                []]}},
+     {?eh,tc_done,{ct_fail_init_tc_SUITE,test_case,
+                   {failed,"Changed skip to fail in post_init_per_testcase"}}},
+     {?eh,cth,{empty_cth,on_tc_fail,
+               [ct_fail_init_tc_SUITE,test_case,
+                "Changed skip to fail in post_init_per_testcase",
+                []]}},
+     {?eh,test_stats,{0,1,{0,0}}},
+     {?eh,tc_start,{ct_fail_init_tc_SUITE,end_per_suite}},
+     {?eh,cth,{empty_cth,pre_end_per_suite,[ct_fail_init_tc_SUITE,'$proplist',[]]}},
+     {?eh,cth,{empty_cth,post_end_per_suite,
+               [ct_fail_init_tc_SUITE,'$proplist',ok,[]]}},
+     {?eh,tc_done,{ct_fail_init_tc_SUITE,end_per_suite,ok}},
+     {?eh,test_done,{'DEF','STOP_TIME'}},
+     {?eh,cth,{empty_cth,terminate,[[]]}},
+     {?eh,stop_logging,[]}
+    ];
+
 test_events(recover_post_suite_cth) ->
     Suite = ct_cth_fail_per_suite_SUITE,
     [
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_fail_init_tc_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_fail_init_tc_SUITE.erl
new file mode 100644
index 0000000000..96ddfc5782
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_fail_init_tc_SUITE.erl
@@ -0,0 +1,49 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_fail_init_tc_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_testcase(TestCase, _Config) ->
+    ct:fail("Failed in init_per_testcase").
+
+end_per_testcase(_TestCase, _Config) ->
+    ok.
+
+all() ->
+    [test_case].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+    ok.
+
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_init_tc_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_init_tc_cth.erl
new file mode 100644
index 0000000000..ca9f05c40f
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_init_tc_cth.erl
@@ -0,0 +1,81 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2017. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(fail_post_init_tc_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+    empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+    empty_cth:pre_init_per_suite(Suite,Config,State).
+
+post_init_per_suite(Suite,Config,Return,State) ->
+    empty_cth:post_init_per_suite(Suite,Config,Return,State).
+
+pre_end_per_suite(Suite,Config,State) ->
+    empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+    empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Suite,Group,Config,State) ->
+    empty_cth:pre_init_per_group(Suite,Group,Config,State).
+
+post_init_per_group(Suite,Group,Config,Return,State) ->
+    empty_cth:post_init_per_group(Suite,Group,Config,Return,State).
+
+pre_end_per_group(Suite,Group,Config,State) ->
+    empty_cth:pre_end_per_group(Suite,Group,Config,State).
+
+post_end_per_group(Suite,Group,Config,Return,State) ->
+    empty_cth:post_end_per_group(Suite,Group,Config,Return,State).
+
+pre_init_per_testcase(Suite,TC,Config,State) ->
+    empty_cth:pre_init_per_testcase(Suite,TC,Config,State).
+
+post_init_per_testcase(Suite,TC,Config,{skip,_}=Return,State) ->
+    empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State),
+    {{fail,"Changed skip to fail in post_init_per_testcase"},State};
+post_init_per_testcase(Suite,TC,Config,Return,State) ->
+    empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State).
+
+pre_end_per_testcase(Suite,TC,Config,State) ->
+    empty_cth:pre_end_per_testcase(Suite,TC,Config,State).
+
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
+    empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State).
+
+on_tc_fail(Suite,TC, Reason, State) ->
+    empty_cth:on_tc_fail(Suite,TC,Reason,State).
+
+on_tc_skip(Suite,TC, Reason, State) ->
+    empty_cth:on_tc_skip(Suite,TC,Reason,State).
+
+terminate(State) ->
+    empty_cth:terminate(State).