Merge branch 'siri/ct_hooks/callbacks-on-skip/OTP-10599' into maint

* siri/ct_hooks/callbacks-on-skip/OTP-10599:
  [ct] Update built-in ct hooks with new Suite parameter
  [cth_surefire] Handle skips from test spec
  [ct] Remove excessive skipped/failed tag in hook function
  [ct] Send tc_start event on force_stop and failed sequence
  [ct] Correctly handle process died in init and end_per_testcase
  [ct] Fix faulty hook callbacks for skipped tests
  Add dummy end_per_suite/1
  [ct] Fix hooks and fail when one of init/end_per_* does not exit
  [ct] Add Suite argument to hook callback functions
  [ct] Fix function_clause in ct_framework when hook function crashes

4 files changed, 134 insertions, 58 deletions

diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml
index 48ffe653e4..d407a0a53f 100644
--- a/lib/common_test/doc/src/common_test_app.xml
+++ b/lib/common_test/doc/src/common_test_app.xml
@@ -224,7 +224,9 @@
 	</type>
 	<desc>	
 
-	  <p>OPTIONAL</p>
+	  <p>OPTIONAL; if this function is defined, then <seealso
+	  marker="#Module:end_per_suite-1"><c>end_per_suite/1</c></seealso>
+	  must also be defined.</p>
 
 	  <p>This configuration function is called as the first function in the
 	  suite. It typically contains initializations that are common for
@@ -256,7 +258,9 @@
       </type>
 
       <desc>	
-	<p>OPTIONAL</p>
+	<p>OPTIONAL; if this function is defined, then <seealso
+	  marker="#Module:init_per_suite-1"><c>init_per_suite/1</c></seealso>
+	  must also be defined.</p>
 	
 	<p>This function is called as the last test case in the
 	  suite. It is meant to be used for cleaning up after
@@ -360,7 +364,9 @@
 	</type>
 	<desc>	
 
-	  <p>OPTIONAL</p>
+	  <p>OPTIONAL; if this function is defined, then <seealso
+	    marker="#Module:end_per_group-2"><c>end_per_group/2</c></seealso>
+	    must also be defined.</p>
 
 	  <p>This configuration function is called before execution of a
 	    test case group. It typically contains initializations that are 
@@ -396,7 +402,9 @@
 	</type>
 
 	<desc>	
-	  <p>OPTIONAL</p>
+	  <p>OPTIONAL; if this function is defined, then <seealso
+	    marker="#Module:init_per_group-2"><c>init_per_group/2</c></seealso>
+	    must also be defined.</p>
 
 	  <p>This function is called after the execution of a test case group
 	    is finished. It is meant to be used for cleaning up after
@@ -427,7 +435,10 @@
 	</type>
 	<desc>
 
-	  <p>OPTIONAL</p>
+	  <p>OPTIONAL; if this function is defined,
+	    then <seealso marker="#Module:end_per_testcase-2">
+	      <c>end_per_testcase/2</c></seealso> must also be
+	    defined.</p>
 
 	  <p>This function is called before each test case. Argument
 	  <c>TestCase</c> is the test case name, and
@@ -454,7 +465,10 @@
 	</type>
 	<desc>	
 
-	  <p>OPTIONAL</p>
+	  <p>OPTIONAL; if this function is defined,
+	    then <seealso marker="#Module:init_per_testcase-2">
+	      <c>init_per_testcase/2</c></seealso> must also be
+	    defined.</p>
 
 	  <p>This function is called after each test case, and can be used
 	    to clean up after
diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml
index c2cf29c530..a085f30262 100644
--- a/lib/common_test/doc/src/ct_hooks.xml
+++ b/lib/common_test/doc/src/ct_hooks.xml
@@ -208,9 +208,10 @@
     </func>
 
     <func>
-      <name>Module:pre_init_per_group(GroupName, InitData, CTHState) -&gt; Result</name>
+      <name>Module:pre_init_per_group(SuiteName, GroupName, InitData, CTHState) -&gt; Result</name>
       <fsummary>Called before init_per_group.</fsummary>
       <type>
+        <v>SuiteName = atom()</v>
         <v>GroupName = atom()</v>
         <v>InitData = Config | SkipOrFail</v>
         <v>Config = NewConfig = [{Key,Value}]</v>
@@ -231,13 +232,19 @@
           but for function
           <seealso marker="common_test#Module:init_per_group-2"><c>init_per_group</c></seealso>
           instead.</p>
+
+	<p>If <c>Module:pre_init_per_group/4</c> is not exported, common_test
+	  will attempt to call <c>Module:pre_init_per_group(GroupName,
+	  InitData, CTHState)</c> instead. This is for backwards
+	  compatibility.</p>
       </desc>
     </func>
 
     <func>
-      <name>Module:post_init_per_group(GroupName, Config, Return, CTHState) -&gt; Result</name>
+      <name>Module:post_init_per_group(SuiteName, GroupName, Config, Return, CTHState) -&gt; Result</name>
       <fsummary>Called after init_per_group.</fsummary>
       <type>
+        <v>SuiteName = atom()</v>
         <v>GroupName = atom()</v>
         <v>Config = [{Key,Value}]</v>
         <v>Return = NewReturn = Config | SkipOrFail | term()</v>
@@ -258,13 +265,19 @@
           but for function
           <seealso marker="common_test#Module:init_per_group-2"><c>init_per_group</c></seealso>
           instead.</p>
+
+	<p>If <c>Module:post_init_per_group/5</c> is not exported, common_test
+	  will attempt to call <c>Module:post_init_per_group(GroupName,
+	  Config, Return, CTHState)</c> instead. This is for backwards
+	  compatibility.</p>
       </desc>
     </func>
 
     <func>
-      <name>Module:pre_init_per_testcase(TestcaseName, InitData, CTHState) -&gt; Result</name>
+      <name>Module:pre_init_per_testcase(SuiteName, TestcaseName, InitData, CTHState) -&gt; Result</name>
       <fsummary>Called before init_per_testcase.</fsummary>
       <type>
+        <v>SuiteName = atom()</v>
         <v>TestcaseName = atom()</v>
         <v>InitData = Config | SkipOrFail</v>
         <v>Config = NewConfig = [{Key,Value}]</v>
@@ -286,6 +299,11 @@
           <seealso marker="common_test#Module:init_per_testcase-2"><c>init_per_testcase</c></seealso>
           instead.</p>
 
+	<p>If <c>Module:pre_init_per_testcase/4</c> is not exported, common_test
+	  will attempt to call <c>Module:pre_init_per_testcase(TestcaseName,
+	  InitData, CTHState)</c> instead. This is for backwards
+	  compatibility.</p>
+
         <p>CTHs cannot be added here right now. That feature may be added in
           a later release, but it would right now break backwards
           compatibility.</p>
@@ -293,9 +311,10 @@
     </func>
 
     <func>
-      <name>Module:post_init_per_testcase(TestcaseName, Config, Return, CTHState) -&gt; Result</name>
+      <name>Module:post_init_per_testcase(SuiteName, TestcaseName, Config, Return, CTHState) -&gt; Result</name>
       <fsummary>Called after init_per_testcase.</fsummary>
       <type>
+        <v>SuiteName = atom()</v>
         <v>TestcaseName = atom()</v>
         <v>Config = [{Key,Value}]</v>
         <v>Return = NewReturn = Config | SkipOrFail | term()</v>
@@ -316,15 +335,21 @@
           but for function
           <seealso marker="common_test#Module:init_per_testcase-2"><c>init_per_testcase</c></seealso>
           instead.</p>
+
+	<p>If <c>Module:post_init_per_testcase/5</c> is not exported, common_test
+	  will attempt to call <c>Module:post_init_per_testcase(TestcaseName,
+	  Config, Return, CTHState)</c> instead. This is for backwards
+	  compatibility.</p>
       </desc>
     </func>
 
     <func>
-      <name>Module:pre_end_per_testcase(TestcaseName, InitData, CTHState) -&gt; Result</name>
+      <name>Module:pre_end_per_testcase(SuiteName, TestcaseName, EndData, CTHState) -&gt; Result</name>
       <fsummary>Called before end_per_testcase.</fsummary>
       <type>
+        <v>SuiteName = atom()</v>
         <v>TestcaseName = atom()</v>
-        <v>InitData = Config</v>
+        <v>EndData = Config</v>
         <v>Config = NewConfig = [{Key,Value}]</v>
         <v>CTHState = NewCTHState = term()</v>
         <v>Result = {NewConfig, NewCTHState}</v>
@@ -345,14 +370,20 @@
 
         <p>This function can not change the result of the test case by returning skip or fail
 	tuples, but it may insert items in <c>Config</c> that can be read in
-	<c>end_per_testcase/2</c> or in <c>post_end_per_testcase/4</c>.</p>
+	<c>end_per_testcase/2</c> or in <c>post_end_per_testcase/5</c>.</p>
+
+	<p>If <c>Module:pre_end_per_testcase/4</c> is not exported, common_test
+	  will attempt to call <c>Module:pre_end_per_testcase(TestcaseName,
+	  EndData, CTHState)</c> instead. This is for backwards
+	  compatibility.</p>
       </desc>
     </func>
 
     <func>
-      <name>Module:post_end_per_testcase(TestcaseName, Config, Return, CTHState) -&gt; Result</name>
+      <name>Module:post_end_per_testcase(SuiteName, TestcaseName, Config, Return, CTHState) -&gt; Result</name>
       <fsummary>Called after end_per_testcase.</fsummary>
       <type>
+        <v>SuiteName = atom()</v>
         <v>TestcaseName = atom()</v>
         <v>Config = [{Key,Value}]</v>
         <v>Return = NewReturn = Config | SkipOrFail | term()</v>
@@ -373,13 +404,19 @@
           but for function
           <seealso marker="common_test#Module:end_per_testcase-2"><c>end_per_testcase</c></seealso>
           instead.</p>
+
+	<p>If <c>Module:post_end_per_testcase/5</c> is not exported, common_test
+	  will attempt to call <c>Module:post_end_per_testcase(TestcaseName,
+	  Config, Return, CTHState)</c> instead. This is for backwards
+	  compatibility.</p>
       </desc>
     </func>
 
     <func>
-      <name>Module:pre_end_per_group(GroupName, EndData, CTHState) -&gt; Result</name>
+      <name>Module:pre_end_per_group(SuiteName, GroupName, EndData, CTHState) -&gt; Result</name>
       <fsummary>Called before end_per_group.</fsummary>
       <type>
+        <v>SuiteName = atom()</v>
         <v>GroupName = atom()</v>
         <v>EndData = Config | SkipOrFail</v>
         <v>Config = NewConfig = [{Key,Value}]</v>
@@ -400,13 +437,19 @@
           but for function
           <seealso marker="common_test#Module:end_per_group-2"><c>end_per_group</c></seealso>
           instead.</p>
+
+	<p>If <c>Module:pre_end_per_group/4</c> is not exported, common_test
+	  will attempt to call <c>Module:pre_end_per_group(GroupName,
+	  EndData, CTHState)</c> instead. This is for backwards
+	  compatibility.</p>
       </desc>
     </func>
 
     <func>
-      <name>Module:post_end_per_group(GroupName, Config, Return, CTHState) -&gt; Result</name>
+      <name>Module:post_end_per_group(SuiteName, GroupName, Config, Return, CTHState) -&gt; Result</name>
       <fsummary>Called after end_per_group.</fsummary>
       <type>
+        <v>SuiteName = atom()</v>
         <v>GroupName = atom()</v>
         <v>Config = [{Key,Value}]</v>
         <v>Return = NewReturn = Config | SkipOrFail | term()</v>
@@ -427,6 +470,11 @@
           but for function
           <seealso marker="common_test#Module:end_per_group-2">end_per_group</seealso>
           instead.</p>
+
+	<p>If <c>Module:post_end_per_group/5</c> is not exported, common_test
+	  will attempt to call <c>Module:post_end_per_group(GroupName,
+	  Config, Return, CTHState)</c> instead. This is for backwards
+	  compatibility.</p>
       </desc>
     </func>
 
@@ -485,9 +533,10 @@
     </func>
 
     <func>
-      <name>Module:on_tc_fail(TestName, Reason, CTHState) -&gt; NewCTHState</name>
+      <name>Module:on_tc_fail(SuiteName, TestName, Reason, CTHState) -&gt; NewCTHState</name>
       <fsummary>Called after the CTH scope ends.</fsummary>
       <type>
+        <v>SuiteName = atom()</v>
         <v>TestName = init_per_suite | end_per_suite | {init_per_group,GroupName} | {end_per_group,GroupName} | {FuncName,GroupName} | FuncName</v>
         <v>FuncName = atom()</v>
         <v>GroupName = atom()</v>
@@ -505,7 +554,7 @@
           <item><p>If <c>init_per_suite</c> fails, this function is called after
             <seealso marker="#Module:post_init_per_suite-4"><c>post_init_per_suite</c></seealso>.</p></item>
           <item><p>If a test case fails, this funcion is called after
-          <seealso marker="#Module:post_end_per_testcase-4"><c>post_end_per_testcase</c></seealso>.</p></item>
+          <seealso marker="#Module:post_end_per_testcase-5"><c>post_end_per_testcase</c></seealso>.</p></item>
         </list>
 
          <p>If the failed test case belongs to a test case group, the first
@@ -519,13 +568,19 @@
           For details, see section
           <seealso marker="event_handler_chapter#events">Event Handling</seealso>
           in the User's Guide.</p>
+
+	<p>If <c>Module:on_tc_fail/4</c> is not exported, common_test
+	  will attempt to call <c>Module:on_tc_fail(TestName, Reason,
+	  CTHState)</c> instead. This is for backwards
+	  compatibility.</p>
       </desc>
     </func>
 
     <func>
-      <name>Module:on_tc_skip(TestName, Reason, CTHState) -&gt; NewCTHState</name>
+      <name>Module:on_tc_skip(SuiteName, TestName, Reason, CTHState) -&gt; NewCTHState</name>
       <fsummary>Called after the CTH scope ends.</fsummary>
       <type>
+        <v>SuiteName = atom()</v>
         <v>TestName = init_per_suite | end_per_suite | {init_per_group,GroupName} | {end_per_group,GroupName} | {FuncName,GroupName} | FuncName</v>
         <v>FuncName = atom()</v>
         <v>GroupName = atom()</v>
@@ -542,9 +597,9 @@
         <list type="bulleted">
           <item><p>If <c>init_per_group</c> is skipped, this function is
             called after 
-            <seealso marker="#Module:post_init_per_group-4"><c>post_init_per_group</c></seealso>.</p></item>
+            <seealso marker="#Module:post_init_per_group-5"><c>post_init_per_group</c></seealso>.</p></item>
           <item><p>If a test case is skipped, this function is called after
-          <seealso marker="#Module:post_end_per_testcase-4"><c>post_end_per_testcase</c></seealso>.</p></item>
+          <seealso marker="#Module:post_end_per_testcase-5"><c>post_end_per_testcase</c></seealso>.</p></item>
         </list>
 
         <p>If the skipped test case belongs to a test case group, the first
@@ -559,6 +614,11 @@
           For details, see section
           <seealso marker="event_handler_chapter#events">Event Handling</seealso>
           in the User's Guide.</p>
+
+	<p>If <c>Module:on_tc_skip/4</c> is not exported, common_test
+	  will attempt to call <c>Module:on_tc_skip(TestName, Reason,
+	  CTHState)</c> instead. This is for backwards
+	  compatibility.</p>
       </desc>
     </func>
 
diff --git a/lib/common_test/doc/src/ct_hooks_chapter.xml b/lib/common_test/doc/src/ct_hooks_chapter.xml
index 0e4c35e11f..bfad96e489 100644
--- a/lib/common_test/doc/src/ct_hooks_chapter.xml
+++ b/lib/common_test/doc/src/ct_hooks_chapter.xml
@@ -38,7 +38,7 @@
       extensions of the default behavior of <c>Common Test</c> using hooks 
       before and after all test suite calls. CTHs allow advanced <c>Common Test</c>
       users to abstract out behavior that is common to multiple test suites
-      without littering all test suites with library calls. this can be used 
+      without littering all test suites with library calls. This can be used
       for logging, starting, and monitoring external systems, 
       building C files needed by the tests, and so on.</p>
 
@@ -175,10 +175,10 @@
       <row>
 	<cell><seealso marker="common_test#Module:init_per_group-2">
 	  init_per_group/2</seealso></cell>
-	<cell><seealso marker="ct_hooks#Module:post_init_per_group-4">
-	    post_init_per_group/4</seealso> is called</cell>
-	<cell><seealso marker="ct_hooks#Module:post_end_per_suite-4">
-	  post_end_per_group/4</seealso> has been called for that group</cell>
+	<cell><seealso marker="ct_hooks#Module:post_init_per_group-5">
+	    post_init_per_group/5</seealso> is called</cell>
+	<cell><seealso marker="ct_hooks#Module:post_end_per_group-5">
+	  post_end_per_group/5</seealso> has been called for that group</cell>
       </row>
     <tcaption>Scope of a CTH</tcaption>
     </table>
@@ -245,16 +245,18 @@
      </list>
 
         <p>
-	This is done in the CTH functions called pre_&lt;name of function&gt;.
-	These functions take the same three arguments, <c>Name</c>, 
+	This is done in the CTH functions called <c>pre_&lt;name of function&gt;</c>.
+	These functions take the arguments <c>SuiteName</c>, <c>Name</c> (group or test case name, if applicable), 
 	<c>Config</c>, and <c>CTHState</c>. The return value of the CTH function
 	is always a combination of a result for the suite/group/test and an 
 	updated <c>CTHState</c>.</p>
 
 	<p>To let the test suite continue on executing, return the configuration 
-	list that you want the test to use as the result. To skip or 
-	fail the test, return a tuple with <c>skip</c> or <c>fail</c>, and a reason 
-	as the result.</p>
+	list that you want the test to use as the result.</p>
+
+        <p>All pre hooks, except <c>pre_end_per_testcase/4</c>, can
+        skip or fail the test by returning a tuple with <c>skip</c> or
+        <c>fail</c>, and a reason as the result.</p>
 
 	<p><em>Example:</em></p>
 	<code>
@@ -290,7 +292,7 @@
 
      <p>
       This is done in the CTH functions called <c>post_&lt;name of function&gt;</c>. 
-      These functions take the same four arguments, <c>Name</c>, 
+      These functions take the arguments <c>SuiteName</c>, <c>Name</c> (group or test case name, if applicable),
       <c>Config</c>, <c>Return</c>, and <c>CTHState</c>. <c>Config</c> in this
       case is the same <c>Config</c> as the testcase is called with. 
       <c>Return</c> is the value returned by the testcase. If the testcase 
@@ -308,7 +310,7 @@
 	
 	<p><em>Example:</em></p>
 	<code>
- post_end_per_testcase(_TC, Config, {'EXIT',{_,_}}, CTHState) -&gt;
+ post_end_per_testcase(_Suite, _TC, Config, {'EXIT',{_,_}}, CTHState) -&gt;
    case db:check_consistency() of
      true ->
        %% DB is good, pass the test.
@@ -317,7 +319,7 @@
        %% DB is not good, mark as skipped instead of failing
        {{skip, "DB is inconsisten!"}, CTHState}
    end;
- post_end_per_testcase(_TC, Config, Return, CTHState) -&gt;
+ post_end_per_testcase(_Suite, _TC, Config, Return, CTHState) -&gt;
    %% Do nothing if tc does not crash.
    {Return, CTHState}.</code>
 
@@ -331,8 +333,8 @@
       <title>Skip and Fail Hooks</title>
       <p>
 	After any post hook has been executed for all installed CTHs, 
-	<seealso marker="ct_hooks#Module:on_tc_fail-3">on_tc_fail</seealso>
-	or <seealso marker="ct_hooks#Module:on_tc_skip-3">on_tc_skip</seealso> 
+	<seealso marker="ct_hooks#Module:on_tc_fail-4">on_tc_fail</seealso>
+	or <seealso marker="ct_hooks#Module:on_tc_skip-4">on_tc_skip</seealso>
 	is called if the testcase failed or was skipped, respectively. 
 	You cannot affect the outcome of the tests any further at this point. 
       </p>
@@ -389,18 +391,18 @@
  -export([pre_end_per_suite/3]).
  -export([post_end_per_suite/4]).
 
- -export([pre_init_per_group/3]).
- -export([post_init_per_group/4]).
- -export([pre_end_per_group/3]).
- -export([post_end_per_group/4]).
+ -export([pre_init_per_group/4]).
+ -export([post_init_per_group/5]).
+ -export([pre_end_per_group/4]).
+ -export([post_end_per_group/5]).
 
- -export([pre_init_per_testcase/3]).
- -export([post_init_per_testcase/4]).
- -export([pre_end_per_testcase/3]).
- -export([post_end_per_testcase/4]).
+ -export([pre_init_per_testcase/4]).
+ -export([post_init_per_testcase/5]).
+ -export([pre_end_per_testcase/4]).
+ -export([post_end_per_testcase/5]).
 
- -export([on_tc_fail/3]).
- -export([on_tc_skip/3]).
+ -export([on_tc_fail/4]).
+ -export([on_tc_skip/4]).
 
  -export([terminate/1]).
 
@@ -435,46 +437,46 @@
                            total = State#state.total + State#state.suite_total } }.
 
  %% @doc Called before each init_per_group.
- pre_init_per_group(Group,Config,State) ->
+ pre_init_per_group(Suite,Group,Config,State) ->
      {Config, State}.
 
  %% @doc Called after each init_per_group.
- post_init_per_group(Group,Config,Return,State) ->
+ post_init_per_group(Suite,Group,Config,Return,State) ->
      {Return, State}.
 
  %% @doc Called before each end_per_group. 
- pre_end_per_group(Group,Config,State) ->
+ pre_end_per_group(Suite,Group,Config,State) ->
      {Config, State}.
 
  %% @doc Called after each end_per_group. 
- post_end_per_group(Group,Config,Return,State) ->
+ post_end_per_group(Suite,Group,Config,Return,State) ->
      {Return, State}.
 
  %% @doc Called before each init_per_testcase.
- pre_init_per_testcase(TC,Config,State) ->
+ pre_init_per_testcase(Suite,TC,Config,State) ->
      {Config, State#state{ ts = now(), total = State#state.suite_total + 1 } }.
 
  %% Called after each init_per_testcase (immediately before the test case).
- post_init_per_testcase(TC,Config,Return,State) ->
+ post_init_per_testcase(Suite,TC,Config,Return,State) ->
      {Return, State}
 
 %% @doc Called before each end_per_testcase (immediately after the test case).
- pre_end_per_testcase(TC,Config,State) ->
+ pre_end_per_testcase(Suite,TC,Config,State) ->
      {Config, State}.
 
  %% @doc Called after each end_per_testcase.
- post_end_per_testcase(TC,Config,Return,State) ->
-     TCInfo = {testcase, TC, Return, timer:now_diff(now(), State#state.ts)},
+ post_end_per_testcase(Suite,TC,Config,Return,State) ->
+     TCInfo = {testcase, Suite, TC, Return, timer:now_diff(now(), State#state.ts)},
      {Return, State#state{ ts = undefined, tcs = [TCInfo | State#state.tcs] } }.
 
  %% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group,
  %% post_end_per_group and post_end_per_testcase if the suite, group or test case failed.
- on_tc_fail(TC, Reason, State) ->
+ on_tc_fail(Suite, TC, Reason, State) ->
      State.
 
  %% @doc Called when a test case is skipped by either user action
  %% or due to an init function failing.  
- on_tc_skip(TC, Reason, State) ->
+ on_tc_skip(Suite, TC, Reason, State) ->
      State.
 
  %% @doc Called when the scope of the CTH is done
diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml
index f70bdb16c5..6a0d87bcaf 100644
--- a/lib/common_test/doc/src/write_test_chapter.xml
+++ b/lib/common_test/doc/src/write_test_chapter.xml
@@ -566,7 +566,7 @@
     for the test cases in the group. After execution of the group is finished, function
     <seealso marker="common_test#Module:end_per_group-2"><c>end_per_group(GroupName, Config)</c></seealso> 
     is called. This function is meant to be used for cleaning up after 
-    <c>init_per_group/2</c>.</p>
+    <c>init_per_group/2</c>. If the init function is defined, so must the end function be.</p>
 
     <p>Whenever a group is executed, if <c>init_per_group</c> and
       <c>end_per_group</c> do not exist in the suite, <c>Common Test</c> calls