[ct] Fix hooks and fail when one of init/end_per_* does not exit

The following bugs are corrected:

- if init_per_suite is exported from a test suite, but not
  end_per_suite, then pre/post_end_per_suite will be called with
  Suite=ct_framework instead of the correct suite name.

- if end_per_group is exported from a suite, but not init_per_group,
  then end_per_group is never called.

According to the documentation, if implementing an init config
function, you must also implement the end config function, so the two
scenarios above are really not allowed. To make this more visible,
common_test will now mark the non-exported config function as failed
with reason 'undef' if the other function is exported.

For example, if init_per_suite is exported, but not end_per_suite,
then end_per_suite will be marked as failed with reason undef. (If
none of them exist, then they will both be marked as passed since the
default functions in ct_framework are called instead.)

All hook functions are always called with the correct suite name,
i.e. never with Suite=ct_framework.

Conflicts:
	lib/common_test/test/ct_hooks_SUITE.erl

4 files changed, 315 insertions, 0 deletions

diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl
index 32f8ad1a22..835e3d4805 100644
--- a/lib/common_test/test/ct_hooks_SUITE.erl
+++ b/lib/common_test/test/ct_hooks_SUITE.erl
@@ -85,6 +85,7 @@ all(suite) ->
        skip_post_suite_cth, recover_post_suite_cth, update_config_cth,
        state_update_cth, options_cth, same_id_cth,
        fail_n_skip_with_minimal_cth, prio_cth, no_config,
+       no_init_suite_config, no_init_config, no_end_config,
        fallback, data_dir, cth_log
       ]
     ).
@@ -223,6 +224,16 @@ no_config(Config) when is_list(Config) ->
     do_test(no_config, "ct_no_config_SUITE.erl",
 	    [verify_config_cth],Config).
 
+no_init_suite_config(Config) when is_list(Config) ->
+    do_test(no_init_suite_config, "ct_no_init_suite_config_SUITE.erl",
+            [empty_cth],Config).
+
+no_init_config(Config) when is_list(Config) ->
+    do_test(no_init_config, "ct_no_init_config_SUITE.erl",[empty_cth],Config).
+
+no_end_config(Config) when is_list(Config) ->
+    do_test(no_end_config, "ct_no_end_config_SUITE.erl",[empty_cth],Config).
+
 data_dir(Config) when is_list(Config) ->
     do_test(data_dir, "ct_data_dir_SUITE.erl",
 	    [verify_data_dir_cth],Config).
@@ -1260,6 +1271,166 @@ test_events(no_config) ->
      {?eh,stop_logging,[]}
     ];
 
+test_events(no_init_suite_config) ->
+    [
+     {?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+     {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+     {?eh,start_info,{1,1,1}},
+     {?eh,tc_start,{ct_no_init_suite_config_SUITE,init_per_suite}},
+     {?eh,cth,{empty_cth,pre_init_per_suite,
+	       [ct_no_init_suite_config_SUITE,'$proplist',[]]}},
+     {?eh,cth,{empty_cth,post_init_per_suite,
+	       [ct_no_init_suite_config_SUITE,'$proplist','_',[]]}},
+     {?eh,tc_done,{ct_no_init_suite_config_SUITE,init_per_suite,
+                   {failed,{error,{undef,'_'}}}}},
+     {?eh,cth,{empty_cth,on_tc_fail,[ct_no_init_suite_config_SUITE,
+                                     init_per_suite,
+                                     {failed,{undef,'_'}},[]]}},
+      {?eh,tc_auto_skip,{ct_no_init_suite_config_SUITE,test_case,
+                         {failed,{ct_no_init_suite_config_SUITE,init_per_suite,
+                                  {'EXIT',{undef,'_'}}}}}},
+     {?eh,cth,{empty_cth,on_tc_skip,
+               [ct_no_init_suite_config_SUITE,
+                test_case,
+                {tc_auto_skip,
+                 {failed,{ct_no_init_suite_config_SUITE,init_per_suite,
+                          {'EXIT',{undef,'_'}}}}},
+                []]}},
+     {?eh,test_stats,{0,0,{0,1}}},
+     {?eh,tc_auto_skip,{ct_no_init_suite_config_SUITE,end_per_suite,
+                        {failed,{ct_no_init_suite_config_SUITE,init_per_suite,
+                                 {'EXIT',{undef,'_'}}}}}},
+     {?eh,cth,{empty_cth,on_tc_skip,
+               [ct_no_init_suite_config_SUITE,
+                end_per_suite,
+                {tc_auto_skip,
+                 {failed,{ct_no_init_suite_config_SUITE,init_per_suite,
+                          {'EXIT',{undef,'_'}}}}},
+                []]}},
+     {?eh,test_done,{'DEF','STOP_TIME'}},
+     {?eh,cth,{empty_cth,terminate,[[]]}},
+     {?eh,stop_logging,[]}
+    ];
+
+test_events(no_init_config) ->
+    [
+     {?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+     {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+     {?eh,start_info,{1,1,2}},
+     {?eh,tc_start,{ct_no_init_config_SUITE,init_per_suite}},
+     {?eh,cth,{empty_cth,pre_init_per_suite,
+	       [ct_no_init_config_SUITE,'$proplist',[]]}},
+     {?eh,cth,{empty_cth,post_init_per_suite,
+	       [ct_no_init_config_SUITE,'$proplist','$proplist',[]]}},
+     {?eh,tc_done,{ct_no_init_config_SUITE,init_per_suite,ok}},
+     {?eh,tc_start,{ct_no_init_config_SUITE,test_case_1}},
+     {?eh,cth,{empty_cth,pre_init_per_testcase,
+	       [ct_no_init_config_SUITE,test_case_1,'$proplist',[]]}},
+     {?eh,cth,{empty_cth,post_end_per_testcase,
+	       [ct_no_init_config_SUITE,test_case_1,'$proplist',ok,[]]}},
+     {?eh,tc_done,{ct_no_init_config_SUITE,test_case_1,ok}},
+     {?eh,test_stats,{1,0,{0,0}}},
+     [{?eh,tc_start,{ct_no_init_config_SUITE,{init_per_group,test_group,[]}}},
+      {?eh,cth,{empty_cth,pre_init_per_group,
+		[ct_no_init_config_SUITE,test_group,'$proplist',[]]}},
+      {?eh,cth,{empty_cth,post_init_per_group,
+		[ct_no_init_config_SUITE,test_group,'$proplist','_',[]]}},
+      {?eh,tc_done,{ct_no_init_config_SUITE,{init_per_group,test_group,[]},
+                    {failed,{error,{undef,'_'}}}}},
+      {?eh,cth,{empty_cth,on_tc_fail,[ct_no_init_config_SUITE,
+                                      {init_per_group,test_group},
+                                      {failed,{undef,'_'}},[]]}},
+      {?eh,tc_auto_skip,{ct_no_init_config_SUITE,{test_case_2,test_group},
+                         {failed,{ct_no_init_config_SUITE,init_per_group,
+                                  {'EXIT',{undef,'_'}}}}}},
+      {?eh,cth,{empty_cth,on_tc_skip,[ct_no_init_config_SUITE,
+                                      {test_case_2,test_group},
+                                      {tc_auto_skip,
+                                       {failed,
+                                        {ct_no_init_config_SUITE,init_per_group,
+                                         {'EXIT',{undef,'_'}}}}},
+                                       []]}},
+      {?eh,test_stats,{1,0,{0,1}}},
+      {?eh,tc_auto_skip,{ct_no_init_config_SUITE,{end_per_group,test_group},
+                         {failed,{ct_no_init_config_SUITE,init_per_group,
+                                  {'EXIT',{undef,'_'}}}}}},
+      {?eh,cth,{empty_cth,on_tc_skip,[ct_no_init_config_SUITE,
+                                      {end_per_group,test_group},
+                                      {tc_auto_skip,
+                                       {failed,
+                                        {ct_no_init_config_SUITE,init_per_group,
+                                         {'EXIT',{undef,'_'}}}}},
+                                       []]}}],
+     {?eh,tc_start,{ct_no_init_config_SUITE,end_per_suite}},
+     {?eh,cth,{empty_cth,pre_end_per_suite,
+	       [ct_no_init_config_SUITE,'$proplist',[]]}},
+     {?eh,cth,{empty_cth,post_end_per_suite,
+	       [ct_no_init_config_SUITE,'$proplist',ok,[]]}},
+     {?eh,tc_done,{ct_no_init_config_SUITE,end_per_suite,ok}},
+     {?eh,test_done,{'DEF','STOP_TIME'}},
+     {?eh,cth,{empty_cth,terminate,[[]]}},
+     {?eh,stop_logging,[]}
+    ];
+
+test_events(no_end_config) ->
+    [
+     {?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+     {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+     {?eh,start_info,{1,1,2}},
+     {?eh,tc_start,{ct_no_end_config_SUITE,init_per_suite}},
+     {?eh,cth,{empty_cth,pre_init_per_suite,
+	       [ct_no_end_config_SUITE,'$proplist',[]]}},
+     {?eh,cth,{empty_cth,post_init_per_suite,
+	       [ct_no_end_config_SUITE,'$proplist','$proplist',[]]}},
+     {?eh,tc_done,{ct_no_end_config_SUITE,init_per_suite,ok}},
+     {?eh,tc_start,{ct_no_end_config_SUITE,test_case_1}},
+     {?eh,cth,{empty_cth,pre_init_per_testcase,
+	       [ct_no_end_config_SUITE,test_case_1,'$proplist',[]]}},
+     {?eh,cth,{empty_cth,post_end_per_testcase,
+	       [ct_no_end_config_SUITE,test_case_1,'$proplist',ok,[]]}},
+     {?eh,tc_done,{ct_no_end_config_SUITE,test_case_1,ok}},
+     {?eh,test_stats,{1,0,{0,0}}},
+     [{?eh,tc_start,{ct_no_end_config_SUITE,
+                     {init_per_group,test_group,'$proplist'}}},
+      {?eh,cth,{empty_cth,pre_init_per_group,
+		[ct_no_end_config_SUITE,test_group,'$proplist',[]]}},
+      {?eh,cth,{empty_cth,post_init_per_group,
+		[ct_no_end_config_SUITE,test_group,'$proplist','$proplist',[]]}},
+      {?eh,tc_done,{ct_no_end_config_SUITE,
+		    {init_per_group,test_group,'$proplist'},ok}},
+      {?eh,tc_start,{ct_no_end_config_SUITE,test_case_2}},
+      {?eh,cth,{empty_cth,pre_init_per_testcase,
+		[ct_no_end_config_SUITE,test_case_2,'$proplist',[]]}},
+      {?eh,cth,{empty_cth,post_end_per_testcase,
+		[ct_no_end_config_SUITE,test_case_2,'$proplist',ok,[]]}},
+      {?eh,tc_done,{ct_no_end_config_SUITE,test_case_2,ok}},
+      {?eh,test_stats,{2,0,{0,0}}},
+      {?eh,tc_start,{ct_no_end_config_SUITE,
+                     {end_per_group,test_group,'$proplist'}}},
+      {?eh,cth,{empty_cth,pre_end_per_group,
+		[ct_no_end_config_SUITE,test_group,'$proplist',[]]}},
+      {?eh,cth,{empty_cth,post_end_per_group,
+		[ct_no_end_config_SUITE,test_group,'$proplist','_',[]]}},
+      {?eh,tc_done,{ct_no_end_config_SUITE,{end_per_group,test_group,[]},
+                    {failed,{error,{undef,'_'}}}}},
+      {?eh,cth,{empty_cth,on_tc_fail,[ct_no_end_config_SUITE,
+                                      {end_per_group,test_group},
+                                      {failed,{undef,'_'}},[]]}}],
+     {?eh,tc_start,{ct_no_end_config_SUITE,end_per_suite}},
+     {?eh,cth,{empty_cth,pre_end_per_suite,
+	       [ct_no_end_config_SUITE,'$proplist',[]]}},
+     {?eh,cth,{empty_cth,post_end_per_suite,
+	       [ct_no_end_config_SUITE,'$proplist','_',[]]}},
+     {?eh,tc_done,{ct_no_end_config_SUITE,end_per_suite,
+                   {failed,{error,{undef,'_'}}}}},
+     {?eh,test_done,{'DEF','STOP_TIME'}},
+     {?eh,cth,{empty_cth,terminate,[[]]}},
+     {?eh,stop_logging,[]}
+    ];
+
 test_events(data_dir) ->
     [
      {?eh,start_logging,{'DEF','RUNDIR'}},
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_end_config_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_end_config_SUITE.erl
new file mode 100644
index 0000000000..7cdaf2024b
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_end_config_SUITE.erl
@@ -0,0 +1,51 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_no_end_config_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+%%% This suite is used to verify that all pre/post_end_per_* callbacks
+%%% are called with correct SuiteName even if no end_per_* config
+%%% function exist in the suite, and that the non-exported config
+%%% functions fail with 'undef'.
+
+init_per_suite(Config) ->
+    Config.
+
+init_per_group(_Group,Config) ->
+    Config.
+
+init_per_testcase(_TC,Config) ->
+    Config.
+
+all() ->
+    [test_case_1, {group,test_group}].
+
+groups() ->
+    [{test_group,[],[test_case_2]}].
+
+test_case_1(Config) ->
+    ok.
+
+test_case_2(Config) ->
+    ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_config_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_config_SUITE.erl
new file mode 100644
index 0000000000..43c062d66f
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_config_SUITE.erl
@@ -0,0 +1,54 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_no_init_config_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+%%% This suite is used to verify that all
+%%% pre/post_init_per_group/testcase callbacks are called with correct
+%%% SuiteName even if no init_per_group/testcase function exist in the
+%%% suite, and that the non-exported config functions fail with 'undef'.
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(Config) ->
+    Config.
+
+end_per_group(_Group,Config) ->
+    Config.
+
+end_per_testcase(_TC,Config) ->
+    Config.
+
+all() ->
+    [test_case_1, {group,test_group}].
+
+groups() ->
+    [{test_group,[],[test_case_2]}].
+
+test_case_1(Config) ->
+    ok.
+
+test_case_2(Config) ->
+    ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_suite_config_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_suite_config_SUITE.erl
new file mode 100644
index 0000000000..85dfe8ca4b
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_suite_config_SUITE.erl
@@ -0,0 +1,39 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_no_init_suite_config_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+%%% This suite is used to verify that pre/post_init_per_suite
+%%% callbacks are called with correct SuiteName even if no
+%%% init_per_suite function exist in the suite, and that the
+%%% non-exported config function fails with 'undef'.
+
+end_per_suite(Config) ->
+    Config.
+
+all() ->
+    [test_case].
+
+test_case(Config) ->
+    ok.