Merge remote branch 'origin/peppe/common_test/restore_config'

* origin/peppe/common_test/restore_config:
  Add verification terms in test suite
  Add test case to verify that config data gets properly restored

OTP-10070

3 files changed, 476 insertions, 3 deletions

diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl
index 6d90b29f41..86ec71c98e 100644
--- a/lib/common_test/test/ct_error_SUITE.erl
+++ b/lib/common_test/test/ct_error_SUITE.erl
@@ -44,8 +44,12 @@
 %% there will be clashes with logging processes etc).
 %%--------------------------------------------------------------------
 init_per_suite(Config) ->
-    Config1 = ct_test_support:init_per_suite(Config),
-    Config1.
+    DataDir = ?config(data_dir, Config),
+    TestDir = filename:join(DataDir, "error/test/"),
+    CTH = filename:join(TestDir, "verify_config.erl"),
+    ct:pal("Compiling ~p: ~p",
+	   [CTH,compile:file(CTH,[{outdir,TestDir},debug_info])]),
+    ct_test_support:init_per_suite([{path_dirs,[TestDir]} | Config]).
 
 end_per_suite(Config) ->
     ct_test_support:end_per_suite(Config).
@@ -61,7 +65,8 @@ suite() -> [{ct_hooks,[ts_install_cth]}].
 all() -> 
     [cfg_error, lib_error, no_compile, timetrap_end_conf,
      timetrap_normal, timetrap_extended, timetrap_parallel,
-     timetrap_fun, timetrap_fun_group, misc_errors].
+     timetrap_fun, timetrap_fun_group, misc_errors,
+     config_restored].
 
 groups() -> 
     [].
@@ -285,6 +290,24 @@ misc_errors(Config) when is_list(Config) ->
     TestEvents = events_to_check(misc_errors),
     ok = ct_test_support:verify_events(TestEvents, Events, Config).
 
+%%%-----------------------------------------------------------------
+%%%
+config_restored(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+    Suite = filename:join(DataDir, "error/test/config_restored_SUITE"),
+    {Opts,ERPid} = setup([{suite,Suite},
+			  {ct_hooks,[verify_config]}], 
+			 Config),
+    ok = ct_test_support:run(Opts, Config),
+    Events = ct_test_support:get_events(ERPid, Config),
+
+    ct_test_support:log_events(config_restored,
+			       reformat(Events, ?eh),
+			       ?config(priv_dir, Config),
+			       Opts),
+
+    TestEvents = events_to_check(config_restored),
+    ok = ct_test_support:verify_events(TestEvents, Events, Config).
 
 %%%-----------------------------------------------------------------
 %%% HELP FUNCTIONS
@@ -1439,4 +1462,40 @@ test_events(misc_errors) ->
      {?eh,test_stats,{2,7,{0,0}}},
      {?eh,test_done,{'DEF','STOP_TIME'}},
      {?eh,stop_logging,[]}
+    ];
+
+test_events(config_restored) ->
+    [
+     {?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+     {?eh,start_info,{1,1,4}},
+     {?eh,tc_start,{config_restored_SUITE,init_per_suite}},
+     {?eh,tc_done,{config_restored_SUITE,init_per_suite,ok}},
+     {?eh,tc_start,{config_restored_SUITE,to_tc}},
+     {?eh,cth,{verify_config,post_end_per_testcase,{to_tc,diff_ok}}},
+     {?eh,tc_done,
+      {config_restored_SUITE,to_tc,{failed,{timetrap_timeout,1000}}}},
+     {?eh,test_stats,{0,1,{0,0}}},
+     {?eh,tc_start,{config_restored_SUITE,exit_tc}},
+     {?eh,cth,{verify_config,post_end_per_testcase,{exit_tc,diff_ok}}},
+     {?eh,tc_done,{config_restored_SUITE,exit_tc,
+		   {failed,{error,{test_case_failed,"Goodbye!"}}}}},
+     {?eh,test_stats,{0,2,{0,0}}},
+     [{?eh,tc_start,{config_restored_SUITE,{init_per_group,g1,[]}}},
+      {?eh,tc_start,{config_restored_SUITE,to_tc}},
+      {?eh,cth,{verify_config,post_end_per_testcase,{to_tc,diff_ok}}},
+      {?eh,tc_done,
+       {config_restored_SUITE,to_tc,{failed,{timetrap_timeout,1000}}}},
+      {?eh,test_stats,{0,3,{0,0}}},
+      {?eh,tc_start,{config_restored_SUITE,exit_tc}},
+      {?eh,cth,{verify_config,post_end_per_testcase,{exit_tc,diff_ok}}},
+      {?eh,tc_done,{config_restored_SUITE,exit_tc,
+		    {failed,{error,{test_case_failed,"Goodbye!"}}}}},
+      {?eh,test_stats,{0,4,{0,0}}},
+      {?eh,tc_start,{config_restored_SUITE,{end_per_group,g1,[]}}},
+      {?eh,tc_done,{config_restored_SUITE,{end_per_group,g1,[]},ok}}],
+     {?eh,tc_start,{config_restored_SUITE,end_per_suite}},
+     {?eh,tc_done,{config_restored_SUITE,end_per_suite,ok}},
+     {?eh,test_done,{'DEF','STOP_TIME'}},
+     {?eh,stop_logging,[]}
     ].
diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/config_restored_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/config_restored_SUITE.erl
new file mode 100644
index 0000000000..fb47f2bcb9
--- /dev/null
+++ b/lib/common_test/test/ct_error_SUITE_data/error/test/config_restored_SUITE.erl
@@ -0,0 +1,175 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(config_restored_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% Function: suite() -> Info
+%% Info = [tuple()]
+%%--------------------------------------------------------------------
+suite() ->
+    [{timetrap,1000}].
+
+%%--------------------------------------------------------------------
+%% Function: init_per_suite(Config0) ->
+%%               Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+    Config1 = [{init_per_suite,?MODULE} | Config],
+    TabPid = spawn(fun() ->
+			   ets:new(?MODULE, [named_table, set, public]),
+			   receive _ -> ok end
+		   end),
+    [{tab,TabPid} | Config1].
+
+%%--------------------------------------------------------------------
+%% Function: end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_suite(Config) ->
+    ets:delete(?MODULE),
+    exit(?config(tab, Config), kill),
+    ok.
+
+%%--------------------------------------------------------------------
+%% Function: init_per_group(GroupName, Config0) ->
+%%               Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_group(GroupName, Config) ->
+    [{init_per_group,GroupName} | Config].
+
+%%--------------------------------------------------------------------
+%% Function: end_per_group(GroupName, Config0) ->
+%%               void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+    ok.
+
+%%--------------------------------------------------------------------
+%% Function: init_per_testcase(TestCase, Config0) ->
+%%               Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_testcase(TC, Config) ->
+    Config1 = [{init_per_testcase,TC} | Config],
+    ets:insert(?MODULE, {config,Config}),
+    %% ct:pal("Config after init_per_testcase(~w) = ~p", [TC,Config1]),
+    Config1.
+   
+%%--------------------------------------------------------------------
+%% Function: end_per_testcase(TestCase, Config0) ->
+%%               void() | {save_config,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_testcase(TC, Config) ->
+    ct:pal("Config in end_per_testcase(~w) = ~p", [TC,Config]),
+    [{_,MemConfig}] = ets:lookup(?MODULE, config), 
+    diff_config(Config, MemConfig, [tc_status]),
+    ?MODULE = proplists:get_value(init_per_suite, Config),
+    TC = proplists:get_value(init_per_testcase, Config),
+    case ?config(tc_group_properties, Config) of
+	undefined ->
+	    ok;
+	Props ->
+	    GName = proplists:get_value(name, Props),
+	    GName = proplists:get_value(init_per_group, Config)
+    end,
+    ok.
+
+%%--------------------------------------------------------------------
+%% Function: groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%%              repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%%--------------------------------------------------------------------
+groups() ->
+    [{g1,[],[to_tc, exit_tc]}].
+
+%%--------------------------------------------------------------------
+%% Function: all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%%--------------------------------------------------------------------
+all() ->
+    [to_tc, exit_tc,
+     {group,g1}].
+
+
+to_tc(Config) ->
+    %% ct:pal("Config for to_tc = ~p", [Config]),
+    [{_,MemConfig}] = ets:lookup(?MODULE, config), 
+    diff_config(Config, MemConfig, []),
+    ct:sleep(2000).
+
+exit_tc(Config) ->
+    %% ct:pal("Config for exit_tc = ~p", [Config]),
+    [{_,MemConfig}] = ets:lookup(?MODULE, config), 
+    diff_config(Config, MemConfig, []),
+    ct:fail("Goodbye!").
+    
+    
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+
+diff_config(Cfg1, Cfg2, DiffKeys) ->
+    diff_config(Cfg1, Cfg2, DiffKeys, []).
+
+diff_config([{K,V} | Cfg1], Cfg2, DiffKeys, RemKeys) ->
+    case proplists:get_value(K, Cfg2) of
+	undefined ->
+	    diff_config(Cfg1, Cfg2, proplists:delete(K, DiffKeys), RemKeys);
+	V ->
+	    diff_config(Cfg1, proplists:delete(K, Cfg2), DiffKeys, RemKeys);
+	_Other ->
+	    case proplists:is_defined(K, DiffKeys) of
+		true ->
+		    diff_config(Cfg1, Cfg2, proplists:delete(K, DiffKeys), RemKeys);
+		false ->
+		    diff_config(Cfg1, Cfg2, DiffKeys, [K | RemKeys])
+	    end
+    end;
+diff_config([], [], [], []) ->
+    ct:pal("Diff ok!", []),
+    ok;
+diff_config([], Cfg2, DiffKeys, RemKeys) ->
+    Result = {diff_failed, {cfg2,Cfg2}, {diffkeys,DiffKeys}, {remkeys,RemKeys}},
+    ct:pal("Diff failed! Result = ~p", [Result]),
+    exit(Result).
+
diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/verify_config.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/verify_config.erl
new file mode 100644
index 0000000000..40a54b9f99
--- /dev/null
+++ b/lib/common_test/test/ct_error_SUITE_data/error/test/verify_config.erl
@@ -0,0 +1,239 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%% @doc Common Test Example Suite Callback module.
+%%%
+%%% <p>This module gives an example of a common test CTH (Common Test Hook).
+%%% There are many ways to add a CTH to a test run, you can do it either in
+%%% the command line using -ct_hook, in a test spec using
+%%% {ct_hook,M} or in the suite it self by returning ct_hook
+%%% from either suite/0, init_per_suite/1, init_per_group/2 and
+%%% init_per_testcase/2. The scope of the CTH is determined by where is it
+%%% started. If it is started in the command line or test spec then it will
+%%% be stopped at the end of all tests. If it is started in init_per_suite,
+%%% it will be stopped after end_per_suite and so on. See terminate
+%%% documentation for a table describing the scoping machanics. 
+%%%
+%%% All of callbacks except init/1 in a CTH are optional.</p>
+
+-module(verify_config).
+
+%% CT Hooks
+-export([id/1]).
+-export([init/2]).
+
+-export([pre_init_per_suite/3]).
+-export([post_init_per_suite/4]).
+-export([pre_end_per_suite/3]).
+-export([post_end_per_suite/4]).
+
+-export([pre_init_per_group/3]).
+-export([post_init_per_group/4]).
+-export([pre_end_per_group/3]).
+-export([post_end_per_group/4]).
+
+-export([pre_init_per_testcase/3]).
+-export([post_end_per_testcase/4]).
+
+-export([on_tc_fail/3]).
+-export([on_tc_skip/3]).
+
+-export([terminate/1]).
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-type config() :: proplists:proplist().
+-type reason() :: term().
+-type skip_or_fail() :: {skip, reason()} |
+                        {auto_skip, reason()} |
+                        {fail, reason()} | 
+			{'EXIT',reason()}.
+
+-record(state, { id = ?MODULE :: term()}).
+
+%% @doc Always called before any other callback function. Use this to initiate
+%% any common state. It should return an state for this CTH.
+-spec init(Id :: term(), Opts :: proplists:proplist()) ->
+    {ok, State :: #state{}}.
+init(Id, Opts) ->
+    {ok,Opts}.
+
+%% @doc The ID is used to uniquly identify an CTH instance, if two CTH's 
+%% return the same ID the seconds CTH is ignored. This function should NOT 
+%% have any side effects as it might be called multiple times by common test.
+-spec id(Opts :: proplists:proplist()) ->
+    Id :: term().
+id(Opts) ->
+    now().
+
+%% @doc Called before init_per_suite is called. Note that this callback is
+%% only called if the CTH is added before init_per_suite is run (eg. in a test
+%% specification, suite/0 function etc).
+%% You can change the config in the this function.
+-spec pre_init_per_suite(Suite :: atom(),
+		     Config :: config(),
+		     State :: #state{}) ->
+    {config() | skip_or_fail(), NewState :: #state{}}.
+pre_init_per_suite(Suite,Config,State) ->
+    {Config, State}.
+
+%% @doc Called after init_per_suite.
+%% you can change the return value in this function.
+-spec post_init_per_suite(Suite :: atom(),
+			  Config :: config(),
+			  Return :: config() | skip_or_fail(),
+			  State :: #state{}) ->
+    {config() | skip_or_fail(), NewState :: #state{}}.
+post_init_per_suite(Suite,Config,Return,State) ->
+    {Return, State}.
+
+%% @doc Called before end_per_suite. The config/state can be changed here,
+%% though it will only affect the *end_per_suite function.
+-spec pre_end_per_suite(Suite :: atom(),
+		    Config :: config() | skip_or_fail(),
+		    State :: #state{}) ->
+    {ok | skip_or_fail(), NewState :: #state{}}.
+pre_end_per_suite(Suite,Config,State) ->
+    {Config, State}.
+
+%% @doc Called after end_per_suite. Note that the config cannot be
+%% changed here, only the status of the suite.
+-spec post_end_per_suite(Suite :: atom(),
+			 Config :: config(),
+			 Return :: term(),
+			 State :: #state{}) ->
+    {ok | skip_or_fail(), NewState :: #state{}}.
+post_end_per_suite(Suite,Config,Return,State) ->
+    {Return, State}.
+
+%% @doc Called before each init_per_group.
+%% You can change the config in this function.
+-spec pre_init_per_group(Group :: atom(),
+		     Config :: config(),
+		     State :: #state{}) ->
+    {config() | skip_or_fail(), NewState :: #state{}}.
+pre_init_per_group(Group,Config,State) ->
+    {Config, State}.
+
+%% @doc Called after each init_per_group.
+%% You can change the return value in this function.
+-spec post_init_per_group(Group :: atom(),
+			  Config :: config(),
+			  Return :: config() | skip_or_fail(),
+			  State :: #state{}) ->
+    {config() | skip_or_fail(), NewState :: #state{}}.
+post_init_per_group(Group,Config,Return,State) ->
+    {Return, State}.
+
+%% @doc Called after each end_per_group. The config/state can be changed here,
+%% though it will only affect the *end_per_group functions.
+-spec pre_end_per_group(Group :: atom(),
+			Config :: config() | skip_or_fail(),
+			State :: #state{}) ->
+    {ok | skip_or_fail(), NewState :: #state{}}.
+pre_end_per_group(Group,Config,State) ->
+    {Config, State}.
+
+%% @doc Called after each end_per_group. Note that the config cannot be
+%% changed here, only the status of the group.
+-spec post_end_per_group(Group :: atom(),
+			 Config :: config(),
+			 Return :: term(),
+			 State :: #state{}) ->
+    {ok | skip_or_fail(), NewState :: #state{}}.
+post_end_per_group(Group,Config,Return,State) ->
+     {Return, State}.
+
+%% @doc Called before each test case.
+%% You can change the config in this function.
+-spec pre_init_per_testcase(TC :: atom(),
+		  Config :: config(),
+		  State :: #state{}) ->
+    {config() | skip_or_fail(), NewState :: #state{}}.
+pre_init_per_testcase(TC,Config,State) ->
+     {Config, State}.
+
+%% @doc Called after each test case. Note that the config cannot be
+%% changed here, only the status of the test case.
+-spec post_end_per_testcase(TC :: atom(),
+			    Config :: config(),
+			    Return :: term(),
+			    State :: #state{}) ->
+    {ok | skip_or_fail(), NewState :: #state{}}.
+post_end_per_testcase(TC,Config,Return,State) ->
+    %% check that config has been restored
+    ct:pal("Config in verify_config:post_end_per_testcase(~w) = ~p",
+	   [TC,Config]),
+    [{_,MemConfig}] = ets:lookup(config_restored_SUITE, config), 
+    try config_restored_SUITE:diff_config(Config, MemConfig, [tc_status]) of
+	ok ->
+	    gen_event:notify(
+	      ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+				     data = {?MODULE, post_end_per_testcase,
+					     {TC,diff_ok}}})
+    catch
+	_:_ ->
+	    gen_event:notify(
+	      ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+				     data = {?MODULE, post_end_per_testcase,
+					     {TC,diff_failed}}})
+    end,
+    {Return, State}.
+
+%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group,
+%% post_end_per_group and post_end_per_tc if the suite, group or test case failed.
+%% This function should be used for extra cleanup which might be needed.
+%% It is not possible to modify the config or the status of the test run.
+-spec on_tc_fail(TC :: init_per_suite | end_per_suite |
+		 init_per_group | end_per_group | atom(),
+		 Reason :: term(), State :: #state{}) ->
+    NewState :: #state{}.
+on_tc_fail(TC, Reason, State) ->
+    State.
+
+%% @doc Called when a test case is skipped by either user action
+%% or due to an init function failing. Test case can be
+%% end_per_suite, init_per_group, end_per_group and the actual test cases. 
+-spec on_tc_skip(TC :: end_per_suite |
+		 init_per_group | end_per_group | atom(),
+		 {tc_auto_skip, {failed, {Mod :: atom(), Function :: atom(), Reason :: term()}}} |
+         {tc_user_skip, {skipped, Reason :: term()}},
+          State :: #state{}) ->
+    NewState :: #state{}.
+on_tc_skip(TC, Reason, State) ->
+    State.
+
+%% @doc Called when the scope of the CTH is done, this depends on
+%% when the CTH was specified. This translation table describes when this
+%% function is called.
+%%
+%%  | Started in          |     terminate called    |
+%%  |---------------------|-------------------------|
+%%  | command_line        | after all tests are run |
+%%  | test spec           | after all tests are run |
+%%  | suite/0             | after SUITE is done     |
+%%  | init_per_suite/1    | after SUITE is done     |
+%%  | init_per_group/2    | after group is done     |
+%%  |-----------------------------------------------|
+%%
+-spec terminate(State :: #state{}) ->
+    term().
+terminate(State) ->
+    ok.