aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test
diff options
context:
space:
mode:
authorPeter Andersson <peppe@erlang.org>2013-01-27 02:26:52 +0100
committerPeter Andersson <peppe@erlang.org>2013-01-27 02:27:50 +0100
commitcb178e9743baa5e3923d561e01b06972ac2a3d59 (patch)
tree04024ef5777a6e91bce6f09c8fcdb9b5a56648a9 /lib/common_test
parent57abbf21ff6ce0a97182f848177338c1e2bceab2 (diff)
parent58f3ac07d68fdf32c1e4e201d409523eb0ddc4fa (diff)
downloadotp-cb178e9743baa5e3923d561e01b06972ac2a3d59.tar.gz
otp-cb178e9743baa5e3923d561e01b06972ac2a3d59.tar.bz2
otp-cb178e9743baa5e3923d561e01b06972ac2a3d59.zip
Merge remote branch 'origin/peppe/common_test/restore_config'
* origin/peppe/common_test/restore_config: Add verification terms in test suite Add test case to verify that config data gets properly restored OTP-10070
Diffstat (limited to 'lib/common_test')
-rw-r--r--lib/common_test/test/ct_error_SUITE.erl65
-rw-r--r--lib/common_test/test/ct_error_SUITE_data/error/test/config_restored_SUITE.erl175
-rw-r--r--lib/common_test/test/ct_error_SUITE_data/error/test/verify_config.erl239
3 files changed, 476 insertions, 3 deletions
diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl
index 6d90b29f41..86ec71c98e 100644
--- a/lib/common_test/test/ct_error_SUITE.erl
+++ b/lib/common_test/test/ct_error_SUITE.erl
@@ -44,8 +44,12 @@
%% there will be clashes with logging processes etc).
%%--------------------------------------------------------------------
init_per_suite(Config) ->
- Config1 = ct_test_support:init_per_suite(Config),
- Config1.
+ DataDir = ?config(data_dir, Config),
+ TestDir = filename:join(DataDir, "error/test/"),
+ CTH = filename:join(TestDir, "verify_config.erl"),
+ ct:pal("Compiling ~p: ~p",
+ [CTH,compile:file(CTH,[{outdir,TestDir},debug_info])]),
+ ct_test_support:init_per_suite([{path_dirs,[TestDir]} | Config]).
end_per_suite(Config) ->
ct_test_support:end_per_suite(Config).
@@ -61,7 +65,8 @@ suite() -> [{ct_hooks,[ts_install_cth]}].
all() ->
[cfg_error, lib_error, no_compile, timetrap_end_conf,
timetrap_normal, timetrap_extended, timetrap_parallel,
- timetrap_fun, timetrap_fun_group, misc_errors].
+ timetrap_fun, timetrap_fun_group, misc_errors,
+ config_restored].
groups() ->
[].
@@ -285,6 +290,24 @@ misc_errors(Config) when is_list(Config) ->
TestEvents = events_to_check(misc_errors),
ok = ct_test_support:verify_events(TestEvents, Events, Config).
+%%%-----------------------------------------------------------------
+%%%
+config_restored(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Suite = filename:join(DataDir, "error/test/config_restored_SUITE"),
+ {Opts,ERPid} = setup([{suite,Suite},
+ {ct_hooks,[verify_config]}],
+ Config),
+ ok = ct_test_support:run(Opts, Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(config_restored,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config),
+ Opts),
+
+ TestEvents = events_to_check(config_restored),
+ ok = ct_test_support:verify_events(TestEvents, Events, Config).
%%%-----------------------------------------------------------------
%%% HELP FUNCTIONS
@@ -1439,4 +1462,40 @@ test_events(misc_errors) ->
{?eh,test_stats,{2,7,{0,0}}},
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}
+ ];
+
+test_events(config_restored) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,4}},
+ {?eh,tc_start,{config_restored_SUITE,init_per_suite}},
+ {?eh,tc_done,{config_restored_SUITE,init_per_suite,ok}},
+ {?eh,tc_start,{config_restored_SUITE,to_tc}},
+ {?eh,cth,{verify_config,post_end_per_testcase,{to_tc,diff_ok}}},
+ {?eh,tc_done,
+ {config_restored_SUITE,to_tc,{failed,{timetrap_timeout,1000}}}},
+ {?eh,test_stats,{0,1,{0,0}}},
+ {?eh,tc_start,{config_restored_SUITE,exit_tc}},
+ {?eh,cth,{verify_config,post_end_per_testcase,{exit_tc,diff_ok}}},
+ {?eh,tc_done,{config_restored_SUITE,exit_tc,
+ {failed,{error,{test_case_failed,"Goodbye!"}}}}},
+ {?eh,test_stats,{0,2,{0,0}}},
+ [{?eh,tc_start,{config_restored_SUITE,{init_per_group,g1,[]}}},
+ {?eh,tc_start,{config_restored_SUITE,to_tc}},
+ {?eh,cth,{verify_config,post_end_per_testcase,{to_tc,diff_ok}}},
+ {?eh,tc_done,
+ {config_restored_SUITE,to_tc,{failed,{timetrap_timeout,1000}}}},
+ {?eh,test_stats,{0,3,{0,0}}},
+ {?eh,tc_start,{config_restored_SUITE,exit_tc}},
+ {?eh,cth,{verify_config,post_end_per_testcase,{exit_tc,diff_ok}}},
+ {?eh,tc_done,{config_restored_SUITE,exit_tc,
+ {failed,{error,{test_case_failed,"Goodbye!"}}}}},
+ {?eh,test_stats,{0,4,{0,0}}},
+ {?eh,tc_start,{config_restored_SUITE,{end_per_group,g1,[]}}},
+ {?eh,tc_done,{config_restored_SUITE,{end_per_group,g1,[]},ok}}],
+ {?eh,tc_start,{config_restored_SUITE,end_per_suite}},
+ {?eh,tc_done,{config_restored_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}
].
diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/config_restored_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/config_restored_SUITE.erl
new file mode 100644
index 0000000000..fb47f2bcb9
--- /dev/null
+++ b/lib/common_test/test/ct_error_SUITE_data/error/test/config_restored_SUITE.erl
@@ -0,0 +1,175 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(config_restored_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% Function: suite() -> Info
+%% Info = [tuple()]
+%%--------------------------------------------------------------------
+suite() ->
+ [{timetrap,1000}].
+
+%%--------------------------------------------------------------------
+%% Function: init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config1 = [{init_per_suite,?MODULE} | Config],
+ TabPid = spawn(fun() ->
+ ets:new(?MODULE, [named_table, set, public]),
+ receive _ -> ok end
+ end),
+ [{tab,TabPid} | Config1].
+
+%%--------------------------------------------------------------------
+%% Function: end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_suite(Config) ->
+ ets:delete(?MODULE),
+ exit(?config(tab, Config), kill),
+ ok.
+
+%%--------------------------------------------------------------------
+%% Function: init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_group(GroupName, Config) ->
+ [{init_per_group,GroupName} | Config].
+
+%%--------------------------------------------------------------------
+%% Function: end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% Function: init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_testcase(TC, Config) ->
+ Config1 = [{init_per_testcase,TC} | Config],
+ ets:insert(?MODULE, {config,Config}),
+ %% ct:pal("Config after init_per_testcase(~w) = ~p", [TC,Config1]),
+ Config1.
+
+%%--------------------------------------------------------------------
+%% Function: end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_testcase(TC, Config) ->
+ ct:pal("Config in end_per_testcase(~w) = ~p", [TC,Config]),
+ [{_,MemConfig}] = ets:lookup(?MODULE, config),
+ diff_config(Config, MemConfig, [tc_status]),
+ ?MODULE = proplists:get_value(init_per_suite, Config),
+ TC = proplists:get_value(init_per_testcase, Config),
+ case ?config(tc_group_properties, Config) of
+ undefined ->
+ ok;
+ Props ->
+ GName = proplists:get_value(name, Props),
+ GName = proplists:get_value(init_per_group, Config)
+ end,
+ ok.
+
+%%--------------------------------------------------------------------
+%% Function: groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%%--------------------------------------------------------------------
+groups() ->
+ [{g1,[],[to_tc, exit_tc]}].
+
+%%--------------------------------------------------------------------
+%% Function: all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%%--------------------------------------------------------------------
+all() ->
+ [to_tc, exit_tc,
+ {group,g1}].
+
+
+to_tc(Config) ->
+ %% ct:pal("Config for to_tc = ~p", [Config]),
+ [{_,MemConfig}] = ets:lookup(?MODULE, config),
+ diff_config(Config, MemConfig, []),
+ ct:sleep(2000).
+
+exit_tc(Config) ->
+ %% ct:pal("Config for exit_tc = ~p", [Config]),
+ [{_,MemConfig}] = ets:lookup(?MODULE, config),
+ diff_config(Config, MemConfig, []),
+ ct:fail("Goodbye!").
+
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+
+diff_config(Cfg1, Cfg2, DiffKeys) ->
+ diff_config(Cfg1, Cfg2, DiffKeys, []).
+
+diff_config([{K,V} | Cfg1], Cfg2, DiffKeys, RemKeys) ->
+ case proplists:get_value(K, Cfg2) of
+ undefined ->
+ diff_config(Cfg1, Cfg2, proplists:delete(K, DiffKeys), RemKeys);
+ V ->
+ diff_config(Cfg1, proplists:delete(K, Cfg2), DiffKeys, RemKeys);
+ _Other ->
+ case proplists:is_defined(K, DiffKeys) of
+ true ->
+ diff_config(Cfg1, Cfg2, proplists:delete(K, DiffKeys), RemKeys);
+ false ->
+ diff_config(Cfg1, Cfg2, DiffKeys, [K | RemKeys])
+ end
+ end;
+diff_config([], [], [], []) ->
+ ct:pal("Diff ok!", []),
+ ok;
+diff_config([], Cfg2, DiffKeys, RemKeys) ->
+ Result = {diff_failed, {cfg2,Cfg2}, {diffkeys,DiffKeys}, {remkeys,RemKeys}},
+ ct:pal("Diff failed! Result = ~p", [Result]),
+ exit(Result).
+
diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/verify_config.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/verify_config.erl
new file mode 100644
index 0000000000..40a54b9f99
--- /dev/null
+++ b/lib/common_test/test/ct_error_SUITE_data/error/test/verify_config.erl
@@ -0,0 +1,239 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%% @doc Common Test Example Suite Callback module.
+%%%
+%%% <p>This module gives an example of a common test CTH (Common Test Hook).
+%%% There are many ways to add a CTH to a test run, you can do it either in
+%%% the command line using -ct_hook, in a test spec using
+%%% {ct_hook,M} or in the suite it self by returning ct_hook
+%%% from either suite/0, init_per_suite/1, init_per_group/2 and
+%%% init_per_testcase/2. The scope of the CTH is determined by where is it
+%%% started. If it is started in the command line or test spec then it will
+%%% be stopped at the end of all tests. If it is started in init_per_suite,
+%%% it will be stopped after end_per_suite and so on. See terminate
+%%% documentation for a table describing the scoping machanics.
+%%%
+%%% All of callbacks except init/1 in a CTH are optional.</p>
+
+-module(verify_config).
+
+%% CT Hooks
+-export([id/1]).
+-export([init/2]).
+
+-export([pre_init_per_suite/3]).
+-export([post_init_per_suite/4]).
+-export([pre_end_per_suite/3]).
+-export([post_end_per_suite/4]).
+
+-export([pre_init_per_group/3]).
+-export([post_init_per_group/4]).
+-export([pre_end_per_group/3]).
+-export([post_end_per_group/4]).
+
+-export([pre_init_per_testcase/3]).
+-export([post_end_per_testcase/4]).
+
+-export([on_tc_fail/3]).
+-export([on_tc_skip/3]).
+
+-export([terminate/1]).
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-type config() :: proplists:proplist().
+-type reason() :: term().
+-type skip_or_fail() :: {skip, reason()} |
+ {auto_skip, reason()} |
+ {fail, reason()} |
+ {'EXIT',reason()}.
+
+-record(state, { id = ?MODULE :: term()}).
+
+%% @doc Always called before any other callback function. Use this to initiate
+%% any common state. It should return an state for this CTH.
+-spec init(Id :: term(), Opts :: proplists:proplist()) ->
+ {ok, State :: #state{}}.
+init(Id, Opts) ->
+ {ok,Opts}.
+
+%% @doc The ID is used to uniquly identify an CTH instance, if two CTH's
+%% return the same ID the seconds CTH is ignored. This function should NOT
+%% have any side effects as it might be called multiple times by common test.
+-spec id(Opts :: proplists:proplist()) ->
+ Id :: term().
+id(Opts) ->
+ now().
+
+%% @doc Called before init_per_suite is called. Note that this callback is
+%% only called if the CTH is added before init_per_suite is run (eg. in a test
+%% specification, suite/0 function etc).
+%% You can change the config in the this function.
+-spec pre_init_per_suite(Suite :: atom(),
+ Config :: config(),
+ State :: #state{}) ->
+ {config() | skip_or_fail(), NewState :: #state{}}.
+pre_init_per_suite(Suite,Config,State) ->
+ {Config, State}.
+
+%% @doc Called after init_per_suite.
+%% you can change the return value in this function.
+-spec post_init_per_suite(Suite :: atom(),
+ Config :: config(),
+ Return :: config() | skip_or_fail(),
+ State :: #state{}) ->
+ {config() | skip_or_fail(), NewState :: #state{}}.
+post_init_per_suite(Suite,Config,Return,State) ->
+ {Return, State}.
+
+%% @doc Called before end_per_suite. The config/state can be changed here,
+%% though it will only affect the *end_per_suite function.
+-spec pre_end_per_suite(Suite :: atom(),
+ Config :: config() | skip_or_fail(),
+ State :: #state{}) ->
+ {ok | skip_or_fail(), NewState :: #state{}}.
+pre_end_per_suite(Suite,Config,State) ->
+ {Config, State}.
+
+%% @doc Called after end_per_suite. Note that the config cannot be
+%% changed here, only the status of the suite.
+-spec post_end_per_suite(Suite :: atom(),
+ Config :: config(),
+ Return :: term(),
+ State :: #state{}) ->
+ {ok | skip_or_fail(), NewState :: #state{}}.
+post_end_per_suite(Suite,Config,Return,State) ->
+ {Return, State}.
+
+%% @doc Called before each init_per_group.
+%% You can change the config in this function.
+-spec pre_init_per_group(Group :: atom(),
+ Config :: config(),
+ State :: #state{}) ->
+ {config() | skip_or_fail(), NewState :: #state{}}.
+pre_init_per_group(Group,Config,State) ->
+ {Config, State}.
+
+%% @doc Called after each init_per_group.
+%% You can change the return value in this function.
+-spec post_init_per_group(Group :: atom(),
+ Config :: config(),
+ Return :: config() | skip_or_fail(),
+ State :: #state{}) ->
+ {config() | skip_or_fail(), NewState :: #state{}}.
+post_init_per_group(Group,Config,Return,State) ->
+ {Return, State}.
+
+%% @doc Called after each end_per_group. The config/state can be changed here,
+%% though it will only affect the *end_per_group functions.
+-spec pre_end_per_group(Group :: atom(),
+ Config :: config() | skip_or_fail(),
+ State :: #state{}) ->
+ {ok | skip_or_fail(), NewState :: #state{}}.
+pre_end_per_group(Group,Config,State) ->
+ {Config, State}.
+
+%% @doc Called after each end_per_group. Note that the config cannot be
+%% changed here, only the status of the group.
+-spec post_end_per_group(Group :: atom(),
+ Config :: config(),
+ Return :: term(),
+ State :: #state{}) ->
+ {ok | skip_or_fail(), NewState :: #state{}}.
+post_end_per_group(Group,Config,Return,State) ->
+ {Return, State}.
+
+%% @doc Called before each test case.
+%% You can change the config in this function.
+-spec pre_init_per_testcase(TC :: atom(),
+ Config :: config(),
+ State :: #state{}) ->
+ {config() | skip_or_fail(), NewState :: #state{}}.
+pre_init_per_testcase(TC,Config,State) ->
+ {Config, State}.
+
+%% @doc Called after each test case. Note that the config cannot be
+%% changed here, only the status of the test case.
+-spec post_end_per_testcase(TC :: atom(),
+ Config :: config(),
+ Return :: term(),
+ State :: #state{}) ->
+ {ok | skip_or_fail(), NewState :: #state{}}.
+post_end_per_testcase(TC,Config,Return,State) ->
+ %% check that config has been restored
+ ct:pal("Config in verify_config:post_end_per_testcase(~w) = ~p",
+ [TC,Config]),
+ [{_,MemConfig}] = ets:lookup(config_restored_SUITE, config),
+ try config_restored_SUITE:diff_config(Config, MemConfig, [tc_status]) of
+ ok ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, post_end_per_testcase,
+ {TC,diff_ok}}})
+ catch
+ _:_ ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, post_end_per_testcase,
+ {TC,diff_failed}}})
+ end,
+ {Return, State}.
+
+%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group,
+%% post_end_per_group and post_end_per_tc if the suite, group or test case failed.
+%% This function should be used for extra cleanup which might be needed.
+%% It is not possible to modify the config or the status of the test run.
+-spec on_tc_fail(TC :: init_per_suite | end_per_suite |
+ init_per_group | end_per_group | atom(),
+ Reason :: term(), State :: #state{}) ->
+ NewState :: #state{}.
+on_tc_fail(TC, Reason, State) ->
+ State.
+
+%% @doc Called when a test case is skipped by either user action
+%% or due to an init function failing. Test case can be
+%% end_per_suite, init_per_group, end_per_group and the actual test cases.
+-spec on_tc_skip(TC :: end_per_suite |
+ init_per_group | end_per_group | atom(),
+ {tc_auto_skip, {failed, {Mod :: atom(), Function :: atom(), Reason :: term()}}} |
+ {tc_user_skip, {skipped, Reason :: term()}},
+ State :: #state{}) ->
+ NewState :: #state{}.
+on_tc_skip(TC, Reason, State) ->
+ State.
+
+%% @doc Called when the scope of the CTH is done, this depends on
+%% when the CTH was specified. This translation table describes when this
+%% function is called.
+%%
+%% | Started in | terminate called |
+%% |---------------------|-------------------------|
+%% | command_line | after all tests are run |
+%% | test spec | after all tests are run |
+%% | suite/0 | after SUITE is done |
+%% | init_per_suite/1 | after SUITE is done |
+%% | init_per_group/2 | after group is done |
+%% |-----------------------------------------------|
+%%
+-spec terminate(State :: #state{}) ->
+ term().
+terminate(State) ->
+ ok.