aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test
diff options
context:
space:
mode:
authorSiri Hansen <[email protected]>2013-03-25 14:51:26 +0100
committerSiri Hansen <[email protected]>2013-03-28 12:19:57 +0100
commit25df55fa1987313b38a6a28d49571f378acd2223 (patch)
treeb6e85258289d038740d8202f82a56dd90a1ce858 /lib/common_test
parentcea6232b9f5c44f76ff1023aa78ea603cb4ed0d9 (diff)
downloadotp-25df55fa1987313b38a6a28d49571f378acd2223.tar.gz
otp-25df55fa1987313b38a6a28d49571f378acd2223.tar.bz2
otp-25df55fa1987313b38a6a28d49571f378acd2223.zip
[common_test] Add tests for repeated testruns
ct_repeate_testrun_SUITE is added, which tests the following options to ct_run: -repeat N -duration [-force_stop [skip_rest]] -until [-force_stop [skip_rest]]
Diffstat (limited to 'lib/common_test')
-rw-r--r--lib/common_test/test/Makefile1
-rw-r--r--lib/common_test/test/ct_repeat_testrun_SUITE.erl378
-rw-r--r--lib/common_test/test/ct_repeat_testrun_SUITE_data/a_test/r1_SUITE.erl75
-rw-r--r--lib/common_test/test/ct_repeat_testrun_SUITE_data/b_test/r2_SUITE.erl75
4 files changed, 529 insertions, 0 deletions
diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile
index bd746f87a7..e8607a73af 100644
--- a/lib/common_test/test/Makefile
+++ b/lib/common_test/test/Makefile
@@ -38,6 +38,7 @@ MODULES= \
ct_groups_spec_SUITE \
ct_sequence_1_SUITE \
ct_repeat_1_SUITE \
+ ct_repeat_testrun_SUITE \
ct_testspec_1_SUITE \
ct_testspec_2_SUITE \
ct_testspec_3_SUITE \
diff --git a/lib/common_test/test/ct_repeat_testrun_SUITE.erl b/lib/common_test/test/ct_repeat_testrun_SUITE.erl
new file mode 100644
index 0000000000..7ec384c932
--- /dev/null
+++ b/lib/common_test/test/ct_repeat_testrun_SUITE.erl
@@ -0,0 +1,378 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2013. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_repeat_test_SUITE
+%%%
+%%% Description:
+%%% Test different options for repeating test runs:
+%%% -repeat N
+%%% -duration T [-force_stop [skip_rest]]
+%%% -until T [-force_stop [skip_rest]]
+%%%
+%%%-------------------------------------------------------------------
+-module(ct_repeat_testrun_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-define(eh, ct_test_support_eh).
+-define(skip_reason, "Repeated test stopped by force_stop option").
+-define(skipped, {skipped, ?skip_reason}).
+
+
+%% Timers used in this test.
+%% Each test suite consists of
+%%
+%% [tc1,tc2,{group,g,[tc1,tc2]},tc2]
+%%
+%% In r1_SUITE tc1 has a sleep of 10 sec - all other test cases just
+%% return ok.
+%%
+%% => One complete test run of two suites r1_SUITE + r2_SUITE is at
+%% least 20 seconds (10 sec for each r1_SUITE:tc1)
+%%
+-define(t1,30). % time shall expire during second run of r1_SUITE
+-define(t2,6). % time shall expire during first run of tc1
+-define(t3,16). % time shall expire during second run of tc1
+
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% Description: Since Common Test starts another Test Server
+%% instance, the tests need to be performed on a separate node (or
+%% there will be clashes with logging processes etc).
+%%--------------------------------------------------------------------
+init_per_suite(Config0) ->
+ Config = ct_test_support:init_per_suite(Config0),
+ DataDir = ?config(data_dir, Config),
+ Suite1 = filename:join([DataDir,"a_test","r1_SUITE"]),
+ Suite2 = filename:join([DataDir,"b_test","r2_SUITE"]),
+ Opts0 = ct_test_support:get_opts(Config),
+ Opts1 = Opts0 ++ [{suite,Suite1},{testcase,tc2},{label,timing1}],
+ Opts2 = Opts0 ++ [{suite,Suite2},{testcase,tc2},{label,timing2}],
+
+ %% Make sure both suites are compiled
+ {1,0,{0,0}} = ct_test_support:run(ct,run_test,[Opts1],Config),
+ {1,0,{0,0}} = ct_test_support:run(ct,run_test,[Opts2],Config),
+
+ %% Time the shortest testcase to use for offset
+ {T0,{1,0,{0,0}}} = timer:tc(ct_test_support,run,[ct,run_test,[Opts1],Config]),
+
+ %% -2 is to ensure we hit inside the target test case and not after
+% T = round(T0/1000000)-2,
+ T=0,
+ [{offset,T}|Config].
+
+end_per_suite(Config) ->
+ ct_test_support:end_per_suite(Config).
+
+init_per_testcase(TestCase, Config) ->
+ ct_test_support:init_per_testcase(TestCase, Config).
+
+end_per_testcase(TestCase, Config) ->
+ ct_test_support:end_per_testcase(TestCase, Config).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+ [
+ repeat_n,
+ duration,
+ duration_force_stop,
+ duration_force_stop_skip_rest,
+ duration_force_stop_skip_rest_group,
+ until,
+ until_force_stop,
+ until_force_stop_skip_rest,
+ until_force_stop_skip_rest_group
+ ].
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+%%%-----------------------------------------------------------------
+%%%
+repeat_n(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Dirs = filelib:wildcard(filename:join(DataDir,"*")),
+ {Opts,ERPid} = setup([{dir,Dirs},
+ {label,repeat_n},
+ {repeat,2}],
+ Config),
+ ok = execute(repeat_n, Opts, ERPid, Config).
+
+duration(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Dirs = filelib:wildcard(filename:join(DataDir,"*")),
+ {Opts,ERPid} = setup([{dir,Dirs},
+ {label,duration},
+ {duration,duration_str(?t1,2,Config)}],
+ Config),
+ ok = execute(duration, Opts, ERPid, Config).
+
+duration_force_stop(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Dirs = filelib:wildcard(filename:join(DataDir,"*")),
+ {Opts,ERPid} = setup([{dir,Dirs},
+ {label,duration_force_stop},
+ {duration,duration_str(?t1,2,Config)},
+ {force_stop,true}],
+ Config),
+ ok = execute(duration_force_stop, Opts, ERPid, Config).
+
+duration_force_stop_skip_rest(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Dirs = filelib:wildcard(filename:join(DataDir,"*")),
+ {Opts,ERPid} = setup([{dir,Dirs},
+ {label,duration_force_stop_skip_rest},
+ {duration,duration_str(?t2,1,Config)},
+ {force_stop,skip_rest}],
+ Config),
+ ok = execute(duration_force_stop_skip_rest, Opts, ERPid, Config).
+
+duration_force_stop_skip_rest_group(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Dirs = filelib:wildcard(filename:join(DataDir,"*")),
+ {Opts,ERPid} = setup([{dir,Dirs},
+ {label,duration_force_stop_skip_rest_group},
+ {duration,duration_str(?t3,1,Config)},
+ {force_stop,skip_rest}],
+ Config),
+ ok = execute(duration_force_stop_skip_rest_group, Opts, ERPid, Config).
+
+until(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Dirs = filelib:wildcard(filename:join(DataDir,"*")),
+ {Opts,ERPid} = setup([{dir,Dirs},
+ {label,until}],
+ Config),
+ ExecuteFun =
+ fun() ->
+ [_,_] = ct_test_support:run_ct_run_test(
+ Opts++[{until,until_str(?t1,2,Config)}],Config),
+ 0 = ct_test_support:run_ct_script_start(
+ Opts++[{until,until_str(?t1,2,Config)}],Config)
+ end,
+ ok = execute(ExecuteFun, until, Opts, ERPid, Config).
+
+until_force_stop(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Dirs = filelib:wildcard(filename:join(DataDir,"*")),
+ {Opts,ERPid} = setup([{dir,Dirs},
+ {label,until_force_stop},
+ {force_stop,true}],
+ Config),
+ ExecuteFun =
+ fun() ->
+ [_,_] = ct_test_support:run_ct_run_test(
+ Opts++[{until,until_str(?t1,2,Config)}],Config),
+ 0 = ct_test_support:run_ct_script_start(
+ Opts++[{until,until_str(?t1,2,Config)}],Config)
+ end,
+ ok = execute(ExecuteFun, until_force_stop, Opts, ERPid, Config).
+
+until_force_stop_skip_rest(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Dirs = filelib:wildcard(filename:join(DataDir,"*")),
+ {Opts,ERPid} = setup([{dir,Dirs},
+ {label,until_force_stop_skip_rest},
+ {force_stop,skip_rest}],
+ Config),
+ ExecuteFun =
+ fun() ->
+ [_] = ct_test_support:run_ct_run_test(
+ Opts++[{until,until_str(?t2,1,Config)}],Config),
+ 1 = ct_test_support:run_ct_script_start(
+ Opts++[{until,until_str(?t2,1,Config)}],Config)
+ end,
+ ok = execute(ExecuteFun, until_force_stop_skip_rest,
+ Opts, ERPid, Config).
+
+until_force_stop_skip_rest_group(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Dirs = filelib:wildcard(filename:join(DataDir,"*")),
+ {Opts,ERPid} = setup([{dir,Dirs},
+ {label,until_force_stop_skip_rest_group},
+ {force_stop,skip_rest}],
+ Config),
+ ExecuteFun =
+ fun() ->
+ [_] = ct_test_support:run_ct_run_test(
+ Opts++[{until,until_str(?t3,1,Config)}],Config),
+ 0 = ct_test_support:run_ct_script_start(
+ Opts++[{until,until_str(?t3,1,Config)}],Config)
+ end,
+ ok = execute(ExecuteFun,
+ until_force_stop_skip_rest_group,
+ Opts, ERPid, Config).
+
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+
+setup(Test, Config) ->
+ Opts0 = ct_test_support:get_opts(Config),
+ Level = ?config(trace_level, Config),
+ EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+ Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test],
+ ERPid = ct_test_support:start_event_receiver(Config),
+ {Opts,ERPid}.
+
+%% Execute test, first with ct:run_test, then with ct:script_start
+execute(Name, Opts, ERPid, Config) ->
+ ExecuteFun = fun() -> ok = ct_test_support:run(Opts, Config) end,
+ execute(ExecuteFun, Name, Opts, ERPid, Config).
+
+execute(ExecuteFun, Name, Opts, ERPid, Config) ->
+ ExecuteFun(),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(Name,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config),
+ Opts),
+
+ TestEvents = events_to_check(Name),
+ ct_test_support:verify_events(TestEvents, Events, Config).
+
+reformat(Events, EH) ->
+ ct_test_support:reformat(Events, EH).
+
+%% N is the expected number of repeats
+until_str(Secs0,N,Config) ->
+ Offset = ?config(offset,Config),
+ Secs = Secs0 + N*Offset,
+ Now = calendar:datetime_to_gregorian_seconds(calendar:local_time()),
+ {{Y,Mo,D},{H,M,S}} = calendar:gregorian_seconds_to_datetime(Now+Secs),
+ lists:flatten(io_lib:format("~2..0w~2..0w~2..0w~2..0w~2..0w~2..0w",
+ [Y rem 100, Mo, D, H, M, S])).
+
+%% N is the expected number of repeats
+duration_str(Secs0,N,Config) ->
+ Offset = ?config(offset,Config),
+ Secs = Secs0 + N*Offset,
+ "0000" ++ lists:flatten(io_lib:format("~2..0w",[Secs])).
+
+%%%-----------------------------------------------------------------
+%%% TEST EVENTS
+%%%-----------------------------------------------------------------
+%% 2 tests (ct:run_test + script_start) is default
+events_to_check(C) when C==repeat_n; C==duration; C==until ->
+ dupl(4, start_logging() ++ all_succ() ++ stop_logging());
+events_to_check(C) when C==duration_force_stop; C==until_force_stop ->
+ dupl(2, start_logging() ++
+ all_succ() ++
+ stop_logging() ++
+ start_logging() ++
+ all_succ(r1_SUITE) ++
+ stop_logging());
+events_to_check(C) when C==duration_force_stop_skip_rest;
+ C==until_force_stop_skip_rest ->
+ dupl(2, start_logging() ++ skip_first_tc1(r1_SUITE) ++ stop_logging());
+events_to_check(C) when C==duration_force_stop_skip_rest_group;
+ C==until_force_stop_skip_rest_group ->
+ dupl(2, start_logging() ++ skip_tc1_in_group(r1_SUITE) ++ stop_logging()).
+
+dupl(N,List) ->
+ lists:flatten(lists:duplicate(N,List)).
+
+start_logging() ->
+ [{?eh,start_logging,{'DEF','RUNDIR'}}].
+stop_logging() ->
+ [{?eh,stop_logging,[]}].
+
+
+all_succ() ->
+ all_succ(r1_SUITE) ++ all_succ(r2_SUITE).
+
+all_succ(Suite) ->
+ [{?eh,tc_start,{Suite,init_per_suite}},
+ {?eh,tc_done,{Suite,init_per_suite,ok}},
+ {?eh,tc_start,{Suite,tc1}},
+ {?eh,tc_done,{Suite,tc1,ok}},
+ {?eh,test_stats,{'_',0,{0,0}}},
+ {?eh,tc_start,{Suite,tc2}},
+ {?eh,tc_done,{Suite,tc2,ok}},
+ {?eh,test_stats,{'_',0,{0,0}}},
+ [{?eh,tc_start,{Suite,{init_per_group,g,[]}}},
+ {?eh,tc_done,{Suite,{init_per_group,g,[]},ok}},
+ {?eh,tc_start,{Suite,tc1}},
+ {?eh,tc_done,{Suite,tc1,ok}},
+ {?eh,test_stats,{'_',0,{0,0}}},
+ {?eh,tc_start,{Suite,tc2}},
+ {?eh,tc_done,{Suite,tc2,ok}},
+ {?eh,test_stats,{'_',0,{0,0}}},
+ {?eh,tc_start,{Suite,{end_per_group,g,[]}}},
+ {?eh,tc_done,{Suite,{end_per_group,g,[]},ok}}],
+ {?eh,tc_start,{Suite,tc2}},
+ {?eh,tc_done,{Suite,tc2,ok}},
+ {?eh,test_stats,{'_',0,{0,0}}},
+ {?eh,tc_start,{Suite,end_per_suite}},
+ {?eh,tc_done,{Suite,end_per_suite,ok}}].
+
+skip_first_tc1(Suite) ->
+ [{?eh,tc_start,{Suite,init_per_suite}},
+ {?eh,tc_done,{Suite,init_per_suite,ok}},
+ {?eh,tc_start,{Suite,tc1}},
+ {?eh,tc_done,{Suite,tc1,ok}},
+ {?eh,test_stats,{'_',0,{0,0}}},
+ {?eh,tc_done,{Suite,tc2,?skipped}},
+ {?eh,test_stats,{'_',0,{1,0}}},
+ {?eh,tc_done,{Suite,{init_per_group,g,[]},?skipped}},
+ {?eh,tc_auto_skip,{Suite,tc1,?skip_reason}},
+ {?eh,test_stats,{'_',0,{1,1}}},
+ {?eh,tc_auto_skip,{Suite,tc2,?skip_reason}},
+ {?eh,test_stats,{'_',0,{1,2}}},
+ {?eh,tc_auto_skip,{Suite,end_per_group,?skip_reason}},
+ {?eh,tc_done,{Suite,tc2,?skipped}},
+ {?eh,test_stats,{'_',0,{2,2}}},
+ {?eh,tc_start,{Suite,end_per_suite}},
+ {?eh,tc_done,{Suite,end_per_suite,ok}}].
+
+
+skip_tc1_in_group(Suite) ->
+ [{?eh,tc_start,{Suite,init_per_suite}},
+ {?eh,tc_done,{Suite,init_per_suite,ok}},
+ {?eh,tc_start,{Suite,tc1}},
+ {?eh,tc_done,{Suite,tc1,ok}},
+ {?eh,test_stats,{'_',0,{0,0}}},
+ {?eh,tc_start,{Suite,tc2}},
+ {?eh,tc_done,{Suite,tc2,ok}},
+ {?eh,test_stats,{'_',0,{0,0}}},
+ [{?eh,tc_start,{Suite,{init_per_group,g,[]}}},
+ {?eh,tc_done,{Suite,{init_per_group,g,[]},ok}},
+ {?eh,tc_start,{Suite,tc1}},
+ {?eh,tc_done,{Suite,tc1,ok}},
+ {?eh,test_stats,{'_',0,{0,0}}},
+ {?eh,tc_done,{Suite,tc2,?skipped}},
+ {?eh,test_stats,{'_',0,{1,0}}},
+ {?eh,tc_start,{Suite,{end_per_group,g,[]}}},
+ {?eh,tc_done,{Suite,{end_per_group,g,[]},ok}}],
+ {?eh,tc_done,{Suite,tc2,?skipped}},
+ {?eh,test_stats,{'_',0,{2,0}}},
+ {?eh,tc_start,{Suite,end_per_suite}},
+ {?eh,tc_done,{Suite,end_per_suite,ok}}].
diff --git a/lib/common_test/test/ct_repeat_testrun_SUITE_data/a_test/r1_SUITE.erl b/lib/common_test/test/ct_repeat_testrun_SUITE_data/a_test/r1_SUITE.erl
new file mode 100644
index 0000000000..3fd5943691
--- /dev/null
+++ b/lib/common_test/test/ct_repeat_testrun_SUITE_data/a_test/r1_SUITE.erl
@@ -0,0 +1,75 @@
+%%--------------------------------------------------------------------
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2013. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+%%----------------------------------------------------------------------
+%% File: r1_SUITE.erl
+%%
+%% Description:
+%%
+%%
+%% @author Support
+%% @doc
+%% @end
+%%----------------------------------------------------------------------
+%%----------------------------------------------------------------------
+-module(r1_SUITE).
+-include_lib("common_test/include/ct.hrl").
+
+-compile(export_all).
+
+%% Default timetrap timeout (set in init_per_testcase).
+-define(default_timeout, ?t:seconds(30)).
+
+all() ->
+ testcases() ++ [{group,g}, tc2].
+
+groups() ->
+ [{g,testcases()}].
+
+testcases() ->
+ [tc1,tc2].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(Config) ->
+ Config.
+
+init_per_group(_, Config) ->
+ Config.
+
+end_per_group(_Group, Config) ->
+ Config.
+
+init_per_testcase(_Case, Config) ->
+ Dog = test_server:timetrap(?default_timeout),
+ [{watchdog, Dog}|Config].
+
+end_per_testcase(_Case, Config) ->
+ Dog=?config(watchdog, Config),
+ test_server:timetrap_cancel(Dog),
+ ok.
+
+%%%-----------------------------------------------------------------
+%%% Test cases
+tc1(_Config) ->
+ timer:sleep(10000),
+ ok.
+
+tc2(_Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_repeat_testrun_SUITE_data/b_test/r2_SUITE.erl b/lib/common_test/test/ct_repeat_testrun_SUITE_data/b_test/r2_SUITE.erl
new file mode 100644
index 0000000000..dc9abc2863
--- /dev/null
+++ b/lib/common_test/test/ct_repeat_testrun_SUITE_data/b_test/r2_SUITE.erl
@@ -0,0 +1,75 @@
+%%--------------------------------------------------------------------
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2013. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+%%----------------------------------------------------------------------
+%% File: r2_SUITE.erl
+%%
+%% Description:
+%%
+%%
+%% @author Support
+%% @doc
+%% @end
+%%----------------------------------------------------------------------
+%%----------------------------------------------------------------------
+-module(r2_SUITE).
+-include_lib("common_test/include/ct.hrl").
+
+-compile(export_all).
+
+%% Default timetrap timeout (set in init_per_testcase).
+-define(default_timeout, ?t:seconds(30)).
+
+all() ->
+ testcases() ++ [{group,g}, tc2].
+
+groups() ->
+ [{g,testcases()}].
+
+testcases() ->
+ [tc1,tc2].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(Config) ->
+ Config.
+
+init_per_group(_, Config) ->
+ Config.
+
+end_per_group(_Group, Config) ->
+ Config.
+
+init_per_testcase(_Case, Config) ->
+ Dog = test_server:timetrap(?default_timeout),
+ [{watchdog, Dog}|Config].
+
+end_per_testcase(_Case, Config) ->
+ Dog=?config(watchdog, Config),
+ test_server:timetrap_cancel(Dog),
+ ok.
+
+%%%-----------------------------------------------------------------
+%%% Test cases
+tc1(_Config) ->
+ %% timer:sleep(3000),
+ ok.
+
+tc2(_Config) ->
+ ok.