%%
%% %CopyrightBegin%
%%
%% Copyright Ericsson AB 2013-2017. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%
%% %CopyrightEnd%
%%
%%%-------------------------------------------------------------------
%%% File: ct_repeat_test_SUITE
%%%
%%% Description:
%%% Test different options for repeating test runs:
%%% -repeat N
%%% -duration T [-force_stop [skip_rest]]
%%% -until T [-force_stop [skip_rest]]
%%%
%%%-------------------------------------------------------------------
-module(ct_repeat_testrun_SUITE).
-compile(export_all).
-include_lib("common_test/include/ct.hrl").
-include_lib("common_test/include/ct_event.hrl").
-define(eh, ct_test_support_eh).
-define(skip_reason, "Repeated test stopped by force_stop option").
-define(skipped, {auto_skipped, ?skip_reason}).
%% Timers used in this test.
%% Each test suite consists of
%%
%% [tc1,tc2,{group,g,[tc1,tc2]},tc2]
%%
%% In r1_SUITE tc1 has a sleep of 10 sec - all other test cases just
%% return ok.
%%
%% => One complete test run of two suites r1_SUITE + r2_SUITE is at
%% least 20 seconds (10 sec for each r1_SUITE:tc1)
%%
-define(t1,30). % time shall expire during second run of r1_SUITE
-define(t2,9). % time shall expire during first run of tc1
-define(t3,19). % time shall expire during second run of tc1
%%--------------------------------------------------------------------
%% TEST SERVER CALLBACK FUNCTIONS
%%--------------------------------------------------------------------
%%--------------------------------------------------------------------
%% Description: Since Common Test starts another Test Server
%% instance, the tests need to be performed on a separate node (or
%% there will be clashes with logging processes etc).
%%--------------------------------------------------------------------
init_per_suite(Config0) ->
TTInfo = {_T,{_Scaled,ScaleVal}} = ct:get_timetrap_info(),
ct:pal("Timetrap info = ~w", [TTInfo]),
if ScaleVal > 1 ->
{skip,"Skip on systems running e.g. cover or debug!"};
ScaleVal =< 1 ->
Config = ct_test_support:init_per_suite(Config0),
DataDir = ?config(data_dir, Config),
Suite1 = filename:join([DataDir,"a_test","r1_SUITE"]),
Suite2 = filename:join([DataDir,"b_test","r2_SUITE"]),
Opts0 = ct_test_support:get_opts(Config),
Opts1 = Opts0 ++ [{suite,Suite1},{testcase,tc2},{label,timing1}],
Opts2 = Opts0 ++ [{suite,Suite2},{testcase,tc2},{label,timing2}],
%% Make sure both suites are compiled
{1,0,{0,0}} = ct_test_support:run(ct,run_test,[Opts1],Config),
{1,0,{0,0}} = ct_test_support:run(ct,run_test,[Opts2],Config),
%% Check if file i/o is too slow for correct measurements
Opts3 = Opts0 ++ [{suite,Suite1},{testcase,tc1},{label,timing3}],
{T,_} =
timer:tc(
fun() ->
{1,0,{0,0}} = ct_test_support:run(ct,run_test,
[Opts3],Config),
{1,0,{0,0}} = ct_test_support:run(ct,run_test,
[Opts3],Config)
end),
%% The time to compare with here must match the timeout value
%% in the test suite. Accept 30% logging overhead (26 sec total).
if T > 26000000 ->
ct:pal("Timing test took ~w sec (< 27 sec expected). "
"Skipping the suite!",
[trunc(T/1000000)]),
ct_test_support:end_per_suite(Config),
{skip,"File I/O too slow for this suite"};
true ->
ct:pal("Timing test took ~w sec. Proceeding...",
[trunc(T/1000000)]),
[{offset,0}|Config]
end
end.
end_per_suite(Config) ->
ct_test_support:end_per_suite(Config).
init_per_testcase(TestCase, Config) ->
ct_test_support:init_per_testcase(TestCase, Config).
end_per_testcase(TestCase, Config) ->
ct_test_support:end_per_testcase(TestCase, Config).
suite() -> [{ct_hooks,[ts_install_cth]}].
all() ->
[
repeat_n,
duration,
duration_force_stop,
duration_force_stop_skip_rest,
duration_force_stop_skip_rest_group,
until,
until_force_stop,
until_force_stop_skip_rest,
until_force_stop_skip_rest_group
].
%%--------------------------------------------------------------------
%% TEST CASES
%%--------------------------------------------------------------------
%%%-----------------------------------------------------------------
%%%
repeat_n(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
Dirs = filelib:wildcard(filename:join(DataDir,"*")),
{Opts,ERPid} = setup([{dir,Dirs},
{label,repeat_n},
{repeat,2}],
Config),
ok = execute(repeat_n, Opts, ERPid, Config).
duration(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
Dirs = filelib:wildcard(filename:join(DataDir,"*")),
{Opts,ERPid} = setup([{dir,Dirs},
{label,duration},
{duration,duration_str(?t1,2,Config)}],
Config),
ok = execute(duration, Opts, ERPid, Config).
duration_force_stop(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
Dirs = filelib:wildcard(filename:join(DataDir,"*")),
{Opts,ERPid} = setup([{dir,Dirs},
{label,duration_force_stop},
{duration,duration_str(?t1,2,Config)},
{force_stop,true}],
Config),
ok = execute(duration_force_stop, Opts, ERPid, Config).
duration_force_stop_skip_rest(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
Dirs = filelib:wildcard(filename:join(DataDir,"*")),
{Opts,ERPid} = setup([{dir,Dirs},
{label,duration_force_stop_skip_rest},
{duration,duration_str(?t2,1,Config)},
{force_stop,skip_rest}],
Config),
ok = execute(duration_force_stop_skip_rest, Opts, ERPid, Config).
duration_force_stop_skip_rest_group(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
Dirs = filelib:wildcard(filename:join(DataDir,"*")),
{Opts,ERPid} = setup([{dir,Dirs},
{label,duration_force_stop_skip_rest_group},
{duration,duration_str(?t3,1,Config)},
{force_stop,skip_rest}],
Config),
ok = execute(duration_force_stop_skip_rest_group, Opts, ERPid, Config).
until(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
Dirs = filelib:wildcard(filename:join(DataDir,"*")),
{Opts,ERPid} = setup([{dir,Dirs},
{label,until}],
Config),
ExecuteFun =
fun() ->
[_,_] = ct_test_support:run_ct_run_test(
Opts++[{until,until_str(?t1,2,Config)}],Config),
0 = ct_test_support:run_ct_script_start(
Opts++[{until,until_str(?t1,2,Config)}],Config)
end,
ok = execute(ExecuteFun, until, Opts, ERPid, Config).
until_force_stop(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
Dirs = filelib:wildcard(filename:join(DataDir,"*")),
{Opts,ERPid} = setup([{dir,Dirs},
{label,until_force_stop},
{force_stop,true}],
Config),
ExecuteFun =
fun() ->
[_,_] = ct_test_support:run_ct_run_test(
Opts++[{until,until_str(?t1,2,Config)}],Config),
0 = ct_test_support:run_ct_script_start(
Opts++[{until,until_str(?t1,2,Config)}],Config)
end,
ok = execute(ExecuteFun, until_force_stop, Opts, ERPid, Config).
until_force_stop_skip_rest(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
Dirs = filelib:wildcard(filename:join(DataDir,"*")),
{Opts,ERPid} = setup([{dir,Dirs},
{label,until_force_stop_skip_rest},
{force_stop,skip_rest}],
Config),
ExecuteFun =
fun() ->
[_] = ct_test_support:run_ct_run_test(
Opts++[{until,until_str(?t2,1,Config)}],Config),
1 = ct_test_support:run_ct_script_start(
Opts++[{until,until_str(?t2,1,Config)}],Config)
end,
ok = execute(ExecuteFun, until_force_stop_skip_rest,
Opts, ERPid, Config).
until_force_stop_skip_rest_group(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
Dirs = filelib:wildcard(filename:join(DataDir,"*")),
{Opts,ERPid} = setup([{dir,Dirs},
{label,until_force_stop_skip_rest_group},
{force_stop,skip_rest}],
Config),
ExecuteFun =
fun() ->
[_] = ct_test_support:run_ct_run_test(
Opts++[{until,until_str(?t3,1,Config)}],Config),
1 = ct_test_support:run_ct_script_start(
Opts++[{until,until_str(?t3,1,Config)}],Config)
end,
ok = execute(ExecuteFun,
until_force_stop_skip_rest_group,
Opts, ERPid, Config).
%%%-----------------------------------------------------------------
%%% HELP FUNCTIONS
%%%-----------------------------------------------------------------
setup(Test, Config) ->
Opts0 = ct_test_support:get_opts(Config),
Level = ?config(trace_level, Config),
EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test],
ERPid = ct_test_support:start_event_receiver(Config),
{Opts,ERPid}.
%% Execute test, first with ct:run_test, then with ct:script_start
execute(Name, Opts, ERPid, Config) ->
ExecuteFun = fun() -> ok = ct_test_support:run(Opts, Config) end,
execute(ExecuteFun, Name, Opts, ERPid, Config).
execute(ExecuteFun, Name, Opts, ERPid, Config) ->
ExecuteFun(),
Events = ct_test_support:get_events(ERPid, Config),
ct_test_support:log_events(Name,
reformat(Events, ?eh),
?config(priv_dir, Config),
Opts),
TestEvents = events_to_check(Name),
ct_test_support:verify_events(TestEvents, Events, Config).
reformat(Events, EH) ->
ct_test_support:reformat(Events, EH).
%% N is the expected number of repeats
until_str(Secs0,N,Config) ->
Offset = ?config(offset,Config),
Secs = Secs0 + N*Offset,
Now = calendar:datetime_to_gregorian_seconds(calendar:local_time()),
{{Y,Mo,D},{H,M,S}} = calendar:gregorian_seconds_to_datetime(Now+Secs),
lists:flatten(io_lib:format("~2..0w~2..0w~2..0w~2..0w~2..0w~2..0w",
[Y rem 100, Mo, D, H, M, S])).
%% N is the expected number of repeats
duration_str(Secs0,N,Config) ->
Offset = ?config(offset,Config),
Secs = Secs0 + N*Offset,
"0000" ++ lists:flatten(io_lib:format("~2..0w",[Secs])).
%%%-----------------------------------------------------------------
%%% TEST EVENTS
%%%-----------------------------------------------------------------
%% 2 tests (ct:run_test + script_start) is default
events_to_check(C) when C==repeat_n; C==duration; C==until ->
dupl(4, start_logging() ++ all_succ() ++ stop_logging());
events_to_check(C) when C==duration_force_stop; C==until_force_stop ->
dupl(2, start_logging() ++
all_succ() ++
stop_logging() ++
start_logging() ++
all_succ(r1_SUITE) ++
stop_logging());
events_to_check(C) when C==duration_force_stop_skip_rest;
C==until_force_stop_skip_rest ->
dupl(2, start_logging() ++ skip_first_tc1(r1_SUITE) ++ stop_logging());
events_to_check(C) when C==duration_force_stop_skip_rest_group;
C==until_force_stop_skip_rest_group ->
dupl(2, start_logging() ++ skip_tc1_in_group(r1_SUITE) ++ stop_logging()).
dupl(N,List) ->
lists:flatten(lists:duplicate(N,List)).
start_logging() ->
[{?eh,start_logging,{'DEF','RUNDIR'}}].
stop_logging() ->
[{?eh,stop_logging,[]}].
all_succ() ->
all_succ(r1_SUITE) ++ all_succ(r2_SUITE).
all_succ(Suite) ->
[{?eh,tc_start,{Suite,init_per_suite}},
{?eh,tc_done,{Suite,init_per_suite,ok}},
{?eh,tc_start,{Suite,tc1}},
{?eh,tc_done,{Suite,tc1,ok}},
{?eh,test_stats,{'_',0,{0,0}}},
{?eh,tc_start,{Suite,tc2}},
{?eh,tc_done,{Suite,tc2,ok}},
{?eh,test_stats,{'_',0,{0,0}}},
[{?eh,tc_start,{Suite,{init_per_group,g,[]}}},
{?eh,tc_done,{Suite,{init_per_group,g,[]},ok}},
{?eh,tc_start,{Suite,tc1}},
{?eh,tc_done,{Suite,tc1,ok}},
{?eh,test_stats,{'_',0,{0,0}}},
{?eh,tc_start,{Suite,tc2}},
{?eh,tc_done,{Suite,tc2,ok}},
{?eh,test_stats,{'_',0,{0,0}}},
{?eh,tc_start,{Suite,{end_per_group,g,[]}}},
{?eh,tc_done,{Suite,{end_per_group,g,[]},ok}}],
{?eh,tc_start,{Suite,tc2}},
{?eh,tc_done,{Suite,tc2,ok}},
{?eh,test_stats,{'_',0,{0,0}}},
{?eh,tc_start,{Suite,end_per_suite}},
{?eh,tc_done,{Suite,end_per_suite,ok}}].
skip_first_tc1(Suite) ->
[{?eh,tc_start,{Suite,init_per_suite}},
{?eh,tc_done,{Suite,init_per_suite,ok}},
{?eh,tc_start,{Suite,tc1}},
{?eh,tc_done,{Suite,tc1,ok}},
{?eh,test_stats,{'_',0,{0,0}}},
{?eh,tc_start,{Suite,tc2}},
{?eh,tc_done,{Suite,tc2,?skipped}},
{?eh,test_stats,{'_',0,{0,1}}},
{?eh,tc_start,{Suite,{init_per_group,g,[]}}},
{?eh,tc_done,{Suite,{init_per_group,g,[]},?skipped}},
{?eh,tc_auto_skip,{Suite,{tc1,g},?skip_reason}},
{?eh,test_stats,{'_',0,{0,2}}},
{?eh,tc_auto_skip,{Suite,{tc2,g},?skip_reason}},
{?eh,test_stats,{'_',0,{0,3}}},
{?eh,tc_auto_skip,{Suite,{end_per_group,g},?skip_reason}},
{?eh,tc_start,{Suite,tc2}},
{?eh,tc_done,{Suite,tc2,?skipped}},
{?eh,test_stats,{'_',0,{0,4}}},
{?eh,tc_start,{Suite,end_per_suite}},
{?eh,tc_done,{Suite,end_per_suite,ok}}].
skip_tc1_in_group(Suite) ->
[{?eh,tc_start,{Suite,init_per_suite}},
{?eh,tc_done,{Suite,init_per_suite,ok}},
{?eh,tc_start,{Suite,tc1}},
{?eh,tc_done,{Suite,tc1,ok}},
{?eh,test_stats,{'_',0,{0,0}}},
{?eh,tc_start,{Suite,tc2}},
{?eh,tc_done,{Suite,tc2,ok}},
{?eh,test_stats,{'_',0,{0,0}}},
[{?eh,tc_start,{Suite,{init_per_group,g,[]}}},
{?eh,tc_done,{Suite,{init_per_group,g,[]},ok}},
{?eh,tc_start,{Suite,tc1}},
{?eh,tc_done,{Suite,tc1,ok}},
{?eh,test_stats,{'_',0,{0,0}}},
{?eh,tc_start,{Suite,tc2}},
{?eh,tc_done,{Suite,tc2,?skipped}},
{?eh,test_stats,{'_',0,{0,1}}},
{?eh,tc_start,{Suite,{end_per_group,g,[]}}},
{?eh,tc_done,{Suite,{end_per_group,g,[]},ok}}],
{?eh,tc_start,{Suite,tc2}},
{?eh,tc_done,{Suite,tc2,?skipped}},
{?eh,test_stats,{'_',0,{0,2}}},
{?eh,tc_start,{Suite,end_per_suite}},
{?eh,tc_done,{Suite,end_per_suite,ok}}].
