aboutsummaryrefslogblamecommitdiffstats
path: root/lib/common_test/test/ct_repeat_testrun_SUITE.erl
blob: f8b6a379f67986b462de14b484c85cf0e0ca16f4 (plain) (tree)
1
2
3
4
5
6
7
8
9


                   
                                                        
  


                                                                   
  






                                                                           






















                                                                      
                                               













                                                                    

                                                             











                                                                      



                                                                  
                       






                                                                             
 



                                                                          

                                                                             







                                                                            





                                                                             
                                                          





                                                                    
        









































































































































                                                                           
                                                        





















































































































                                                                              
                                    
                                                          
                                                     
                                    
                                                     

                                                               
                                        
                                    


                                             














                                                     
                                     


                                                     
                                    

                                             
%%
%% %CopyrightBegin%
%%
%% Copyright Ericsson AB 2013-2016. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%%     http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%
%% %CopyrightEnd%
%%

%%%-------------------------------------------------------------------
%%% File: ct_repeat_test_SUITE
%%%
%%% Description:
%%% Test different options for repeating test runs:
%%%   -repeat N
%%%   -duration T [-force_stop [skip_rest]]
%%%   -until T [-force_stop [skip_rest]]
%%%
%%%-------------------------------------------------------------------
-module(ct_repeat_testrun_SUITE).

-compile(export_all).

-include_lib("common_test/include/ct.hrl").
-include_lib("common_test/include/ct_event.hrl").

-define(eh, ct_test_support_eh).
-define(skip_reason, "Repeated test stopped by force_stop option").
-define(skipped, {auto_skipped, ?skip_reason}).


%% Timers used in this test.
%% Each test suite consists of
%%
%% [tc1,tc2,{group,g,[tc1,tc2]},tc2]
%%
%% In r1_SUITE tc1 has a sleep of 10 sec - all other test cases just
%% return ok.
%%
%% => One complete test run of two suites r1_SUITE + r2_SUITE is at
%% least 20 seconds (10 sec for each r1_SUITE:tc1)
%%
-define(t1,30). % time shall expire during second run of r1_SUITE
-define(t2,9).  % time shall expire during first run of tc1
-define(t3,19).  % time shall expire during second run of tc1


%%--------------------------------------------------------------------
%% TEST SERVER CALLBACK FUNCTIONS
%%--------------------------------------------------------------------

%%--------------------------------------------------------------------
%% Description: Since Common Test starts another Test Server
%% instance, the tests need to be performed on a separate node (or
%% there will be clashes with logging processes etc).
%%--------------------------------------------------------------------
init_per_suite(Config0) ->
    TTInfo = {_T,{_Scaled,ScaleVal}} = ct:get_timetrap_info(),
    ct:pal("Timetrap info = ~w", [TTInfo]),
    if ScaleVal > 1 ->
	    {skip,"Skip on systems running e.g. cover or debug!"};
       ScaleVal =< 1 ->
	    Config = ct_test_support:init_per_suite(Config0),
	    DataDir = ?config(data_dir, Config),
	    Suite1 = filename:join([DataDir,"a_test","r1_SUITE"]),
	    Suite2 = filename:join([DataDir,"b_test","r2_SUITE"]),
	    Opts0 = ct_test_support:get_opts(Config),
	    Opts1 = Opts0 ++ [{suite,Suite1},{testcase,tc2},{label,timing1}],
	    Opts2 = Opts0 ++ [{suite,Suite2},{testcase,tc2},{label,timing2}],

	    %% Make sure both suites are compiled
	    {1,0,{0,0}} = ct_test_support:run(ct,run_test,[Opts1],Config),
	    {1,0,{0,0}} = ct_test_support:run(ct,run_test,[Opts2],Config),
	    
	    %% Check if file i/o is too slow for correct measurements
	    Opts3 = Opts0 ++ [{suite,Suite1},{testcase,tc1},{label,timing3}],
	    {T,_} = 
		timer:tc(
		  fun() ->
			  {1,0,{0,0}} = ct_test_support:run(ct,run_test,
							    [Opts3],Config),
			  {1,0,{0,0}} = ct_test_support:run(ct,run_test,
							    [Opts3],Config)
		  end),
	    %% The time to compare with here must match the timeout value
	    %% in the test suite. Accept 30% logging overhead (26 sec total).
	    if T > 26000000 ->
		    ct:pal("Timing test took ~w sec (< 27 sec expected). "
			   "Skipping the suite!",
			   [trunc(T/1000000)]),
		    ct_test_support:end_per_suite(Config),
		    {skip,"File I/O too slow for this suite"};
	       true ->
		    ct:pal("Timing test took ~w sec. Proceeding...",
			   [trunc(T/1000000)]),
		    [{offset,0}|Config]
	    end
    end.

end_per_suite(Config) ->
    ct_test_support:end_per_suite(Config).

init_per_testcase(TestCase, Config) ->
    ct_test_support:init_per_testcase(TestCase, Config).

end_per_testcase(TestCase, Config) ->
    ct_test_support:end_per_testcase(TestCase, Config).

suite() -> [{ct_hooks,[ts_install_cth]}].

all() ->
    [
     repeat_n,
     duration,
     duration_force_stop,
     duration_force_stop_skip_rest,
     duration_force_stop_skip_rest_group,
     until,
     until_force_stop,
     until_force_stop_skip_rest,
     until_force_stop_skip_rest_group
    ].

%%--------------------------------------------------------------------
%% TEST CASES
%%--------------------------------------------------------------------

%%%-----------------------------------------------------------------
%%%
repeat_n(Config) when is_list(Config) ->
    DataDir = ?config(data_dir, Config),
    Dirs = filelib:wildcard(filename:join(DataDir,"*")),
    {Opts,ERPid} = setup([{dir,Dirs},
			  {label,repeat_n},
			  {repeat,2}],
			 Config),
    ok = execute(repeat_n, Opts, ERPid, Config).

duration(Config) when is_list(Config) ->
    DataDir = ?config(data_dir, Config),
    Dirs = filelib:wildcard(filename:join(DataDir,"*")),
    {Opts,ERPid} = setup([{dir,Dirs},
			  {label,duration},
			  {duration,duration_str(?t1,2,Config)}],
			 Config),
    ok = execute(duration, Opts, ERPid, Config).

duration_force_stop(Config) when is_list(Config) ->
    DataDir = ?config(data_dir, Config),
    Dirs = filelib:wildcard(filename:join(DataDir,"*")),
    {Opts,ERPid} = setup([{dir,Dirs},
			  {label,duration_force_stop},
			  {duration,duration_str(?t1,2,Config)},
			  {force_stop,true}],
			 Config),
    ok = execute(duration_force_stop, Opts, ERPid, Config).

duration_force_stop_skip_rest(Config) when is_list(Config) ->
    DataDir = ?config(data_dir, Config),
    Dirs = filelib:wildcard(filename:join(DataDir,"*")),
    {Opts,ERPid} = setup([{dir,Dirs},
			  {label,duration_force_stop_skip_rest},
			  {duration,duration_str(?t2,1,Config)},
			  {force_stop,skip_rest}],
			 Config),
    ok = execute(duration_force_stop_skip_rest, Opts, ERPid, Config).

duration_force_stop_skip_rest_group(Config) when is_list(Config) ->
    DataDir = ?config(data_dir, Config),
    Dirs = filelib:wildcard(filename:join(DataDir,"*")),
    {Opts,ERPid} = setup([{dir,Dirs},
			  {label,duration_force_stop_skip_rest_group},
			  {duration,duration_str(?t3,1,Config)},
			  {force_stop,skip_rest}],
			 Config),
    ok = execute(duration_force_stop_skip_rest_group, Opts, ERPid, Config).

until(Config) when is_list(Config) ->
    DataDir = ?config(data_dir, Config),
    Dirs = filelib:wildcard(filename:join(DataDir,"*")),
    {Opts,ERPid} = setup([{dir,Dirs},
			  {label,until}],
			 Config),
    ExecuteFun =
	fun() ->
		[_,_] = ct_test_support:run_ct_run_test(
			Opts++[{until,until_str(?t1,2,Config)}],Config),
		0 = ct_test_support:run_ct_script_start(
		      Opts++[{until,until_str(?t1,2,Config)}],Config)
	end,
    ok = execute(ExecuteFun, until, Opts, ERPid, Config).

until_force_stop(Config) when is_list(Config) ->
    DataDir = ?config(data_dir, Config),
    Dirs = filelib:wildcard(filename:join(DataDir,"*")),
    {Opts,ERPid} = setup([{dir,Dirs},
			  {label,until_force_stop},
			  {force_stop,true}],
			 Config),
    ExecuteFun =
	fun() ->
		[_,_] = ct_test_support:run_ct_run_test(
			  Opts++[{until,until_str(?t1,2,Config)}],Config),
		0 = ct_test_support:run_ct_script_start(
		      Opts++[{until,until_str(?t1,2,Config)}],Config)
	end,
    ok = execute(ExecuteFun, until_force_stop, Opts, ERPid, Config).

until_force_stop_skip_rest(Config) when is_list(Config) ->
    DataDir = ?config(data_dir, Config),
    Dirs = filelib:wildcard(filename:join(DataDir,"*")),
    {Opts,ERPid} = setup([{dir,Dirs},
			  {label,until_force_stop_skip_rest},
			  {force_stop,skip_rest}],
			 Config),
    ExecuteFun =
	fun() ->
		[_] = ct_test_support:run_ct_run_test(
			Opts++[{until,until_str(?t2,1,Config)}],Config),
		1 = ct_test_support:run_ct_script_start(
		      Opts++[{until,until_str(?t2,1,Config)}],Config)
	end,
    ok = execute(ExecuteFun, until_force_stop_skip_rest,
		 Opts, ERPid, Config).

until_force_stop_skip_rest_group(Config) when is_list(Config) ->
    DataDir = ?config(data_dir, Config),
    Dirs = filelib:wildcard(filename:join(DataDir,"*")),
    {Opts,ERPid} = setup([{dir,Dirs},
			  {label,until_force_stop_skip_rest_group},
			  {force_stop,skip_rest}],
			 Config),
    ExecuteFun =
	fun() ->
		[_] = ct_test_support:run_ct_run_test(
			Opts++[{until,until_str(?t3,1,Config)}],Config),
		1 = ct_test_support:run_ct_script_start(
		      Opts++[{until,until_str(?t3,1,Config)}],Config)
	end,
    ok = execute(ExecuteFun,
		 until_force_stop_skip_rest_group,
		 Opts, ERPid, Config).


%%%-----------------------------------------------------------------
%%% HELP FUNCTIONS
%%%-----------------------------------------------------------------

setup(Test, Config) ->
    Opts0 = ct_test_support:get_opts(Config),
    Level = ?config(trace_level, Config),
    EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
    Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test],
    ERPid = ct_test_support:start_event_receiver(Config),
    {Opts,ERPid}.

%% Execute test, first with ct:run_test, then with ct:script_start
execute(Name, Opts, ERPid, Config) ->
    ExecuteFun = fun() -> ok = ct_test_support:run(Opts, Config) end,
    execute(ExecuteFun, Name, Opts, ERPid, Config).

execute(ExecuteFun, Name, Opts, ERPid, Config) ->
    ExecuteFun(),
    Events = ct_test_support:get_events(ERPid, Config),

    ct_test_support:log_events(Name,
			       reformat(Events, ?eh),
			       ?config(priv_dir, Config),
			       Opts),

    TestEvents = events_to_check(Name),
    ct_test_support:verify_events(TestEvents, Events, Config).

reformat(Events, EH) ->
    ct_test_support:reformat(Events, EH).

%% N is the expected number of repeats
until_str(Secs0,N,Config) ->
    Offset = ?config(offset,Config),
    Secs = Secs0 + N*Offset,
    Now = calendar:datetime_to_gregorian_seconds(calendar:local_time()),
    {{Y,Mo,D},{H,M,S}} = calendar:gregorian_seconds_to_datetime(Now+Secs),
    lists:flatten(io_lib:format("~2..0w~2..0w~2..0w~2..0w~2..0w~2..0w",
				[Y rem 100, Mo, D, H, M, S])).

%% N is the expected number of repeats
duration_str(Secs0,N,Config) ->
    Offset = ?config(offset,Config),
    Secs = Secs0 + N*Offset,
    "0000" ++ lists:flatten(io_lib:format("~2..0w",[Secs])).

%%%-----------------------------------------------------------------
%%% TEST EVENTS
%%%-----------------------------------------------------------------
%% 2 tests (ct:run_test + script_start) is default
events_to_check(C) when C==repeat_n; C==duration; C==until ->
    dupl(4, start_logging() ++ all_succ() ++ stop_logging());
events_to_check(C) when C==duration_force_stop; C==until_force_stop ->
    dupl(2, start_logging() ++
	     all_succ() ++
	     stop_logging() ++
	     start_logging() ++
	     all_succ(r1_SUITE) ++
	     stop_logging());
events_to_check(C) when C==duration_force_stop_skip_rest;
			C==until_force_stop_skip_rest ->
    dupl(2, start_logging() ++ skip_first_tc1(r1_SUITE) ++ stop_logging());
events_to_check(C) when C==duration_force_stop_skip_rest_group;
			C==until_force_stop_skip_rest_group ->
    dupl(2, start_logging() ++ skip_tc1_in_group(r1_SUITE) ++ stop_logging()).

dupl(N,List) ->
    lists:flatten(lists:duplicate(N,List)).

start_logging() ->
    [{?eh,start_logging,{'DEF','RUNDIR'}}].
stop_logging() ->
    [{?eh,stop_logging,[]}].


all_succ() ->
    all_succ(r1_SUITE) ++ all_succ(r2_SUITE).

all_succ(Suite) ->
    [{?eh,tc_start,{Suite,init_per_suite}},
     {?eh,tc_done,{Suite,init_per_suite,ok}},
     {?eh,tc_start,{Suite,tc1}},
     {?eh,tc_done,{Suite,tc1,ok}},
     {?eh,test_stats,{'_',0,{0,0}}},
     {?eh,tc_start,{Suite,tc2}},
     {?eh,tc_done,{Suite,tc2,ok}},
     {?eh,test_stats,{'_',0,{0,0}}},
     [{?eh,tc_start,{Suite,{init_per_group,g,[]}}},
      {?eh,tc_done,{Suite,{init_per_group,g,[]},ok}},
      {?eh,tc_start,{Suite,tc1}},
      {?eh,tc_done,{Suite,tc1,ok}},
      {?eh,test_stats,{'_',0,{0,0}}},
      {?eh,tc_start,{Suite,tc2}},
      {?eh,tc_done,{Suite,tc2,ok}},
      {?eh,test_stats,{'_',0,{0,0}}},
      {?eh,tc_start,{Suite,{end_per_group,g,[]}}},
      {?eh,tc_done,{Suite,{end_per_group,g,[]},ok}}],
     {?eh,tc_start,{Suite,tc2}},
     {?eh,tc_done,{Suite,tc2,ok}},
     {?eh,test_stats,{'_',0,{0,0}}},
     {?eh,tc_start,{Suite,end_per_suite}},
     {?eh,tc_done,{Suite,end_per_suite,ok}}].

skip_first_tc1(Suite) ->
    [{?eh,tc_start,{Suite,init_per_suite}},
     {?eh,tc_done,{Suite,init_per_suite,ok}},
     {?eh,tc_start,{Suite,tc1}},
     {?eh,tc_done,{Suite,tc1,ok}},
     {?eh,test_stats,{'_',0,{0,0}}},
     {?eh,tc_done,{Suite,tc2,?skipped}},
     {?eh,test_stats,{'_',0,{0,1}}},
     {?eh,tc_done,{Suite,{init_per_group,g,[]},?skipped}},
     {?eh,tc_auto_skip,{Suite,{tc1,g},?skip_reason}},
     {?eh,test_stats,{'_',0,{0,2}}},
     {?eh,tc_auto_skip,{Suite,{tc2,g},?skip_reason}},
     {?eh,test_stats,{'_',0,{0,3}}},
     {?eh,tc_auto_skip,{Suite,{end_per_group,g},?skip_reason}},
     {?eh,tc_done,{Suite,tc2,?skipped}},
     {?eh,test_stats,{'_',0,{0,4}}},
     {?eh,tc_start,{Suite,end_per_suite}},
     {?eh,tc_done,{Suite,end_per_suite,ok}}].

skip_tc1_in_group(Suite) ->
    [{?eh,tc_start,{Suite,init_per_suite}},
     {?eh,tc_done,{Suite,init_per_suite,ok}},
     {?eh,tc_start,{Suite,tc1}},
     {?eh,tc_done,{Suite,tc1,ok}},
     {?eh,test_stats,{'_',0,{0,0}}},
     {?eh,tc_start,{Suite,tc2}},
     {?eh,tc_done,{Suite,tc2,ok}},
     {?eh,test_stats,{'_',0,{0,0}}},
     [{?eh,tc_start,{Suite,{init_per_group,g,[]}}},
      {?eh,tc_done,{Suite,{init_per_group,g,[]},ok}},
      {?eh,tc_start,{Suite,tc1}},
      {?eh,tc_done,{Suite,tc1,ok}},
      {?eh,test_stats,{'_',0,{0,0}}},
      {?eh,tc_done,{Suite,tc2,?skipped}},
      {?eh,test_stats,{'_',0,{0,1}}},
      {?eh,tc_start,{Suite,{end_per_group,g,[]}}},
      {?eh,tc_done,{Suite,{end_per_group,g,[]},ok}}],
     {?eh,tc_done,{Suite,tc2,?skipped}},
     {?eh,test_stats,{'_',0,{0,2}}},
     {?eh,tc_start,{Suite,end_per_suite}},
     {?eh,tc_done,{Suite,end_per_suite,ok}}].