diff options
Diffstat (limited to 'lib/common_test')
-rw-r--r-- | lib/common_test/src/ct_framework.erl | 13 | ||||
-rw-r--r-- | lib/common_test/src/test_server.erl | 216 | ||||
-rw-r--r-- | lib/common_test/src/test_server_ctrl.erl | 4 | ||||
-rw-r--r-- | lib/common_test/src/test_server_sup.erl | 2 | ||||
-rw-r--r-- | lib/common_test/test/Makefile | 3 | ||||
-rw-r--r-- | lib/common_test/test/ct_error_SUITE.erl | 49 | ||||
-rw-r--r-- | lib/common_test/test/ct_util_SUITE.erl | 490 | ||||
-rw-r--r-- | lib/common_test/test/ct_util_SUITE_data/ct_util_cth.erl | 105 | ||||
-rw-r--r-- | lib/common_test/test/ct_util_SUITE_data/curr_tc_SUITE.erl | 59 |
9 files changed, 902 insertions, 39 deletions
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 506147474f..ea3422bc43 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -696,9 +696,16 @@ end_tc(Mod,IPTC={init_per_testcase,_Func},_TCPid,Result,Args,Return) -> end end; -end_tc(Mod,Func0,TCPid,Result,Args,Return) -> +end_tc(Mod,Func00,TCPid,Result,Args,Return) -> %% in case Mod == ct_framework, lookup the suite name Suite = get_suite_name(Mod, Args), + {OnlyCleanup,Func0} = + case Func00 of + {cleanup,F0} -> + {true,F0}; + _ -> + {false,Func00} + end, {Func,FuncSpec,HookFunc} = case Func0 of {end_per_testcase_not_run,F} -> @@ -742,6 +749,8 @@ end_tc(Mod,Func0,TCPid,Result,Args,Return) -> case HookFunc of undefined -> {ok,Result}; + _ when OnlyCleanup -> + {ok,Result}; _ -> case ct_hooks:end_tc(Suite,HookFunc,Args,Result,Return) of '$ct_no_change' -> @@ -752,6 +761,8 @@ end_tc(Mod,Func0,TCPid,Result,Args,Return) -> end, FinalResult = case get('$test_server_framework_test') of + _ when OnlyCleanup -> + Result1; undefined -> %% send sync notification so that event handlers may print %% in the log file before it gets closed diff --git a/lib/common_test/src/test_server.erl b/lib/common_test/src/test_server.erl index 9eda3f2152..756cd4d692 100644 --- a/lib/common_test/src/test_server.erl +++ b/lib/common_test/src/test_server.erl @@ -384,8 +384,8 @@ run_test_case_apply({CaseNum,Mod,Func,Args,Name,RunInit,TimetrapData}) -> {Result,DetFail,ProcBef,ProcAft}. -type tc_status() :: 'starting' | 'running' | 'init_per_testcase' | - 'end_per_testcase' | {'framework',atom(),atom()} | - 'tc'. + 'end_per_testcase' | {'framework',{atom(),atom(),list}} | + 'tc'. -record(st, { ref :: reference(), @@ -653,8 +653,8 @@ handle_tc_exit({testcase_aborted,{user_timetrap_error,_}=Msg,_}, St) -> #st{config=Config,mf={Mod,Func},pid=Pid} = St, spawn_fw_call(Mod, Func, Config, Pid, Msg, unknown, self()), St; -handle_tc_exit(Reason, #st{status={framework,FwMod,FwFunc}, - config=Config,pid=Pid}=St) -> +handle_tc_exit(Reason, #st{status={framework,{FwMod,FwFunc,_}=FwMFA}, + config=Config,mf={Mod,Func},pid=Pid}=St) -> R = case Reason of {timetrap_timeout,TVal,_} -> {timetrap,TVal}; @@ -666,7 +666,7 @@ handle_tc_exit(Reason, #st{status={framework,FwMod,FwFunc}, Other end, Error = {framework_error,R}, - spawn_fw_call(FwMod, FwFunc, Config, Pid, Error, unknown, self()), + spawn_fw_call(Mod, Func, Config, Pid, {Error,FwMFA}, unknown, self()), St; handle_tc_exit(Reason, #st{status=tc,config=Config0,mf={Mod,Func},pid=Pid}=St) when is_list(Config0) -> @@ -870,22 +870,48 @@ spawn_fw_call(Mod,EPTC={end_per_testcase,Func},EndConf,Pid, end, spawn_link(FwCall); -spawn_fw_call(FwMod,FwFunc,_,_Pid,{framework_error,FwError},_,SendTo) -> +spawn_fw_call(Mod,Func,Conf,Pid,{{framework_error,FwError}, + {FwMod,FwFunc,[A1,A2|_]}=FwMFA},_,SendTo) -> FwCall = fun() -> ct_util:mark_process(), - test_server_sup:framework_call(report, [framework_error, - {{FwMod,FwFunc}, - FwError}]), + Time = + case FwError of + {timetrap,TVal} -> + TVal/1000; + _ -> + died + end, + {Ret,Loc,WarnOrError} = + cleanup_after_fw_error(Mod,Func,Conf,Pid,FwError,FwMFA), Comment = - lists:flatten( - io_lib:format("<font color=\"red\">" - "WARNING! ~w:~tw failed!</font>", - [FwMod,FwFunc])), + case WarnOrError of + warn -> + group_leader() ! + {printout,12, + "WARNING! ~w:~tw(~w,~tw,...) failed!\n" + " Reason: ~tp\n", + [FwMod,FwFunc,A1,A2,FwError]}, + lists:flatten( + io_lib:format("<font color=\"red\">" + "WARNING! ~w:~tw(~w,~tw,...) " + "failed!</font>", + [FwMod,FwFunc,A1,A2])); + error -> + group_leader() ! + {printout,12, + "Error! ~w:~tw(~w,~tw,...) failed!\n" + " Reason: ~tp\n", + [FwMod,FwFunc,A1,A2,FwError]}, + lists:flatten( + io_lib:format("<font color=\"red\">" + "ERROR! ~w:~tw(~w,~tw,...) " + "failed!</font>", + [FwMod,FwFunc,A1,A2])) + end, %% finished, report back SendTo ! {self(),fw_notify_done, - {died,{error,{FwMod,FwFunc,FwError}}, - {FwMod,FwFunc},[],Comment}} + {Time,Ret,Loc,[],Comment}} end, spawn_link(FwCall); @@ -930,6 +956,163 @@ spawn_fw_call(Mod,Func,CurrConf,Pid,Error,Loc,SendTo) -> end, spawn_link(FwCall). +cleanup_after_fw_error(_Mod,_Func,Conf,Pid,FwError, + {FwMod,FwFunc=init_tc, + [Mod,{init_per_testcase,Func}=IPTC|_]}) -> + %% Failed during pre_init_per_testcase, the test must be skipped + Skip = {auto_skip,{failed,{FwMod,FwFunc,FwError}}}, + try begin do_end_tc_call(Mod,IPTC, {Pid,Skip,[Conf]}, FwError), + do_init_tc_call(Mod,{end_per_testcase_not_run,Func}, + [Conf],{ok,[Conf]}), + do_end_tc_call(Mod,{end_per_testcase_not_run,Func}, + {Pid,Skip,[Conf]}, FwError) end of + _ -> ok + catch + _:FwEndTCErr -> + exit({fw_notify_done,end_tc,FwEndTCErr}) + end, + {Skip,{FwMod,FwFunc},error}; +cleanup_after_fw_error(_Mod,_Func,Conf,Pid,FwError, + {FwMod,FwFunc=end_tc,[Mod,{init_per_testcase,Func}|_]}) -> + %% Failed during post_init_per_testcase, the test must be skipped + Skip = {auto_skip,{failed,{FwMod,FwFunc,FwError}}}, + try begin do_init_tc_call(Mod,{end_per_testcase_not_run,Func}, + [Conf],{ok,[Conf]}), + do_end_tc_call(Mod,{end_per_testcase_not_run,Func}, + {Pid,Skip,[Conf]}, FwError) end of + _ -> ok + catch + _:FwEndTCErr -> + exit({fw_notify_done,end_tc,FwEndTCErr}) + end, + {Skip,{FwMod,FwFunc},error}; +cleanup_after_fw_error(_Mod,_Func,Conf,Pid,FwError, + {FwMod,FwFunc=init_tc,[Mod,{end_per_testcase,Func}|_]}) -> + %% Failed during pre_end_per_testcase. Warn about it. + {RetVal,Loc} = + case {proplists:get_value(tc_status, Conf), + proplists:get_value(tc_fail_loc, Conf, unknown)} of + {undefined,_} -> + {{failed,{FwMod,FwFunc,FwError}},{FwMod,FwFunc}}; + {E = {failed,_Reason},unknown} -> + {E,[{Mod,Func}]}; + {Result,FailLoc} -> + {Result,FailLoc} + end, + try begin do_end_tc_call(Mod,{end_per_testcase_not_run,Func}, + {Pid,RetVal,[Conf]}, FwError) end of + _ -> ok + catch + _:FwEndTCErr -> + exit({fw_notify_done,end_tc,FwEndTCErr}) + end, + {RetVal,Loc,warn}; +cleanup_after_fw_error(Mod,Func,Conf,Pid,FwError, + {FwMod,FwFunc=end_tc,[Mod,{end_per_testcase,Func}|_]}) -> + %% Failed during post_end_per_testcase. Warn about it. + {RetVal,Report,Loc} = + case {proplists:get_value(tc_status, Conf), + proplists:get_value(tc_fail_loc, Conf, unknown)} of + {undefined,_} -> + {{failed,{FwMod,FwFunc,FwError}}, + {{FwMod,FwError},FwError}, + {FwMod,FwFunc}}; + {E = {failed,_Reason},unknown} -> + {E,{Mod,Func,E},[{Mod,Func}]}; + {Result,FailLoc} -> + {Result,{Mod,Func,Result},FailLoc} + end, + try begin do_end_tc_call(Mod,{cleanup,{end_per_testcase_not_run,Func}}, + {Pid,RetVal,[Conf]}, FwError) end of + _ -> ok + catch + _:FwEndTCErr -> + exit({fw_notify_done,end_tc,FwEndTCErr}) + end, + test_server_sup:framework_call(report,[framework_error,Report]), + {RetVal,Loc,warn}; +cleanup_after_fw_error(Mod,Func,Conf,Pid,FwError,{FwMod,FwFunc=init_tc,_}) + when Func =:= init_per_suite; Func =:=init_per_group -> + %% Failed during pre_init_per_suite or pre_init_per_group + RetVal = {failed,{FwMod,FwFunc,FwError}}, + try do_end_tc_call(Mod,Func,{Pid,RetVal,[Conf]},FwError) of + _ -> ok + catch + _:FwEndTCErr -> + exit({fw_notify_done,end_tc,FwEndTCErr}) + end, + {RetVal,{FwMod,FwFunc},error}; +cleanup_after_fw_error(Mod,Func,Conf,Pid,FwError,{FwMod,FwFunc=end_tc,_}) + when Func =:= init_per_suite; Func =:=init_per_group -> + %% Failed during post_init_per_suite or post_init_per_group + RetVal = {failed,{FwMod,FwFunc,FwError}}, + try do_end_tc_call(Mod,{cleanup,Func},{Pid,RetVal,[Conf]},FwError) of + _ -> ok + catch + _:FwEndTCErr -> + exit({fw_notify_done,end_tc,FwEndTCErr}) + end, + ReportFunc = + case Func of + init_per_group -> + case proplists:get_value(tc_group_properties,Conf) of + undefined -> + {Func,unknown,[]}; + GProps -> + Name = proplists:get_value(name,GProps), + {Func,Name,proplists:delete(name,GProps)} + end; + _ -> + Func + end, + test_server_sup:framework_call(report,[framework_error, + {Mod,ReportFunc,RetVal}]), + {RetVal,{FwMod,FwFunc},error}; +cleanup_after_fw_error(Mod,Func,Conf,Pid,FwError,{FwMod,FwFunc=init_tc,_}) + when Func =:= end_per_suite; Func =:=end_per_group -> + %% Failed during pre_end_per_suite or pre_end_per_group + RetVal = {failed,{FwMod,FwFunc,FwError}}, + try do_end_tc_call(Mod,Func,{Pid,RetVal,[Conf]},FwError) of + _ -> ok + catch + _:FwEndTCErr -> + exit({fw_notify_done,end_tc,FwEndTCErr}) + end, + {RetVal,{FwMod,FwFunc},error}; +cleanup_after_fw_error(Mod,Func,Conf,Pid,FwError,{FwMod,FwFunc=end_tc,_}) + when Func =:= end_per_suite; Func =:=end_per_group -> + %% Failed during post_end_per_suite or post_end_per_group + RetVal = {failed,{FwMod,FwFunc,FwError}}, + try do_end_tc_call(Mod,{cleanup,Func},{Pid,RetVal,[Conf]},FwError) of + _ -> ok + catch + _:FwEndTCErr -> + exit({fw_notify_done,end_tc,FwEndTCErr}) + end, + ReportFunc = + case Func of + end_per_group -> + case proplists:get_value(tc_group_properties,Conf) of + undefined -> + {Func,unknown,[]}; + GProps -> + Name = proplists:get_value(name,GProps), + {Func,Name,proplists:delete(name,GProps)} + end; + _ -> + Func + end, + test_server_sup:framework_call(report,[framework_error, + {Mod,ReportFunc,RetVal}]), + {RetVal,{FwMod,FwFunc},error}; +cleanup_after_fw_error(_Mod,_Func,_Conf,_Pid,FwError,{FwMod,FwFunc,_}) -> + %% This is unexpected + test_server_sup:framework_call(report, + [framework_error, + {{FwMod,FwFunc}, + FwError}]), + {FwError,{FwMod,FwFunc},error}. + %% The job proxy process forwards messages between the test case %% process on a shielded node (and its descendants) and the job process. %% @@ -1105,6 +1288,9 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) -> EndConf1 = user_callback(TCCallback, Mod, Func, 'end', EndConf), + %% save updated config in controller loop + set_tc_state(tc, EndConf1), + %% We can't handle fails or skips here EndConf2 = case do_init_tc_call(Mod,{end_per_testcase,Func}, diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl index 1518c6e8d6..0dca46559f 100644 --- a/lib/common_test/src/test_server_ctrl.erl +++ b/lib/common_test/src/test_server_ctrl.erl @@ -3842,6 +3842,10 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, {died,{timetrap_timeout,TimetrapTimeout}} -> progress(failed, Num, Mod, Func, GrName, Loc, timetrap_timeout, TimetrapTimeout, Comment, Style); + {died,Reason={auto_skip,_Why}} -> + %% died in init_per_testcase or in a hook in this context + progress(skip, Num, Mod, Func, GrName, Loc, Reason, + Time, Comment, Style); {died,{Skip,Reason}} when Skip==skip; Skip==skipped -> %% died in init_per_testcase progress(skip, Num, Mod, Func, GrName, Loc, Reason, diff --git a/lib/common_test/src/test_server_sup.erl b/lib/common_test/src/test_server_sup.erl index 26e7534c6c..ab8066a88d 100644 --- a/lib/common_test/src/test_server_sup.erl +++ b/lib/common_test/src/test_server_sup.erl @@ -770,7 +770,7 @@ framework_call(Callback,Func,Args,DefaultReturn) -> end, case SetTcState of true -> - test_server:set_tc_state({framework,Mod,Func}); + test_server:set_tc_state({framework,{Mod,Func,Args}}); false -> ok end, diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile index ecd1f727a2..bb3e7a1eca 100644 --- a/lib/common_test/test/Makefile +++ b/lib/common_test/test/Makefile @@ -74,7 +74,8 @@ MODULES= \ ct_SUITE \ ct_keep_logs_SUITE \ ct_unicode_SUITE \ - ct_auto_clean_SUITE + ct_auto_clean_SUITE \ + ct_util_SUITE ERL_FILES= $(MODULES:%=%.erl) HRL_FILES= test_server_test_lib.hrl diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index 7468ebe9d9..d31bd26273 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -648,33 +648,35 @@ test_events(cfg_error) -> {?eh,tc_start,{cfg_error_11_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_11_SUITE,end_per_suite,ok}}, {?eh,tc_start,{cfg_error_12_SUITE,tc1}}, - {?eh,tc_done,{ct_framework,init_tc,{framework_error,{timetrap,500}}}}, - {?eh,test_stats,{13,8,{0,19}}}, + {?eh,tc_done,{cfg_error_12_SUITE,tc1, + {auto_skipped, + {failed,{ct_framework,init_tc,{timetrap,500}}}}}}, + {?eh,test_stats,{13,7,{0,20}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc2}}, {?eh,tc_done,{cfg_error_12_SUITE,tc2,{failed, {cfg_error_12_SUITE,end_per_testcase, {timetrap_timeout,500}}}}}, - {?eh,test_stats,{14,8,{0,19}}}, + {?eh,test_stats,{14,7,{0,20}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc3}}, {?eh,tc_done,{cfg_error_12_SUITE,tc3,ok}}, - {?eh,test_stats,{15,8,{0,19}}}, + {?eh,test_stats,{15,7,{0,20}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc4}}, {?eh,tc_done,{cfg_error_12_SUITE,tc4,{failed, {cfg_error_12_SUITE,end_per_testcase, {timetrap_timeout,500}}}}}, - {?eh,test_stats,{16,8,{0,19}}}, + {?eh,test_stats,{16,7,{0,20}}}, {?eh,tc_start,{cfg_error_13_SUITE,init_per_suite}}, {?eh,tc_done,{cfg_error_13_SUITE,init_per_suite,ok}}, {?eh,tc_start,{cfg_error_13_SUITE,tc1}}, {?eh,tc_done,{cfg_error_13_SUITE,tc1,ok}}, - {?eh,test_stats,{17,8,{0,19}}}, + {?eh,test_stats,{17,7,{0,20}}}, {?eh,tc_start,{cfg_error_13_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_13_SUITE,end_per_suite,ok}}, {?eh,tc_start,{cfg_error_14_SUITE,init_per_suite}}, {?eh,tc_done,{cfg_error_14_SUITE,init_per_suite,ok}}, {?eh,tc_start,{cfg_error_14_SUITE,tc1}}, {?eh,tc_done,{cfg_error_14_SUITE,tc1,ok}}, - {?eh,test_stats,{18,8,{0,19}}}, + {?eh,test_stats,{18,7,{0,20}}}, {?eh,tc_start,{cfg_error_14_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_14_SUITE,end_per_suite, {comment, @@ -728,25 +730,30 @@ test_events(lib_error) -> {lib_error_1_SUITE,no_lines_throw,{failed,{error,{thrown,catch_me_if_u_can}}}}}, {?eh,test_stats,{0,8,{0,0}}}, {?eh,tc_start,{lib_error_1_SUITE,init_tc_error}}, - {?eh,tc_done,{ct_framework,init_tc, - {framework_error,{{badmatch,[1,2]},'_'}}}}, - {?eh,test_stats,{0,9,{0,0}}}, + {?eh,tc_done,{lib_error_1_SUITE,init_tc_error, + {auto_skipped, + {failed, + {ct_framework,init_tc, + {{badmatch,[1,2]},'_'}}}}}}, + {?eh,test_stats,{0,8,{0,1}}}, {?eh,tc_start,{lib_error_1_SUITE,init_tc_exit}}, - {?eh,tc_done,{ct_framework,init_tc,{framework_error,byebye}}}, - {?eh,test_stats,{0,10,{0,0}}}, + {?eh,tc_done,{lib_error_1_SUITE,init_tc_exit, + {auto_skipped,{failed,{ct_framework,init_tc,byebye}}}}}, + {?eh,test_stats,{0,8,{0,2}}}, {?eh,tc_start,{lib_error_1_SUITE,init_tc_throw}}, - {?eh,tc_done,{ct_framework,init_tc,{framework_error,catch_me_if_u_can}}}, - {?eh,test_stats,{0,11,{0,0}}}, + {?eh,tc_done,{lib_error_1_SUITE,init_tc_throw, + {auto_skipped,{failed,{ct_framework,init_tc, + catch_me_if_u_can}}}}}, + {?eh,test_stats,{0,8,{0,3}}}, {?eh,tc_start,{lib_error_1_SUITE,end_tc_error}}, - {?eh,tc_done,{ct_framework,end_tc, - {framework_error,{{badmatch,[1,2]},'_'}}}}, - {?eh,test_stats,{0,12,{0,0}}}, + {?eh,tc_done,{lib_error_1_SUITE,end_tc_error,ok}}, % warning in comment + {?eh,test_stats,{1,8,{0,3}}}, {?eh,tc_start,{lib_error_1_SUITE,end_tc_exit}}, - {?eh,tc_done,{ct_framework,end_tc,{framework_error,byebye}}}, - {?eh,test_stats,{0,13,{0,0}}}, + {?eh,tc_done,{lib_error_1_SUITE,end_tc_exit,ok}}, % warning in comment + {?eh,test_stats,{2,8,{0,3}}}, {?eh,tc_start,{lib_error_1_SUITE,end_tc_throw}}, - {?eh,tc_done,{ct_framework,end_tc,{framework_error,catch_me_if_u_can}}}, - {?eh,test_stats,{0,14,{0,0}}}, + {?eh,tc_done,{lib_error_1_SUITE,end_tc_throw,ok}}, % warning in comment + {?eh,test_stats,{3,8,{0,3}}}, {?eh,tc_start,{lib_error_1_SUITE,end_per_suite}}, {?eh,tc_done,{lib_error_1_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, diff --git a/lib/common_test/test/ct_util_SUITE.erl b/lib/common_test/test/ct_util_SUITE.erl new file mode 100644 index 0000000000..1d773855da --- /dev/null +++ b/lib/common_test/test/ct_util_SUITE.erl @@ -0,0 +1,490 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(ct_util_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + DataDir = ?config(data_dir, Config), + CTHs = filelib:wildcard(filename:join(DataDir,"*_cth.erl")), + io:format("CTHs: ~p",[CTHs]), + [io:format("Compiling ~p: ~p", + [FileName,compile:file(FileName,[{outdir,DataDir},debug_info])]) || + FileName <- CTHs], + ct_test_support:init_per_suite([{path_dirs,[DataDir]} | Config]). + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + + +suite() -> + [{timetrap,{minutes,1}}]. + +all() -> + all(suite). + +all(suite) -> + [ + pre_init_per_suite, + post_init_per_suite, + pre_end_per_suite, + post_end_per_suite, + pre_init_per_group, + post_init_per_group, + pre_end_per_group, + post_end_per_group, + pre_init_per_testcase, + post_init_per_testcase, + pre_end_per_testcase, + post_end_per_testcase + ]. + + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%%%----------------------------------------------------------------- +%%% +pre_init_per_suite(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{pre_init_per_suite, + {curr_tc_SUITE,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +post_init_per_suite(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_init_per_suite, + {curr_tc_SUITE,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +pre_end_per_suite(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{pre_end_per_suite, + {curr_tc_SUITE,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +post_end_per_suite(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_end_per_suite, + {curr_tc_SUITE,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + + +pre_init_per_group(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{pre_init_per_group, + {curr_tc_SUITE,g,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +post_init_per_group(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_init_per_group, + {curr_tc_SUITE,g,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +pre_end_per_group(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{pre_end_per_group, + {curr_tc_SUITE,g,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +post_end_per_group(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_end_per_group, + {curr_tc_SUITE,g,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +pre_init_per_testcase(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{pre_init_per_testcase, + {curr_tc_SUITE,tc1,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +post_init_per_testcase(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_init_per_testcase, + {curr_tc_SUITE,tc1,{timeout,5000}}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +pre_end_per_testcase(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{pre_end_per_testcase, + {curr_tc_SUITE,tc1,{timeout,5000}}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +post_end_per_testcase(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_end_per_testcase, + {curr_tc_SUITE,tc1,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- + +do_test(Tag, Suite, RunTestArgs, Config) -> + do_test(Tag, Suite, RunTestArgs, Config, 2). + +do_test(Tag, Suite0, RunTestArgs, Config, EC) -> + DataDir = ?config(data_dir, Config), + Suite = filename:join([DataDir,Suite0]), + {Opts,ERPid} = setup([{suite,Suite}]++[{label,Tag}|RunTestArgs],Config), + Res = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + %% io:format("~p~n",[Events]), + + ct_test_support:log_events(Tag, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(Tag, EC), + ok = ct_test_support:verify_events(TestEvents, Events, Config), + Res. + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). + +gen_config(Name,KeyVals,Config) -> + PrivDir = ?config(priv_dir,Config), + File = filename:join(PrivDir,atom_to_list(Name)++".cfg"), + ok = file:write_file(File,[io_lib:format("~p.~n",[{Key,Value}]) + || {Key,Value} <- KeyVals]), + File. + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +%%%----------------------------------------------------------------- +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). + +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + test_events(Test) ++ events_to_check(Test, N-1). + +test_events(IPS) when IPS=:=pre_init_per_suite; IPS=:=post_init_per_suite -> + S = curr_tc_SUITE, + FwFunc = + case IPS of + pre_init_per_suite -> init_tc; + post_init_per_suite -> end_tc + end, + E = {failed,{ct_framework,FwFunc,{test_case_failed,hahahahahah}}}, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{S,init_per_suite}}, + {?eh,tc_done,{S,init_per_suite,E}}, + {?eh,tc_auto_skip,{S,tc1,{failed,{S,init_per_suite,E}}}}, + {?eh,tc_auto_skip,{S,tc2,{failed,{S,init_per_suite,E}}}}, + {?eh,tc_auto_skip,{S,{tc1,g},{failed,{S,init_per_suite,E}}}}, + {?eh,tc_auto_skip,{S,{tc2,g},{failed,{S,init_per_suite,E}}}}, + {?eh,test_stats,{0,0,{0,4}}}, + {?eh,tc_auto_skip,{S,end_per_suite,{failed,{S,init_per_suite,E}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(EPS) when EPS=:=pre_end_per_suite; EPS=:=post_end_per_suite -> + S = curr_tc_SUITE, + FwFunc = + case EPS of + pre_end_per_suite -> init_tc; + post_end_per_suite -> end_tc + end, + E = {failed,{ct_framework,FwFunc,{test_case_failed,hahahahahah}}}, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{S,init_per_suite}}, + {?eh,tc_done,{S,init_per_suite,ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + [{?eh,tc_start,{S,{init_per_group,g,[]}}}, + {?eh,tc_done,{S,{init_per_group,g,[]},ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{3,0,{0,0}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,tc_start,{S,{end_per_group,g,[]}}}, + {?eh,tc_done,{S,{end_per_group,g,[]},ok}}], + {?eh,tc_start,{S,end_per_suite}}, + {?eh,tc_done,{S,end_per_suite,E}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(IPG) when IPG=:=pre_init_per_group; IPG=:=post_init_per_group -> + S = curr_tc_SUITE, + FwFunc = + case IPG of + pre_init_per_group -> init_tc; + post_init_per_group -> end_tc + end, + E = {failed,{ct_framework,FwFunc,{test_case_failed,hahahahahah}}}, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{S,init_per_suite}}, + {?eh,tc_done,{S,init_per_suite,ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + [{?eh,tc_start,{S,{init_per_group,g,[]}}}, + {?eh,tc_done,{S,{init_per_group,g,[]},E}}, + {?eh,tc_auto_skip,{S,{tc1,g},{failed,{S,init_per_group,E}}}}, + {?eh,tc_auto_skip,{S,{tc2,g},{failed,{S,init_per_group,E}}}}, + {?eh,test_stats,{2,0,{0,2}}}, + {?eh,tc_auto_skip,{S,{end_per_group,g},{failed,{S,init_per_group,E}}}}], + {?eh,tc_start,{S,end_per_suite}}, + {?eh,tc_done,{S,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(EPG) when EPG=:=pre_end_per_group; EPG=:=post_end_per_group -> + S = curr_tc_SUITE, + FwFunc = + case EPG of + pre_end_per_group -> init_tc; + post_end_per_group -> end_tc + end, + E = {failed,{ct_framework,FwFunc,{test_case_failed,hahahahahah}}}, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{S,init_per_suite}}, + {?eh,tc_done,{S,init_per_suite,ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + [{?eh,tc_start,{S,{init_per_group,g,[]}}}, + {?eh,tc_done,{S,{init_per_group,g,[]},ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{3,0,{0,0}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,tc_start,{S,{end_per_group,g,[]}}}, + {?eh,tc_done,{S,{end_per_group,g,[]},E}}], + {?eh,tc_start,{S,end_per_suite}}, + {?eh,tc_done,{S,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(IPTC) when IPTC=:=pre_init_per_testcase; + IPTC=:=post_init_per_testcase -> + S = curr_tc_SUITE, + E = case IPTC of + pre_init_per_testcase -> + {failed,{ct_framework,init_tc,{test_case_failed,hahahahahah}}}; + post_init_per_testcase -> + {failed,{ct_framework,end_tc,{timetrap,3000}}} + end, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{S,init_per_suite}}, + {?eh,tc_done,{S,init_per_suite,ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,{auto_skipped,E}}}, + {?eh,test_stats,{0,0,{0,1}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{1,0,{0,1}}}, + [{?eh,tc_start,{S,{init_per_group,g,[]}}}, + {?eh,tc_done,{S,{init_per_group,g,[]},ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,{auto_skipped,E}}}, + {?eh,test_stats,{1,0,{0,2}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{2,0,{0,2}}}, + {?eh,tc_start,{S,{end_per_group,g,[]}}}, + {?eh,tc_done,{S,{end_per_group,g,[]},ok}}], + {?eh,tc_start,{S,end_per_suite}}, + {?eh,tc_done,{S,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(EPTC) when EPTC=:=pre_end_per_testcase; EPTC=:=post_end_per_testcase-> + S = curr_tc_SUITE, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + [{?eh,tc_start,{S,{init_per_group,g,[]}}}, + {?eh,tc_done,{S,{init_per_group,g,[]},ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{3,0,{0,0}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,tc_start,{S,{end_per_group,g,[]}}}, + {?eh,tc_done,{S,{end_per_group,g,[]},ok}}], + {?eh,tc_start,{S,end_per_suite}}, + {?eh,tc_done,{S,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]. + +%% test events help functions +contains(List) -> + fun(Proplist) when is_list(Proplist) -> + contains(List,Proplist) + end. + +contains([{not_in_order,List}|T],Rest) -> + contains_parallel(List,Rest), + contains(T,Rest); +contains([{Ele,Pos}|T] = L,[H|T2]) -> + case element(Pos,H) of + Ele -> + contains(T,T2); + _ -> + contains(L,T2) + end; +contains([Ele|T],[{Ele,_}|T2])-> + contains(T,T2); +contains([Ele|T],[Ele|T2])-> + contains(T,T2); +contains(List,[_|T]) -> + contains(List,T); +contains([],_) -> + match. + +contains_parallel([Key | T], Elems) -> + contains([Key],Elems), + contains_parallel(T,Elems); +contains_parallel([],_Elems) -> + match. + +not_contains(List) -> + fun(Proplist) when is_list(Proplist) -> + [] = [Ele || {Ele,_} <- Proplist, + Test <- List, + Test =:= Ele] + end. diff --git a/lib/common_test/test/ct_util_SUITE_data/ct_util_cth.erl b/lib/common_test/test/ct_util_SUITE_data/ct_util_cth.erl new file mode 100644 index 0000000000..34c1568a87 --- /dev/null +++ b/lib/common_test/test/ct_util_SUITE_data/ct_util_cth.erl @@ -0,0 +1,105 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +-module(ct_util_cth). + + +-include_lib("common_test/src/ct_util.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +%% Send a cth_error event if a callback is called with unexpected arguments + +%% CT Hooks +-compile(export_all). + +id(Opts) -> + erlang:system_time(second). + +init(Id, Opts) -> + {ok,ok}. + +pre_init_per_suite(Suite,Config,State) -> + maybe_sleep(?FUNCTION_NAME,Suite), + {Config, State}. + +post_init_per_suite(Suite,Config,Return,State) -> + maybe_sleep(?FUNCTION_NAME,Suite), + {Return, State}. + +pre_end_per_suite(Suite,Config,State) -> + maybe_sleep(?FUNCTION_NAME,Suite), + {Config, State}. + +post_end_per_suite(Suite,Config,Return,State) -> + maybe_sleep(?FUNCTION_NAME,Suite), + {Return, State}. + +pre_init_per_group(Suite, Group, Config, State) -> + maybe_sleep(?FUNCTION_NAME,Suite,Group), + {Config,State}. + +post_init_per_group(Suite, Group, Config,Return,State) -> + maybe_sleep(?FUNCTION_NAME,Suite,Group), + {Return,State}. + +pre_end_per_group(Suite, Group, Config, State) -> + maybe_sleep(?FUNCTION_NAME,Suite,Group), + {Config,State}. + +post_end_per_group(Suite, Group, Config,Return,State) -> + maybe_sleep(?FUNCTION_NAME,Suite,Group), + {Return,State}. + +pre_init_per_testcase(Suite, TC, Config, State) -> + maybe_sleep(?FUNCTION_NAME,Suite,TC), + {Config,State}. + +post_init_per_testcase(Suite, TC, Config,Return,State) -> + maybe_sleep(?FUNCTION_NAME,Suite,TC), + {Return,State}. + +pre_end_per_testcase(Suite, TC, Config, State) -> + maybe_sleep(?FUNCTION_NAME,Suite,TC), + {Config,State}. + +post_end_per_testcase(Suite, TC, Config,Return,State) -> + maybe_sleep(?FUNCTION_NAME,Suite,TC), + {Return,State}. + +%%%----------------------------------------------------------------- +maybe_sleep(FuncName,Suite) -> + maybe_sleep(FuncName,Suite,undefined). +maybe_sleep(FuncName,Suite,GroupOrTC) -> + case ct:get_config(FuncName) of + {Suite,GroupOrTC,Fail} -> + fail(Fail); + {Suite,Fail} when GroupOrTC=:=undefined -> + fail(Fail); + _ -> + ok + end. + +fail({timeout,T}) -> + timer:sleep(T); +fail(kill) -> + spawn_link(fun() -> ct:fail(hahahahahah) end), + timer:sleep(10000). + diff --git a/lib/common_test/test/ct_util_SUITE_data/curr_tc_SUITE.erl b/lib/common_test/test/ct_util_SUITE_data/curr_tc_SUITE.erl new file mode 100644 index 0000000000..b48ba4d24e --- /dev/null +++ b/lib/common_test/test/ct_util_SUITE_data/curr_tc_SUITE.erl @@ -0,0 +1,59 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(curr_tc_SUITE). + +-suite_defaults([{timetrap, {seconds, 3}}]). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include("ct.hrl"). + +init_per_suite(Config) -> + [{?MODULE,?FUNCTION_NAME}] = ct_util:get_testdata(curr_tc), + Config. + +end_per_suite(Config) -> + [{?MODULE,?FUNCTION_NAME}] = ct_util:get_testdata(curr_tc), + ok. + +init_per_group(_Group,Config) -> + [{?MODULE,?FUNCTION_NAME}] = ct_util:get_testdata(curr_tc), + Config. + +end_per_group(_Group,Config) -> + [{?MODULE,?FUNCTION_NAME}] = ct_util:get_testdata(curr_tc), + ok. + +all() -> + [tc1,tc2,{group,g}]. + +groups() -> + [{g,[tc1,tc2]}]. + +%% Test cases starts here. +tc1(_Config) -> + [{?MODULE,?FUNCTION_NAME}] = ct_util:get_testdata(curr_tc), + ok. + +tc2(_Config) -> + [{?MODULE,?FUNCTION_NAME}] = ct_util:get_testdata(curr_tc), + ok. |