diff options
Diffstat (limited to 'lib/test_server/src')
-rw-r--r-- | lib/test_server/src/test_server.app.src | 5 | ||||
-rw-r--r-- | lib/test_server/src/test_server.erl | 132 | ||||
-rw-r--r-- | lib/test_server/src/test_server_ctrl.erl | 435 | ||||
-rw-r--r-- | lib/test_server/src/test_server_internal.hrl | 9 | ||||
-rw-r--r-- | lib/test_server/src/test_server_node.erl | 10 | ||||
-rw-r--r-- | lib/test_server/src/test_server_sup.erl | 4 | ||||
-rw-r--r-- | lib/test_server/src/ts.erl | 21 | ||||
-rw-r--r-- | lib/test_server/src/ts.unix.config | 2 | ||||
-rw-r--r-- | lib/test_server/src/ts_run.erl | 5 |
9 files changed, 378 insertions, 245 deletions
diff --git a/lib/test_server/src/test_server.app.src b/lib/test_server/src/test_server.app.src index 42e78ed279..5672baa6ef 100644 --- a/lib/test_server/src/test_server.app.src +++ b/lib/test_server/src/test_server.app.src @@ -31,5 +31,8 @@ test_server, test_server_break_process]}, {applications, [kernel,stdlib]}, - {env, []}]}. + {env, []}, + {runtime_dependencies, ["tools-2.6.14","stdlib-2.0","runtime_tools-1.8.14", + "observer-2.0","kernel-3.0","inets-5.10", + "erts-6.0"]}]}. diff --git a/lib/test_server/src/test_server.erl b/lib/test_server/src/test_server.erl index 82672521f7..f63bfdec05 100644 --- a/lib/test_server/src/test_server.erl +++ b/lib/test_server/src/test_server.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% Copyright Ericsson AB 1996-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -22,7 +22,7 @@ %%% TEST_SERVER_CTRL INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -export([run_test_case_apply/1,init_target_info/0,init_purify/0]). --export([cover_compile/1,cover_analyse/3]). +-export([cover_compile/1,cover_analyse/2]). %%% TEST_SERVER_SUP INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -export([get_loc/1,set_tc_state/1]). @@ -80,8 +80,8 @@ init_purify() -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% cover_compile({App,Include,Exclude,Cross}) -> -%% {ok,AnalyseModules} | {error,Reason} +%% cover_compile(#cover{app=App,incl=Include,excl=Exclude,cross=Cross}) -> +%% {ok,#cover{mods=AnalyseModules}} | {error,Reason} %% %% App = atom() , name of application to be compiled %% Exclude = [atom()], list of modules to exclude @@ -90,33 +90,35 @@ init_purify() -> %% Cross = [atoms()], list of modules outside of App shat should be included %% in the cover compilation, but that shall not be part of %% the cover analysis for this application. +%% AnalyseModules = [atom()], list of successfully compiled modules %% -%% Cover compile the given application. Return {ok,AnalyseMods} if application -%% is found, else {error,application_not_found}. +%% Cover compile the given application. Return {ok,CoverInfo} if +%% compilation succeeds, else (if application is not found and there +%% are no modules to compile) {error,application_not_found}. -cover_compile({none,_Exclude,Include,Cross}) -> +cover_compile(CoverInfo=#cover{app=none,incl=Include,cross=Cross}) -> CrossMods = lists:flatmap(fun({_,M}) -> M end,Cross), CompileMods = Include++CrossMods, case length(CompileMods) of 0 -> io:fwrite("WARNING: No modules to cover compile!\n\n",[]), cover:start(), % start cover server anyway - {ok,[]}; + {ok,CoverInfo#cover{mods=[]}}; N -> io:fwrite("Cover compiling ~w modules - " "this may take some time... ",[N]), do_cover_compile(CompileMods), io:fwrite("done\n\n",[]), - {ok,Include} + {ok,CoverInfo#cover{mods=Include}} end; -cover_compile({App,all,Include,Cross}) -> +cover_compile(CoverInfo=#cover{app=App,excl=all,incl=Include,cross=Cross}) -> CrossMods = lists:flatmap(fun({_,M}) -> M end,Cross), CompileMods = Include++CrossMods, case length(CompileMods) of 0 -> io:fwrite("WARNING: No modules to cover compile!\n\n",[]), cover:start(), % start cover server anyway - {ok,[]}; + {ok,CoverInfo#cover{mods=[]}}; N -> io:fwrite("Cover compiling '~w' (~w files) - " "this may take some time... ",[App,N]), @@ -126,9 +128,9 @@ cover_compile({App,all,Include,Cross}) -> "~tp\n", [App,CompileMods]), do_cover_compile(CompileMods), io:fwrite("done\n\n",[]), - {ok,Include} + {ok,CoverInfo=#cover{mods=Include}} end; -cover_compile({App,Exclude,Include,Cross}) -> +cover_compile(CoverInfo=#cover{app=App,excl=Exclude,incl=Include,cross=Cross}) -> CrossMods = lists:flatmap(fun({_,M}) -> M end,Cross), case code:lib_dir(App) of {error,bad_name} -> @@ -146,7 +148,7 @@ cover_compile({App,Exclude,Include,Cross}) -> "~tp\n", [App,Include]), do_cover_compile(CompileMods), io:fwrite("done\n\n",[]), - {ok,Include} + {ok,CoverInfo#cover{mods=Include}} end; LibDir -> EbinDir = filename:join([LibDir,"ebin"]), @@ -158,13 +160,13 @@ cover_compile({App,Exclude,Include,Cross}) -> 0 -> io:fwrite("WARNING: No modules to cover compile!\n\n",[]), cover:start(), % start cover server anyway - {ok,[]}; + {ok,CoverInfo#cover{mods=[]}}; N -> io:fwrite("Cover compiling '~w' (~w files) - " "this may take some time... ",[App,N]), do_cover_compile(CompileMods), io:fwrite("done\n\n",[]), - {ok,AnalyseMods} + {ok,CoverInfo#cover{mods=AnalyseMods}} end end. @@ -174,9 +176,11 @@ module_names(Beams) -> do_cover_compile(Modules) -> - do_cover_compile1(lists:usort(Modules)). % remove duplicates + cover:start(), + pmap1(fun(M) -> do_cover_compile1(M) end,lists:usort(Modules)), + ok. -do_cover_compile1([M|Rest]) -> +do_cover_compile1(M) -> case {code:is_sticky(M),code:is_loaded(M)} of {true,_} -> code:unstick_mod(M), @@ -187,15 +191,13 @@ do_cover_compile1([M|Rest]) -> io:fwrite("\nWARNING: Could not cover compile ~w: ~p\n", [M,Error]) end, - code:stick_mod(M), - do_cover_compile1(Rest); + code:stick_mod(M); {false,false} -> case code:load_file(M) of {module,_} -> - do_cover_compile1([M|Rest]); + do_cover_compile1(M); Error -> - io:fwrite("\nWARNING: Could not load ~w: ~p\n",[M,Error]), - do_cover_compile1(Rest) + io:fwrite("\nWARNING: Could not load ~w: ~p\n",[M,Error]) end; {false,_} -> case cover:compile_beam(M) of @@ -204,23 +206,52 @@ do_cover_compile1([M|Rest]) -> Error -> io:fwrite("\nWARNING: Could not cover compile ~w: ~p\n", [M,Error]) - end, - do_cover_compile1(Rest) - end; -do_cover_compile1([]) -> - ok. + end + end. + +pmap1(Fun,List) -> + NTot = length(List), + NProcs = erlang:system_info(schedulers) * 2, + NPerProc = (NTot div NProcs) + 1, + + {[],Pids} = + lists:foldr( + fun(_,{L,Ps}) -> + {L1,L2} = if length(L)>=NPerProc -> lists:split(NPerProc,L); + true -> {L,[]} % last chunk + end, + {P,_Ref} = + spawn_monitor(fun() -> + exit(lists:map(Fun,L1)) + end), + {L2,[P|Ps]} + end, + {List,[]}, + lists:seq(1,NProcs)), + collect(Pids,[]). + +collect([],Acc) -> + lists:append(Acc); +collect([Pid|Pids],Acc) -> + receive + {'DOWN', _Ref, process, Pid, Result} -> + %% collect(lists:delete(Pid,Pids),[Result|Acc]) + collect(Pids,[Result|Acc]) + end. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% cover_analyse(Analyse,Modules,Stop) -> [{M,{Cov,NotCov,Details}}] +%% cover_analyse(Dir,#cover{level=Analyse,mods=Modules,stop=Stop) -> +%% [{M,{Cov,NotCov,Details}}] %% -%% Analyse = {details,Dir} | details | {overview,void()} | overview +%% Dir = string() +%% Analyse = details | overview %% Modules = [atom()], the modules to analyse %% -%% Cover analysis. If Analyse=={details,Dir} analyse_to_file is used. +%% Cover analysis. If Analyse==details analyse_to_file is used. %% -%% If Analyse=={overview,Dir} analyse_to_file is not used, only an -%% overview containing the number of covered/not covered lines in each -%% module. +%% If Analyse==overview analyse_to_file is not used, only an overview +%% containing the number of covered/not covered lines in each module. %% %% Also, cover data will be exported to a file called all.coverdata in %% the given directory. @@ -235,11 +266,11 @@ do_cover_compile1([]) -> %% which means that the modules will stay cover compiled. Note that %% this is only recommended if the erlang node is being terminated %% after the test is completed. -cover_analyse(Analyse,Modules,Stop) -> - print(stdout, "Cover analysing...\n", []), +cover_analyse(Dir,#cover{level=Analyse,mods=Modules,stop=Stop}) -> + io:fwrite(user, "Cover analysing... ", []), DetailsFun = case Analyse of - {details,Dir} -> + details -> case cover:export(filename:join(Dir,"all.coverdata")) of ok -> fun(M) -> @@ -256,7 +287,7 @@ cover_analyse(Analyse,Modules,Stop) -> Error -> fun(_) -> Error end end; - {overview,Dir} -> + overview -> case cover:export(filename:join(Dir,"all.coverdata")) of ok -> fun(_) -> undefined end; @@ -264,17 +295,19 @@ cover_analyse(Analyse,Modules,Stop) -> fun(_) -> Error end end end, - R = pmap( + R = pmap2( fun(M) -> case cover:analyse(M,module) of {ok,{M,{Cov,NotCov}}} -> {M,{Cov,NotCov,DetailsFun(M)}}; Err -> - io:fwrite("WARNING: Analysis failed for ~w. Reason: ~p\n", + io:fwrite(user, + "\nWARNING: Analysis failed for ~w. Reason: ~p\n", [M,Err]), {M,Err} end end, Modules), + io:fwrite(user, "done\n\n", []), case Stop of true -> @@ -286,12 +319,12 @@ cover_analyse(Analyse,Modules,Stop) -> end, R. -pmap(Fun,List) -> +pmap2(Fun,List) -> Collector = self(), Pids = lists:map(fun(E) -> spawn(fun() -> - Collector ! {res,self(),Fun(E)} - end) + Collector ! {res,self(),Fun(E)} + end) end, List), lists:map(fun(Pid) -> receive @@ -300,7 +333,6 @@ pmap(Fun,List) -> end end, Pids). - do_cover_for_node(Node,CoverFunc) -> do_cover_for_node(Node,CoverFunc,true). do_cover_for_node(Node,CoverFunc,StickUnstick) -> @@ -444,7 +476,7 @@ run_test_case_apply(Mod, Func, Args, Name, RunInit, TimetrapData) -> %% If this process (group leader of the test case) terminates before %% all messages have been replied back to the io server, the io server %% hangs. Fixed by the 20 milli timeout check here, and by using monitor in -%% io.erl (livrem OCH hangslen mao :) +%% io.erl. %% %% A test case is known to have failed if it returns {'EXIT', _} tuple, %% or sends a message {failed, File, Line} to it's group_leader @@ -673,7 +705,7 @@ handle_tc_exit({testcase_aborted,{user_timetrap_error,_}=Msg,_}, St) -> spawn_fw_call(Mod, Func, Config, Pid, Msg, unknown, self()), St; handle_tc_exit(Reason, #st{status={framework,FwMod,FwFunc}, - config=Config,pid=Pid}=St) -> + config=Config,pid=Pid}=St) -> R = case Reason of {timetrap_timeout,TVal,_} -> {timetrap,TVal}; @@ -1208,6 +1240,10 @@ do_init_per_testcase(Mod, Args) -> "a Config list.\n",[]}, {skip,{failed,{Mod,init_per_testcase,bad_return}}} catch + throw:{Skip,Reason} when Skip =:= skip; Skip =:= skipped -> + {skip,Reason}; + exit:{Skip,Reason} when Skip =:= skip; Skip =:= skipped -> + {skip,Reason}; throw:Other -> set_loc(erlang:get_stacktrace()), Line = get_loc(), @@ -1378,6 +1414,10 @@ ts_tc(M, F, A) -> Result = try apply(M, F, A) catch + throw:{skip, Reason} -> {skip, Reason}; + throw:{skipped, Reason} -> {skip, Reason}; + exit:{skip, Reason} -> {skip, Reason}; + exit:{skipped, Reason} -> {skip, Reason}; Type:Reason -> Stk = erlang:get_stacktrace(), set_loc(Stk), diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index dcf905db24..af8921fe75 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2013. All Rights Reserved. +%% Copyright Ericsson AB 2002-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -52,7 +52,9 @@ -export([reject_io_reqs/1, get_levels/0, set_levels/3]). -export([multiply_timetraps/1, scale_timetraps/1, get_timetrap_parameters/0]). -export([create_priv_dir/1]). --export([cover/2, cover/3, cover/8, cross_cover_analyse/2, trc/1, stop_trace/0]). +-export([cover/1, cover/2, cover/3, + cover_compile/7, cover_analyse/2, cross_cover_analyse/2, + trc/1, stop_trace/0]). -export([testcase_callback/1]). -export([set_random_seed/1]). -export([kill_slavenodes/0]). @@ -409,11 +411,26 @@ cover(App, Analyse) when is_atom(App) -> cover(CoverFile, Analyse) -> cover(none, CoverFile, Analyse). cover(App, CoverFile, Analyse) -> - controller_call({cover,{App,CoverFile},Analyse,true}). -cover(App, CoverFile, Exclude, Include, Cross, Export, Analyse, Stop) -> - controller_call({cover, - {App,{CoverFile,Exclude,Include,Cross,Export}}, - Analyse,Stop}). + {Excl,Incl,Cross} = read_cover_file(CoverFile), + CoverInfo = #cover{app=App, + file=CoverFile, + excl=Excl, + incl=Incl, + cross=Cross, + level=Analyse}, + controller_call({cover,CoverInfo}). + +cover(CoverInfo) -> + controller_call({cover,CoverInfo}). + +cover_compile(App,File,Excl,Incl,Cross,Analyse,Stop) -> + cover_compile(#cover{app=App, + file=File, + excl=Excl, + incl=Incl, + cross=Cross, + level=Analyse, + stop=Stop}). testcase_callback(ModFunc) -> controller_call({testcase_callback,ModFunc}). @@ -563,7 +580,7 @@ handle_call({add_job,Dir,Name,TopCase,Skip}, _From, State) -> ExtraTools = case State#state.cover of false -> []; - {App,Analyse,Stop} -> [{cover,App,Analyse,Stop}] + CoverInfo -> [{cover,CoverInfo}] end, ExtraTools1 = case State#state.random_seed of @@ -816,13 +833,13 @@ handle_call(stop_trace, _From, State) -> {reply,R,State#state{trc=false}}; %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% handle_call({cover,App,Analyse,Stop}, _, State) -> ok | {error,Reason} +%% handle_call({cover,CoverInfo}, _, State) -> ok | {error,Reason} %% -%% All modules inn application App are cover compiled -%% Analyse indicates on which level the coverage should be analysed +%% Set specification of cover analysis to be used when running tests +%% (see start_extra_tools/1 and stop_extra_tools/1) -handle_call({cover,App,Analyse,Stop}, _From, State) -> - {reply,ok,State#state{cover={App,Analyse,Stop}}}; +handle_call({cover,CoverInfo}, _From, State) -> + {reply,ok,State#state{cover=CoverInfo}}; %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% handle_call({create_priv_dir,Value}, _, State) -> ok | {error,Reason} @@ -1203,11 +1220,10 @@ elapsed_time(Before, After) -> start_extra_tools(ExtraTools) -> start_extra_tools(ExtraTools, []). -start_extra_tools([{cover,App,Analyse,Stop} | ExtraTools], Started) -> - case cover_compile(App) of - {ok,AnalyseMods} -> - start_extra_tools(ExtraTools, - [{cover,App,Analyse,AnalyseMods,Stop}|Started]); +start_extra_tools([{cover,CoverInfo} | ExtraTools], Started) -> + case start_cover(CoverInfo) of + {ok,NewCoverInfo} -> + start_extra_tools(ExtraTools,[{cover,NewCoverInfo}|Started]); {error,_} -> start_extra_tools(ExtraTools, Started) end; @@ -1226,8 +1242,8 @@ stop_extra_tools(ExtraTools) -> end, stop_extra_tools(ExtraTools, TestDir). -stop_extra_tools([{cover,App,Analyse,AnalyseMods,Stop}|ExtraTools], TestDir) -> - cover_analyse(App, Analyse, AnalyseMods, Stop, TestDir), +stop_extra_tools([{cover,CoverInfo}|ExtraTools], TestDir) -> + stop_cover(CoverInfo,TestDir), stop_extra_tools(ExtraTools, TestDir); %%stop_extra_tools([_ | ExtraTools], TestDir) -> %% stop_extra_tools(ExtraTools, TestDir); @@ -1407,7 +1423,7 @@ remove_conf([{conf, _Ref, Props, _MF}|Cases], NoConf, Repeats) -> end; remove_conf([{make,_Ref,_MF}|Cases], NoConf, Repeats) -> remove_conf(Cases, NoConf, Repeats); -remove_conf([{skip_case,{{_M,all},_Cmt}}|Cases], NoConf, Repeats) -> +remove_conf([{skip_case,{{_M,all},_Cmt},_Mode}|Cases], NoConf, Repeats) -> remove_conf(Cases, NoConf, Repeats); remove_conf([{skip_case,{Type,_Ref,_MF,_Cmt}}|Cases], NoConf, Repeats) when Type==conf; @@ -1431,7 +1447,7 @@ remove_conf([], NoConf, true) -> remove_conf([], NoConf, false) -> lists:reverse(NoConf). -get_suites([{skip_case,{{Mod,_Func},_Cmt}}|Tests], Mods) when is_atom(Mod) -> +get_suites([{skip_case,{{Mod,_F},_Cmt},_Mode}|Tests], Mods) when is_atom(Mod) -> case add_mod(Mod, Mods) of true -> get_suites(Tests, [Mod|Mods]); false -> get_suites(Tests, Mods) @@ -1569,16 +1585,24 @@ do_test_cases(TopCases, SkipCases, ok end end, - + CoverLog = + case get(test_server_cover_log_dir) of + undefined -> + ?coverlog_name; + AbsLogDir -> + AbsLog = filename:join(AbsLogDir,?coverlog_name), + make_relative(AbsLog, TestDir) + end, print(html, "<p><ul>\n" "<li><a href=\"~ts\">Full textual log</a></li>\n" "<li><a href=\"~ts\">Coverage log</a></li>\n" "<li><a href=\"~ts\">Unexpected I/O log</a></li>\n</ul></p>\n", - [?suitelog_name,?coverlog_name,?unexpected_io_log]), + [?suitelog_name,CoverLog,?unexpected_io_log]), print(html, "<p>~ts</p>\n" ++ - xhtml("<table bgcolor=\"white\" border=\"3\" cellpadding=\"5\">", + xhtml(["<table bgcolor=\"white\" border=\"3\" cellpadding=\"5\">\n", + "<thead>\n"], ["<table id=\"",?sortable_table_name,"\">\n", "<thead>\n"]) ++ "<tr><th>Num</th><th>Module</th><th>Group</th>" ++ @@ -1833,7 +1857,7 @@ html_isolate_modules(List, FwMod) -> html_isolate_modules(List, sets:new(), FwMod). html_isolate_modules([], Set, _) -> sets:to_list(Set); -html_isolate_modules([{skip_case,_}|Cases], Set, FwMod) -> +html_isolate_modules([{skip_case,{_Case,_Cmt},_Mode}|Cases], Set, FwMod) -> html_isolate_modules(Cases, Set, FwMod); html_isolate_modules([{conf,_Ref,Props,{FwMod,_Func}}|Cases], Set, FwMod) -> Set1 = case proplists:get_value(suite, Props) of @@ -1937,26 +1961,30 @@ copy_html_file(Src, DestDir) -> add_init_and_end_per_suite([{make,_,_}=Case|Cases], LastMod, LastRef, FwMod) -> [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; -add_init_and_end_per_suite([{skip_case,{{Mod,all},_}}=Case|Cases], LastMod, +add_init_and_end_per_suite([{skip_case,{{Mod,all},_},_}=Case|Cases], LastMod, LastRef, FwMod) when Mod =/= LastMod -> {PreCases, NextMod, NextRef} = do_add_end_per_suite_and_skip(LastMod, LastRef, Mod, FwMod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; -add_init_and_end_per_suite([{skip_case,{{Mod,_},_}}=Case|Cases], LastMod, - LastRef, FwMod) when Mod =/= LastMod -> + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, + NextRef, FwMod)]; +add_init_and_end_per_suite([{skip_case,{{Mod,_},_Cmt},_Mode}=Case|Cases], + LastMod, LastRef, FwMod) when Mod =/= LastMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; -add_init_and_end_per_suite([{skip_case,{conf,_,{Mod,_},_},_}=Case|Cases], LastMod, - LastRef, FwMod) when Mod =/= LastMod -> + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, + NextRef, FwMod)]; +add_init_and_end_per_suite([{skip_case,{conf,_,{Mod,_},_},_}=Case|Cases], + LastMod, LastRef, FwMod) when Mod =/= LastMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, + NextRef, FwMod)]; add_init_and_end_per_suite([{skip_case,{conf,_,{Mod,_},_}}=Case|Cases], LastMod, LastRef, FwMod) when Mod =/= LastMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, + NextRef, FwMod)]; add_init_and_end_per_suite([{conf,Ref,Props,{FwMod,Func}}=Case|Cases], LastMod, LastRef, FwMod) -> %% if Mod == FwMod, this conf test is (probably) a test case group where @@ -1977,7 +2005,8 @@ add_init_and_end_per_suite([{conf,_,_,{Mod,_}}=Case|Cases], LastMod, LastRef, FwMod) when Mod =/= LastMod, Mod =/= FwMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, + NextRef, FwMod)]; add_init_and_end_per_suite([SkipCase|Cases], LastMod, LastRef, FwMod) when element(1,SkipCase) == skip_case -> [SkipCase|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; @@ -1987,12 +2016,14 @@ add_init_and_end_per_suite([{Mod,_}=Case|Cases], LastMod, LastRef, FwMod) when Mod =/= LastMod, Mod =/= FwMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, + NextRef, FwMod)]; add_init_and_end_per_suite([{Mod,_,_}=Case|Cases], LastMod, LastRef, FwMod) when Mod =/= LastMod, Mod =/= FwMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, + NextRef, FwMod)]; add_init_and_end_per_suite([Case|Cases], LastMod, LastRef, FwMod)-> [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; add_init_and_end_per_suite([], _LastMod, undefined, _FwMod) -> @@ -2169,7 +2200,7 @@ run_test_cases(TestSpec, Config, TimetrapData) -> %% comment (which gets printed in the log files) describes why the case %% was skipped. %% -%% {skip_case,{Case,Comment}} A normal test case skipped by the user. +%% {skip_case,{Case,Comment},Mode} A normal test case skipped by the user. %% The comment (which gets printed in the log files) describes why the %% case was skipped. %% @@ -2342,7 +2373,7 @@ run_test_cases_loop([{SkipTag,{Type,Ref,Case,Comment},SkipMode}|Cases], ParentRef -> Reason = {group_result,GrName,failed}, skip_cases_upto(ParentRef, Cases, - Reason, tc, Mode, + Reason, tc, ParentMode, SkipTag) end; false -> @@ -2401,22 +2432,27 @@ run_test_cases_loop([{auto_skip_case,{Case,Comment},SkipMode}|Cases], Config, TimetrapData, Mode, Status) -> {Mod,Func} = skip_case(auto, undefined, get(test_server_case_num)+1, Case, Comment, is_io_buffered(), SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip,{Mod,Func,Comment}]), + test_server_sup:framework_call(report, [tc_auto_skip, + {Mod,{Func,get_name(SkipMode)}, + Comment}]), run_test_cases_loop(Cases, Config, TimetrapData, Mode, update_status(skipped, Mod, Func, Status)); -run_test_cases_loop([{skip_case,{{Mod,all}=Case,Comment}}|Cases], +run_test_cases_loop([{skip_case,{{Mod,all}=Case,Comment},SkipMode}|Cases], Config, TimetrapData, Mode, Status) -> - skip_case(user, undefined, 0, Case, Comment, false, Mode), + skip_case(user, undefined, 0, Case, Comment, false, SkipMode), test_server_sup:framework_call(report, [tc_user_skip, - {Mod,all,Comment}]), + {Mod,{all,get_name(SkipMode)}, + Comment}]), run_test_cases_loop(Cases, Config, TimetrapData, Mode, Status); -run_test_cases_loop([{skip_case,{Case,Comment}}|Cases], +run_test_cases_loop([{skip_case,{Case,Comment},SkipMode}|Cases], Config, TimetrapData, Mode, Status) -> {Mod,Func} = skip_case(user, undefined, get(test_server_case_num)+1, - Case, Comment, is_io_buffered()), - test_server_sup:framework_call(report, [tc_user_skip,{Mod,Func,Comment}]), + Case, Comment, is_io_buffered(), SkipMode), + test_server_sup:framework_call(report, [tc_user_skip, + {Mod,{Func,get_name(SkipMode)}, + Comment}]), run_test_cases_loop(Cases, Config, TimetrapData, Mode, update_status(skipped, Mod, Func, Status)); @@ -2430,8 +2466,9 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, {Ref,Ref} -> case check_props(parallel, tl(Mode0)) of false -> - %% this is an end conf for a top level parallel group, collect - %% results from the test case processes and calc total time + %% this is an end conf for a top level parallel group, + %% collect results from the test case processes + %% and calc total time OkSkipFail = handle_test_case_io_and_status(), file:set_cwd(filename:dirname(get(test_server_dir))), After = ?now, @@ -2955,7 +2992,6 @@ get_tc_results([{_,{OkSkipFail,_}} | _Status]) -> get_tc_results([]) -> % in case init_per_suite crashed {[],[],[]}. - conf(Ref, Props) -> {Ref,Props,?now}. @@ -3160,10 +3196,6 @@ random_order(N, {Pos,NewSeed}, IxCases, Shuffled) -> %% SendSync determines if start and finished messages must be sent so %% that the printouts can be buffered and handled in order with io from %% parallel processes. - -skip_case(Type, Ref, CaseNum, Case, Comment, SendSync) -> - skip_case(Type, Ref, CaseNum, Case, Comment, SendSync, []). - skip_case(Type, Ref, CaseNum, Case, Comment, SendSync, Mode) -> MF = {Mod,Func} = case Case of {M,F,_A} -> {M,F}; @@ -3241,7 +3273,7 @@ skip_case1(Type, CaseNum, Mod, Func, Comment, Mode) -> %% SkipType = skip_case | auto_skip_case %% Mark all cases tagged with Ref as skipped. -skip_cases_upto(Ref, Cases, Reason, Origin, Mode, SkipType) -> +skip_cases_upto(Ref, Cases, Reason, Origin, Mode, SkipType) -> {_,Modified,Rest} = modify_cases_upto(Ref, {skip,Reason,Origin,Mode,SkipType}, Cases), Modified++Rest. @@ -3326,21 +3358,34 @@ modify_cases_upto1(Ref, {copy,NewRef}, {[C|Orig],[{skip_case,{Type,NewRef,MF,Cmt}}|Alt],T}; %% next is a skip_case, could be one test case or 'all' in suite, we must proceed -modify_cases_upto1(Ref, ModOp, [{skip_case,{_F,_Cmt}}=MF|T], Orig, Alt) -> +modify_cases_upto1(Ref, ModOp, [{skip_case,{_F,_Cmt},_Mode}=MF|T], Orig, Alt) -> modify_cases_upto1(Ref, ModOp, T, [MF|Orig], [MF|Alt]); %% next is a normal case (possibly in a sequence), mark as skipped, or copy, and proceed -modify_cases_upto1(Ref, {skip,Reason,_,_,skip_case}=Op, +modify_cases_upto1(Ref, {skip,Reason,_,Mode,skip_case}=Op, [{_M,_F}=MF|T], Orig, Alt) -> - modify_cases_upto1(Ref, Op, T, Orig, [{skip_case,{MF,Reason}}|Alt]); + modify_cases_upto1(Ref, Op, T, Orig, [{skip_case,{MF,Reason},Mode}|Alt]); modify_cases_upto1(Ref, {skip,Reason,_,Mode,auto_skip_case}=Op, [{_M,_F}=MF|T], Orig, Alt) -> modify_cases_upto1(Ref, Op, T, Orig, [{auto_skip_case,{MF,Reason},Mode}|Alt]); modify_cases_upto1(Ref, CopyOp, [{_M,_F}=MF|T], Orig, Alt) -> modify_cases_upto1(Ref, CopyOp, T, [MF|Orig], [MF|Alt]); +%% next is a conf case, modify the Mode arg to keep track of sub groups +modify_cases_upto1(Ref, {skip,Reason,FType,Mode,SkipType}, + [{conf,OtherRef,Props,_MF}|T], Orig, Alt) -> + case hd(Mode) of + {OtherRef,_,_} -> % end conf + modify_cases_upto1(Ref, {skip,Reason,FType,tl(Mode),SkipType}, + T, Orig, Alt); + _ -> % start conf + Mode1 = [conf(OtherRef,Props)|Mode], + modify_cases_upto1(Ref, {skip,Reason,FType,Mode1,SkipType}, + T, Orig, Alt) + end; + %% next is some other case, ignore or copy -modify_cases_upto1(Ref, {skip,_,_,_,_}=Op, [_|T], Orig, Alt) -> +modify_cases_upto1(Ref, {skip,_,_,_,_}=Op, [_Other|T], Orig, Alt) -> modify_cases_upto1(Ref, Op, T, Orig, Alt); modify_cases_upto1(Ref, CopyOp, [C|T], Orig, Alt) -> modify_cases_upto1(Ref, CopyOp, T, [C|Orig], [C|Alt]). @@ -3665,12 +3710,13 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, update_config(hd(Args), [{priv_dir,PrivDir++"/"}, {tc_logfile,MinorName}]) end, - + GrName = get_name(Mode), test_server_sup:framework_call(report, - [tc_start,{{Mod,Func},MinorName}]), + [tc_start,{{Mod,{Func,GrName}}, + MinorName}]), print_props((RunInit==skip_init), get_props(Mode)), - GroupName = case get_name(Mode) of + GrNameStr = case GrName of undefined -> ""; Name -> cast_to_list(Name) end, @@ -3683,14 +3729,14 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, "<td>" ++ Col0 ++ "~ts" ++ Col1 ++ "</td>" "<td><a href=\"~ts\">~w</a></td>" "<td><a href=\"~ts#top\"><</a> <a href=\"~ts#end\">></a></td>", - [num2str(Num),fw_name(Mod),GroupName,EncMinorBase,Func, + [num2str(Num),fw_name(Mod),GrNameStr,EncMinorBase,Func, EncMinorBase,EncMinorBase]), do_unless_parallel(Main, fun erlang:yield/0), %% run the test case {Result,DetectedFail,ProcsBefore,ProcsAfter} = - run_test_case_apply(Num, Mod, Func, [UpdatedArgs], get_name(Mode), + run_test_case_apply(Num, Mod, Func, [UpdatedArgs], GrName, RunInit, TimetrapData), {Time,RetVal,Loc,Opts,Comment} = case Result of @@ -3709,41 +3755,41 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, Status = case {Time,RetVal} of {died,{timetrap_timeout,TimetrapTimeout}} -> - progress(failed, Num, Mod, Func, Loc, + progress(failed, Num, Mod, Func, GrName, Loc, timetrap_timeout, TimetrapTimeout, Comment, Style); {died,Reason} -> - progress(failed, Num, Mod, Func, Loc, Reason, + progress(failed, Num, Mod, Func, GrName, Loc, Reason, Time, Comment, Style); {_,{'EXIT',{Skip,Reason}}} when Skip==skip; Skip==skipped; Skip==auto_skip -> - progress(skip, Num, Mod, Func, Loc, Reason, + progress(skip, Num, Mod, Func, GrName, Loc, Reason, Time, Comment, Style); {_,{'EXIT',_Pid,{Skip,Reason}}} when Skip==skip; Skip==skipped -> - progress(skip, Num, Mod, Func, Loc, Reason, + progress(skip, Num, Mod, Func, GrName, Loc, Reason, Time, Comment, Style); {_,{'EXIT',_Pid,Reason}} -> - progress(failed, Num, Mod, Func, Loc, Reason, + progress(failed, Num, Mod, Func, GrName, Loc, Reason, Time, Comment, Style); {_,{'EXIT',Reason}} -> - progress(failed, Num, Mod, Func, Loc, Reason, + progress(failed, Num, Mod, Func, GrName, Loc, Reason, Time, Comment, Style); {_,{Fail,Reason}} when Fail =:= fail; Fail =:= failed -> - progress(failed, Num, Mod, Func, Loc, Reason, + progress(failed, Num, Mod, Func, GrName, Loc, Reason, Time, Comment, Style); {_,Reason={auto_skip,_Why}} -> - progress(skip, Num, Mod, Func, Loc, Reason, + progress(skip, Num, Mod, Func, GrName, Loc, Reason, Time, Comment, Style); {_,{Skip,Reason}} when Skip==skip; Skip==skipped -> - progress(skip, Num, Mod, Func, Loc, Reason, + progress(skip, Num, Mod, Func, GrName, Loc, Reason, Time, Comment, Style); {Time,RetVal} -> case DetectedFail of [] -> - progress(ok, Num, Mod, Func, Loc, RetVal, + progress(ok, Num, Mod, Func, GrName, Loc, RetVal, Time, Comment, Style); Reason -> - progress(failed, Num, Mod, Func, Loc, Reason, + progress(failed, Num, Mod, Func, GrName, Loc, Reason, Time, Comment, Style) end end, @@ -3848,7 +3894,7 @@ num2str(N) -> integer_to_list(N). %% Note: Strings that are to be written to the minor log must %% be prefixed with "=== " here, or the indentation will be wrong. -progress(skip, CaseNum, Mod, Func, Loc, Reason, Time, +progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time, Comment, {St0,St1}) -> {Reason1,{Color,Ret,ReportTag}} = if_auto_skip(Reason, @@ -3857,7 +3903,7 @@ progress(skip, CaseNum, Mod, Func, Loc, Reason, Time, print(major, "=result ~w: ~p", [ReportTag,Reason1]), print(1, "*** SKIPPED ~ts ***", [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), - test_server_sup:framework_call(report, [tc_done,{Mod,Func, + test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName}, {ReportTag,Reason1}}]), ReasonStr = reason_to_string(Reason1), ReasonStr1 = lists:flatten([string:strip(S,left) || @@ -3882,13 +3928,13 @@ progress(skip, CaseNum, Mod, Func, Loc, Reason, Time, print(minor, "=== reason = ~ts", [ReasonStr1]), Ret; -progress(failed, CaseNum, Mod, Func, Loc, timetrap_timeout, T, +progress(failed, CaseNum, Mod, Func, GrName, Loc, timetrap_timeout, T, Comment0, {St0,St1}) -> print(major, "=result failed: timeout, ~p", [Loc]), print(1, "*** FAILED ~ts ***", [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), test_server_sup:framework_call(report, - [tc_done,{Mod,Func, + [tc_done,{Mod,{Func,GrName}, {failed,timetrap_timeout}}]), FormatLastLoc = test_server_sup:format_loc(get_last_loc(Loc)), ErrorReason = io_lib:format("{timetrap_timeout,~ts}", [FormatLastLoc]), @@ -3908,13 +3954,13 @@ progress(failed, CaseNum, Mod, Func, Loc, timetrap_timeout, T, print(minor, "=== reason = timetrap timeout", []), failed; -progress(failed, CaseNum, Mod, Func, Loc, {testcase_aborted,Reason}, _T, +progress(failed, CaseNum, Mod, Func, GrName, Loc, {testcase_aborted,Reason}, _T, Comment0, {St0,St1}) -> print(major, "=result failed: testcase_aborted, ~p", [Loc]), print(1, "*** FAILED ~ts ***", [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), test_server_sup:framework_call(report, - [tc_done,{Mod,Func, + [tc_done,{Mod,{Func,GrName}, {failed,testcase_aborted}}]), FormatLastLoc = test_server_sup:format_loc(get_last_loc(Loc)), ErrorReason = io_lib:format("{testcase_aborted,~ts}", [FormatLastLoc]), @@ -3934,12 +3980,12 @@ progress(failed, CaseNum, Mod, Func, Loc, {testcase_aborted,Reason}, _T, print(minor, "=== reason = {testcase_aborted,~p}", [Reason]), failed; -progress(failed, CaseNum, Mod, Func, unknown, Reason, Time, +progress(failed, CaseNum, Mod, Func, GrName, unknown, Reason, Time, Comment0, {St0,St1}) -> print(major, "=result failed: ~p, ~w", [Reason,unknown]), print(1, "*** FAILED ~ts ***", [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), - test_server_sup:framework_call(report, [tc_done,{Mod,Func, + test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName}, {failed,Reason}}]), TimeStr = io_lib:format(if is_float(Time) -> "~.3fs"; true -> "~w" @@ -3970,12 +4016,12 @@ progress(failed, CaseNum, Mod, Func, unknown, Reason, Time, print(minor, "=== reason = " ++ FStr, [FormattedReason]), failed; -progress(failed, CaseNum, Mod, Func, Loc, Reason, Time, +progress(failed, CaseNum, Mod, Func, GrName, Loc, Reason, Time, Comment0, {St0,St1}) -> print(major, "=result failed: ~p, ~p", [Reason,Loc]), print(1, "*** FAILED ~ts ***", [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), - test_server_sup:framework_call(report, [tc_done,{Mod,Func, + test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName}, {failed,Reason}}]), TimeStr = io_lib:format(if is_float(Time) -> "~.3fs"; true -> "~w" @@ -3997,10 +4043,10 @@ progress(failed, CaseNum, Mod, Func, Loc, Reason, Time, print(minor, "=== reason = " ++ FStr, [FormattedReason]), failed; -progress(ok, _CaseNum, Mod, Func, _Loc, RetVal, Time, +progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time, Comment0, {St0,St1}) -> print(minor, "successfully completed test case", []), - test_server_sup:framework_call(report, [tc_done,{Mod,Func,ok}]), + test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName},ok}]), Comment = case RetVal of {comment,RetComment} -> @@ -4489,18 +4535,18 @@ update_config(Config, []) -> collect_all_cases(Top, Skip) when is_list(Skip) -> Result = - case collect_cases(Top, #cc{mod=[],skip=Skip}) of + case collect_cases(Top, #cc{mod=[],skip=Skip}, []) of {ok,Cases,_St} -> Cases; Other -> Other end, Result. -collect_cases([], St) -> {ok,[],St}; -collect_cases([Case|Cs0], St0) -> - case collect_cases(Case, St0) of +collect_cases([], St, _) -> {ok,[],St}; +collect_cases([Case|Cs0], St0, Mode) -> + case collect_cases(Case, St0, Mode) of {ok,FlatCases1,St1} -> - case collect_cases(Cs0, St1) of + case collect_cases(Cs0, St1, Mode) of {ok,FlatCases2,St} -> {ok,FlatCases1 ++ FlatCases2,St}; {error,_Reason} = Error -> Error @@ -4509,39 +4555,41 @@ collect_cases([Case|Cs0], St0) -> end; -collect_cases({module,Case}, St) when is_atom(Case), is_atom(St#cc.mod) -> - collect_case({St#cc.mod,Case}, St); -collect_cases({module,Mod,Case}, St) -> - collect_case({Mod,Case}, St); -collect_cases({module,Mod,Case,Args}, St) -> - collect_case({Mod,Case,Args}, St); - -collect_cases({dir,SubDir}, St) -> - collect_files(SubDir, "*_SUITE", St); -collect_cases({dir,SubDir,Pattern}, St) -> - collect_files(SubDir, Pattern++"*", St); - -collect_cases({conf,InitF,CaseList,FinMF}, St) when is_atom(InitF) -> - collect_cases({conf,[],{St#cc.mod,InitF},CaseList,FinMF}, St); -collect_cases({conf,InitMF,CaseList,FinF}, St) when is_atom(FinF) -> - collect_cases({conf,[],InitMF,CaseList,{St#cc.mod,FinF}}, St); -collect_cases({conf,InitMF,CaseList,FinMF}, St0) -> - collect_cases({conf,[],InitMF,CaseList,FinMF}, St0); -collect_cases({conf,Props,InitF,CaseList,FinMF}, St) when is_atom(InitF) -> +collect_cases({module,Case}, St, Mode) when is_atom(Case), is_atom(St#cc.mod) -> + collect_case({St#cc.mod,Case}, St, Mode); +collect_cases({module,Mod,Case}, St, Mode) -> + collect_case({Mod,Case}, St, Mode); +collect_cases({module,Mod,Case,Args}, St, Mode) -> + collect_case({Mod,Case,Args}, St, Mode); + +collect_cases({dir,SubDir}, St, Mode) -> + collect_files(SubDir, "*_SUITE", St, Mode); +collect_cases({dir,SubDir,Pattern}, St, Mode) -> + collect_files(SubDir, Pattern++"*", St, Mode); + +collect_cases({conf,InitF,CaseList,FinMF}, St, Mode) when is_atom(InitF) -> + collect_cases({conf,[],{St#cc.mod,InitF},CaseList,FinMF}, St, Mode); +collect_cases({conf,InitMF,CaseList,FinF}, St, Mode) when is_atom(FinF) -> + collect_cases({conf,[],InitMF,CaseList,{St#cc.mod,FinF}}, St, Mode); +collect_cases({conf,InitMF,CaseList,FinMF}, St0, Mode) -> + collect_cases({conf,[],InitMF,CaseList,FinMF}, St0, Mode); +collect_cases({conf,Props,InitF,CaseList,FinMF}, St, Mode) when is_atom(InitF) -> case init_props(Props) of {error,_} -> {ok,[],St}; Props1 -> - collect_cases({conf,Props1,{St#cc.mod,InitF},CaseList,FinMF}, St) + collect_cases({conf,Props1,{St#cc.mod,InitF},CaseList,FinMF}, + St, Mode) end; -collect_cases({conf,Props,InitMF,CaseList,FinF}, St) when is_atom(FinF) -> +collect_cases({conf,Props,InitMF,CaseList,FinF}, St, Mode) when is_atom(FinF) -> case init_props(Props) of {error,_} -> {ok,[],St}; Props1 -> - collect_cases({conf,Props1,InitMF,CaseList,{St#cc.mod,FinF}}, St) + collect_cases({conf,Props1,InitMF,CaseList,{St#cc.mod,FinF}}, + St, Mode) end; -collect_cases({conf,Props,InitMF,CaseList,FinMF} = Conf, St) -> +collect_cases({conf,Props,InitMF,CaseList,FinMF} = Conf, St, Mode) -> case init_props(Props) of {error,_} -> {ok,[],St}; @@ -4549,13 +4597,13 @@ collect_cases({conf,Props,InitMF,CaseList,FinMF} = Conf, St) -> Ref = make_ref(), Skips = St#cc.skip, Props2 = [{suite,St#cc.mod} | lists:delete(suite,Props1)], - Mode = [{Ref,Props2,undefined}], + Mode1 = [{Ref,Props2,undefined} | Mode], case in_skip_list({St#cc.mod,Conf}, Skips) of {true,Comment} -> % conf init skipped - {ok,[{skip_case,{conf,Ref,InitMF,Comment},Mode} | + {ok,[{skip_case,{conf,Ref,InitMF,Comment},Mode1} | [] ++ [{conf,Ref,[],FinMF}]],St}; {true,Name,Comment} when is_atom(Name) -> % all cases skipped - case collect_cases(CaseList, St) of + case collect_cases(CaseList, St, Mode1) of {ok,[],_St} = Empty -> Empty; {ok,FlatCases,St1} -> @@ -4563,15 +4611,15 @@ collect_cases({conf,Props,InitMF,CaseList,FinMF} = Conf, St) -> keep_name(Props1), FinMF}], Skipped = skip_cases_upto(Ref, Cases2Skip, Comment, - conf, Mode, skip_case), - {ok,[{skip_case,{conf,Ref,InitMF,Comment},Mode} | + conf, Mode1, skip_case), + {ok,[{skip_case,{conf,Ref,InitMF,Comment},Mode1} | Skipped],St1}; {error,_Reason} = Error -> Error end; {true,ToSkip,_} when is_list(ToSkip) -> % some cases skipped case collect_cases(CaseList, - St#cc{skip=ToSkip++Skips}) of + St#cc{skip=ToSkip++Skips}, Mode1) of {ok,[],_St} = Empty -> Empty; {ok,FlatCases,St1} -> @@ -4583,7 +4631,7 @@ collect_cases({conf,Props,InitMF,CaseList,FinMF} = Conf, St) -> Error end; false -> - case collect_cases(CaseList, St) of + case collect_cases(CaseList, St, Mode1) of {ok,[],_St} = Empty -> Empty; {ok,FlatCases,St1} -> @@ -4597,8 +4645,8 @@ collect_cases({conf,Props,InitMF,CaseList,FinMF} = Conf, St) -> end end; -collect_cases({make,InitMFA,CaseList,FinMFA}, St0) -> - case collect_cases(CaseList, St0) of +collect_cases({make,InitMFA,CaseList,FinMFA}, St0, Mode) -> + case collect_cases(CaseList, St0, Mode) of {ok,[],_St} = Empty -> Empty; {ok,FlatCases,St} -> Ref = make_ref(), @@ -4607,62 +4655,62 @@ collect_cases({make,InitMFA,CaseList,FinMFA}, St0) -> {error,_Reason} = Error -> Error end; -collect_cases({Module, Cases}, St) when is_list(Cases) -> - case (catch collect_case(Cases, St#cc{mod=Module}, [])) of +collect_cases({Module, Cases}, St, Mode) when is_list(Cases) -> + case (catch collect_case(Cases, St#cc{mod=Module}, [], Mode)) of {ok, NewCases, NewSt} -> {ok, NewCases, NewSt}; Other -> {error, Other} end; -collect_cases({_Mod,_Case}=Spec, St) -> - collect_case(Spec, St); +collect_cases({_Mod,_Case}=Spec, St, Mode) -> + collect_case(Spec, St, Mode); -collect_cases({_Mod,_Case,_Args}=Spec, St) -> - collect_case(Spec, St); -collect_cases(Case, St) when is_atom(Case), is_atom(St#cc.mod) -> - collect_case({St#cc.mod,Case}, St); -collect_cases(Other, St) -> +collect_cases({_Mod,_Case,_Args}=Spec, St, Mode) -> + collect_case(Spec, St, Mode); +collect_cases(Case, St, Mode) when is_atom(Case), is_atom(St#cc.mod) -> + collect_case({St#cc.mod,Case}, St, Mode); +collect_cases(Other, St, _Mode) -> {error,{bad_subtest_spec,St#cc.mod,Other}}. -collect_case({Mod,{conf,_,_,_,_}=Conf}, St) -> - collect_case_invoke(Mod, Conf, [], St); +collect_case({Mod,{conf,_,_,_,_}=Conf}, St, Mode) -> + collect_case_invoke(Mod, Conf, [], St, Mode); -collect_case(MFA, St) -> +collect_case(MFA, St, Mode) -> case in_skip_list(MFA, St#cc.skip) of {true,Comment} -> - {ok,[{skip_case,{MFA,Comment}}],St}; + {ok,[{skip_case,{MFA,Comment},Mode}],St}; false -> case MFA of - {Mod,Case} -> collect_case_invoke(Mod, Case, MFA, St); + {Mod,Case} -> collect_case_invoke(Mod, Case, MFA, St, Mode); {_Mod,_Case,_Args} -> {ok,[MFA],St} end end. -collect_case([], St, Acc) -> +collect_case([], St, Acc, _Mode) -> {ok, Acc, St}; -collect_case([Case | Cases], St, Acc) -> - {ok, FlatCases, NewSt} = collect_case({St#cc.mod, Case}, St), - collect_case(Cases, NewSt, Acc ++ FlatCases). +collect_case([Case | Cases], St, Acc, Mode) -> + {ok, FlatCases, NewSt} = collect_case({St#cc.mod, Case}, St, Mode), + collect_case(Cases, NewSt, Acc ++ FlatCases, Mode). -collect_case_invoke(Mod, Case, MFA, St) -> +collect_case_invoke(Mod, Case, MFA, St, Mode) -> case get_fw_mod(undefined) of undefined -> case catch apply(Mod, Case, [suite]) of {'EXIT',_} -> {ok,[MFA],St}; Suite -> - collect_subcases(Mod, Case, MFA, St, Suite) + collect_subcases(Mod, Case, MFA, St, Suite, Mode) end; _ -> Suite = test_server_sup:framework_call(get_suite, [Mod,Case], []), - collect_subcases(Mod, Case, MFA, St, Suite) + collect_subcases(Mod, Case, MFA, St, Suite, Mode) end. -collect_subcases(Mod, Case, MFA, St, Suite) -> +collect_subcases(Mod, Case, MFA, St, Suite, Mode) -> case Suite of [] when Case == all -> {ok,[],St}; [] when element(1, Case) == conf -> {ok,[],St}; @@ -4670,28 +4718,28 @@ collect_subcases(Mod, Case, MFA, St, Suite) -> %%%! --- START Kept for backwards compatibility --- %%%! Requirements are not used {req,ReqList} -> - collect_case_deny(Mod, Case, MFA, ReqList, [], St); + collect_case_deny(Mod, Case, MFA, ReqList, [], St, Mode); {req,ReqList,SubCases} -> - collect_case_deny(Mod, Case, MFA, ReqList, SubCases, St); + collect_case_deny(Mod, Case, MFA, ReqList, SubCases, St, Mode); %%%! --- END Kept for backwards compatibility --- {Skip,Reason} when Skip==skip; Skip==skipped -> - {ok,[{skip_case,{MFA,Reason}}],St}; + {ok,[{skip_case,{MFA,Reason},Mode}],St}; {error,Reason} -> throw(Reason); SubCases -> - collect_case_subcases(Mod, Case, SubCases, St) + collect_case_subcases(Mod, Case, SubCases, St, Mode) end. -collect_case_subcases(Mod, Case, SubCases, St0) -> +collect_case_subcases(Mod, Case, SubCases, St0, Mode) -> OldMod = St0#cc.mod, - case collect_cases(SubCases, St0#cc{mod=Mod}) of + case collect_cases(SubCases, St0#cc{mod=Mod}, Mode) of {ok,FlatCases,St} -> {ok,FlatCases,St#cc{mod=OldMod}}; {error,Reason} -> {error,{{Mod,Case},Reason}} end. -collect_files(Dir, Pattern, St) -> +collect_files(Dir, Pattern, St, Mode) -> {ok,Cwd} = file:get_cwd(), Dir1 = filename:join(Cwd, Dir), Wc = filename:join([Dir1,Pattern++code:objfile_extension()]), @@ -4701,7 +4749,7 @@ collect_files(Dir, Pattern, St) -> {error,{collect_fail,Dir,Pattern}}; Mods0 -> Mods = [{path_to_module(Mod),all} || Mod <- lists:sort(Mods0)], - collect_cases(Mods, St) + collect_cases(Mods, St, Mode) end. path_to_module(Path) when is_list(Path) -> @@ -4711,14 +4759,14 @@ path_to_module(Path) when is_list(Path) -> %% anyway. It should be removed or renamed! list_to_atom(filename:rootname(filename:basename(Path))). -collect_case_deny(Mod, Case, MFA, ReqList, SubCases, St) -> +collect_case_deny(Mod, Case, MFA, ReqList, SubCases, St, Mode) -> case {check_deny(ReqList, St#cc.skip),SubCases} of {{denied,Comment},_SubCases} -> - {ok,[{skip_case,{MFA,Comment}}],St}; + {ok,[{skip_case,{MFA,Comment},Mode}],St}; {granted,[]} -> {ok,[MFA],St}; {granted,SubCases} -> - collect_case_subcases(Mod, Case, SubCases, St) + collect_case_subcases(Mod, Case, SubCases, St, Mode) end. check_deny([Req|Reqs], DenyList) -> @@ -5063,15 +5111,15 @@ pinfo(P) -> %% Cover compilation %% The compilation is executed on the target node -cover_compile({App,{_File,Exclude,Include,Cross,_Export}}) -> - cover_compile1({App,Exclude,Include,Cross}); - -cover_compile({App,CoverFile}) -> - {Exclude,Include,Cross} = read_cover_file(CoverFile), - cover_compile1({App,Exclude,Include,Cross}). +start_cover(#cover{}=CoverInfo) -> + cover_compile(CoverInfo); +start_cover({log,CoverLogDir}=CoverInfo) -> + %% Cover is controlled by the framework - here's the log + put(test_server_cover_log_dir,CoverLogDir), + {ok,CoverInfo}. -cover_compile1(What) -> - test_server:cover_compile(What). +cover_compile(CoverInfo) -> + test_server:cover_compile(CoverInfo). %% Read the coverfile for an application and return a list of modules %% that are members of the application but shall not be compiled @@ -5139,25 +5187,45 @@ check_cross([]) -> %% %% This per application analysis writes the file cover.html in the %% application's run.<timestamp> directory. -cover_analyse({App,CoverInfo}, Analyse, AnalyseMods, Stop, TestDir) -> +stop_cover(#cover{}=CoverInfo, TestDir) -> + cover_analyse(CoverInfo, TestDir); +stop_cover(_CoverInfo, _TestDir) -> + %% Cover is probably controlled by the framework + ok. + +make_relative(AbsDir, VsDir) -> + DirTokens = filename:split(AbsDir), + VsTokens = filename:split(VsDir), + filename:join(make_relative1(DirTokens, VsTokens)). + +make_relative1([T | DirTs], [T | VsTs]) -> + make_relative1(DirTs, VsTs); +make_relative1(Last = [_File], []) -> + Last; +make_relative1(Last = [_File], VsTs) -> + Ups = ["../" || _ <- VsTs], + Ups ++ Last; +make_relative1(DirTs, []) -> + DirTs; +make_relative1(DirTs, VsTs) -> + Ups = ["../" || _ <- VsTs], + Ups ++ DirTs. + + +cover_analyse(CoverInfo, TestDir) -> write_default_cross_coverlog(TestDir), {ok,CoverLog} = open_html_file(filename:join(TestDir, ?coverlog_name)), write_coverlog_header(CoverLog), + #cover{app=App, + file=CoverFile, + excl=Excluded, + cross=Cross} = CoverInfo, io:fwrite(CoverLog, "<h1>Coverage for application '~w'</h1>\n", [App]), io:fwrite(CoverLog, "<p><a href=\"~ts\">Coverdata collected over all tests</a></p>", [?cross_coverlog_name]), - {CoverFile,_Included,Excluded,Cross} = - case CoverInfo of - {File,Excl,Incl,Cr,Export} -> - cover:export(Export), - {File,Incl,Excl,Cr}; - File -> - {Excl,Incl,Cr} = read_cover_file(File), - {File,Incl,Excl,Cr} - end, io:fwrite(CoverLog, "<p>CoverFile: <code>~tp</code>\n", [CoverFile]), write_cross_cover_info(TestDir,Cross), @@ -5172,7 +5240,7 @@ cover_analyse({App,CoverInfo}, Analyse, AnalyseMods, Stop, TestDir) -> io:fwrite(CoverLog, "<p>Excluded module(s): <code>~tp</code>\n", [Excluded]), - Coverage = cover_analyse(Analyse, AnalyseMods, Stop), + Coverage = test_server:cover_analyse(TestDir, CoverInfo), write_binary_file(filename:join(TestDir,?raw_coverlog_name), term_to_binary(Coverage)), @@ -5191,11 +5259,6 @@ cover_analyse({App,CoverInfo}, Analyse, AnalyseMods, Stop, TestDir) -> write_binary_file(filename:join(TestDir, ?cover_total), term_to_binary(TotPercent)). -cover_analyse(Analyse, AnalyseMods, Stop) -> - TestDir = get(test_server_log_dir_base), - test_server:cover_analyse({Analyse,TestDir}, AnalyseMods, Stop). - - %% Cover analysis - accumulated over multiple tests %% This can be executed on any node after all tests are finished. %% Analyse = overview | details @@ -5476,10 +5539,16 @@ write_html_file(File,Content) -> %% The 'major' log file, which is a pure text file is also written %% with utf8 encoding open_utf8_file(File) -> - file:open(File,[write,{encoding,utf8}]). + case file:open(File,AllOpts=[write,{encoding,utf8}]) of + {error,Reason} -> {error,{Reason,{File,AllOpts}}}; + Result -> Result + end. open_utf8_file(File,Opts) -> - file:open(File,[{encoding,utf8}|Opts]). + case file:open(File,AllOpts=[{encoding,utf8}|Opts]) of + {error,Reason} -> {error,{Reason,{File,AllOpts}}}; + Result -> Result + end. %% Write a file with specified encoding write_file(File,Content,latin1) -> diff --git a/lib/test_server/src/test_server_internal.hrl b/lib/test_server/src/test_server_internal.hrl index 4e734a330b..bafeeaabfe 100644 --- a/lib/test_server/src/test_server_internal.hrl +++ b/lib/test_server/src/test_server_internal.hrl @@ -49,3 +49,12 @@ master, cookie}). + +-record(cover, {app, % application; Name | none + file, % cover spec file + incl, % explicitly include modules + excl, % explicitly exclude modules + level, % analyse level; details | overview + mods, % actually cover compiled modules + stop=true, % stop cover after analyse; boolean() + cross}).% cross cover analyse info diff --git a/lib/test_server/src/test_server_node.erl b/lib/test_server/src/test_server_node.erl index 582abb2153..acd47788db 100644 --- a/lib/test_server/src/test_server_node.erl +++ b/lib/test_server/src/test_server_node.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2013. All Rights Reserved. +%% Copyright Ericsson AB 2002-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -653,7 +653,7 @@ find_rel_linux(Rel) -> end. find_rel_suse(Rel, SuseRel) -> - Root = "/usr/local/otp/releases/otp_beam_linux_sles", + Root = "/usr/local/otp/releases/sles", case SuseRel of "11" -> %% Try both SuSE 11, SuSE 10 and SuSe 9 in that order. @@ -673,10 +673,10 @@ find_rel_suse(Rel, SuseRel) -> find_rel_suse_1(Rel, RootWc) -> case erlang:system_info(wordsize) of 4 -> - find_rel_suse_2(Rel, RootWc++"_i386"); + find_rel_suse_2(Rel, RootWc++"_32"); 8 -> - find_rel_suse_2(Rel, RootWc++"_x64") ++ - find_rel_suse_2(Rel, RootWc++"_i386") + find_rel_suse_2(Rel, RootWc++"_64") ++ + find_rel_suse_2(Rel, RootWc++"_32") end. find_rel_suse_2(Rel, RootWc) -> diff --git a/lib/test_server/src/test_server_sup.erl b/lib/test_server/src/test_server_sup.erl index 3cfa84a52f..96e369a138 100644 --- a/lib/test_server/src/test_server_sup.erl +++ b/lib/test_server/src/test_server_sup.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2013. All Rights Reserved. +%% Copyright Ericsson AB 1998-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -469,7 +469,7 @@ check_instruction(_, {add_application, Application, Type}, _, _) -> check_instruction(_, Instr, _, _) -> throw({error, {low_level_or_invalid_instruction, Instr}}). -check_module(Module, Modules) when is_atom(Module) -> +check_module(Module, Modules) -> case {is_atom(Module), lists:member(Module, Modules)} of {true, true} -> ok; {true, false} -> throw({error, {unknown_module, Module}}); diff --git a/lib/test_server/src/ts.erl b/lib/test_server/src/ts.erl index 11d6f7af4d..d6d2e865e2 100644 --- a/lib/test_server/src/ts.erl +++ b/lib/test_server/src/ts.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2013. All Rights Reserved. +%% Copyright Ericsson AB 1997-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -24,7 +24,7 @@ -module(ts). --export([run/0, run/1, run/2, run/3, run/4, +-export([run/0, run/1, run/2, run/3, run/4, run/5, tests/0, tests/1, install/0, install/1, bench/0, bench/1, bench/2, benchmarks/0, @@ -389,6 +389,16 @@ run(Testspec, Mod, Grs={group,_Groups}, Config) when is_atom(Testspec), Args = [{suite,Mod},Grs], run_test(atom_to_list(Testspec), Args, Options). +%% run/5 +%% Run one or more test cases in a group with Options. +run(Testspec, Mod, Group, Cases, Config) when is_atom(Testspec), + is_atom(Mod), + is_list(Config) -> + Group1 = if is_tuple(Group) -> Group; true -> {group,Group} end, + Cases1 = if is_tuple(Cases) -> Cases; true -> {testcase,Cases} end, + Options=check_test_get_opts(Testspec, Config), + Args = [{suite,Mod},Group1,Cases1], + run_test(atom_to_list(Testspec), Args, Options). is_list_of_suites(List) -> lists:all(fun(Suite) -> @@ -631,16 +641,17 @@ get_last_app_tests([Dir|Dirs],RE,Acc) -> NewAcc = case re:run(Dir,RE,[{capture,all,list}]) of {match,[Dir,AppStr]} -> + Dir1 = filename:dirname(Dir), % cover logs in ct_run.<t> dir App = list_to_atom(AppStr), case lists:keytake(App,1,Acc) of {value,{App,LastDir},Rest} -> - if Dir > LastDir -> - [{App,Dir}|Rest]; + if Dir1 > LastDir -> + [{App,Dir1}|Rest]; true -> Acc end; false -> - [{App,Dir} | Acc] + [{App,Dir1} | Acc] end; _ -> Acc diff --git a/lib/test_server/src/ts.unix.config b/lib/test_server/src/ts.unix.config index a34857b9e5..1ba5d9033e 100644 --- a/lib/test_server/src/ts.unix.config +++ b/lib/test_server/src/ts.unix.config @@ -3,4 +3,4 @@ %% Always run a (VNC) X server on host %% {xserver, "xserver.example.com:66"}. -{unix,[{telnet,"belegost"},{username,"bofh"},{password,"root"},{keep_alive,true}]}. +{unix,[{telnet,"belegost"},{username,"telnet-test"},{password,"tset-tenlet"},{keep_alive,true}]}. diff --git a/lib/test_server/src/ts_run.erl b/lib/test_server/src/ts_run.erl index d96abfc55a..18d021f780 100644 --- a/lib/test_server/src/ts_run.erl +++ b/lib/test_server/src/ts_run.erl @@ -398,8 +398,9 @@ make_common_test_args(Args0, Options0, _Vars) -> end, ConfigFiles = [{config,[filename:join(ConfigPath,File) || File <- get_config_files()]}], - io_lib:format("~100000p",[Args0++Trace++Cover++Logdir++ - ConfigFiles++Options++TimeTrap]). + io_lib:format("~100000p",[[{abort_if_missing_suites,true} | + Args0++Trace++Cover++Logdir++ + ConfigFiles++Options++TimeTrap]]). to_list(X) when is_atom(X) -> atom_to_list(X); |