diff options
Diffstat (limited to 'lib/common_test')
| -rw-r--r-- | lib/common_test/doc/src/ct_hooks_chapter.xml | 24 | ||||
| -rw-r--r-- | lib/common_test/doc/src/notes.xml | 144 | ||||
| -rw-r--r-- | lib/common_test/src/ct_netconfc.erl | 8 | ||||
| -rw-r--r-- | lib/common_test/src/ct_slave.erl | 35 | ||||
| -rw-r--r-- | lib/common_test/src/cth_conn_log.erl | 2 | ||||
| -rw-r--r-- | lib/common_test/src/cth_surefire.erl | 183 | ||||
| -rw-r--r-- | lib/common_test/test/Makefile | 3 | ||||
| -rw-r--r-- | lib/common_test/test/ct_master_SUITE.erl | 57 | ||||
| -rw-r--r-- | lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl | 9 | ||||
| -rw-r--r-- | lib/common_test/test/ct_surefire_SUITE.erl | 351 | ||||
| -rw-r--r-- | lib/common_test/test/ct_surefire_SUITE_data/surefire_SUITE.erl | 92 | ||||
| -rw-r--r-- | lib/common_test/test/ct_test_support.erl | 10 | ||||
| -rw-r--r-- | lib/common_test/vsn.mk | 2 |
13 files changed, 833 insertions, 87 deletions
diff --git a/lib/common_test/doc/src/ct_hooks_chapter.xml b/lib/common_test/doc/src/ct_hooks_chapter.xml index 86237f5fc1..27d56fd47d 100644 --- a/lib/common_test/doc/src/ct_hooks_chapter.xml +++ b/lib/common_test/doc/src/ct_hooks_chapter.xml @@ -457,12 +457,26 @@ terminate(State) -> <row> <cell>cth_surefire</cell> <cell>no</cell> - <cell>Captures all test results and outputs them as surefire XML into - a file. The file which is created is by default called junit_report.xml. - The name can be by setting the path option for this hook. e.g. + <cell><p>Captures all test results and outputs them as surefire + XML into a file. The file which is created is by default + called junit_report.xml. The file name can be changed by + setting the <c>path</c> option for this hook, e.g.</p> + <code>-ct_hooks cth_surefire [{path,"/tmp/report.xml"}]</code> - Surefire XML can forinstance be used by Jenkins to display test - results.</cell> + + <p>If the <c>url_base</c> option is set, an additional + attribute named <c>url</c> will be added to each + <c>testsuite</c> and <c>testcase</c> XML element. The value will + be a constructed from the <c>url_base</c> and a relative path + to the test suite or test case log respectively, e.g.</p> + + <code>-ct_hooks cth_surefire [{url_base,"http://myserver.com/"}]</code> + <p>will give a url attribute value similar to</p> + + <code>"http://myserver.com/[email protected]_11.19.39/x86_64-unknown-linux-gnu.my_test.logs/run.2012-12-12_11.19.39/suite.log.html"</code> + + <p>Surefire XML can for instance be used by Jenkins to display test + results.</p></cell> </row> </table> diff --git a/lib/common_test/doc/src/notes.xml b/lib/common_test/doc/src/notes.xml index abe8cb2041..8c3b13951d 100644 --- a/lib/common_test/doc/src/notes.xml +++ b/lib/common_test/doc/src/notes.xml @@ -32,6 +32,150 @@ <file>notes.xml</file> </header> +<section><title>Common_Test 1.6.3.1</title> + + <section><title>Known Bugs and Problems</title> + <list> + <item> + <p> + The following corrections/changes are done in the + cth_surefire hook:</p> + <p> + <list> <item> Earlier there would always be a + 'properties' element under the 'testsuites' element. This + would exist even if there were no 'property' element + inside it. This has been changed so if there are no + 'property' elements to display, then there will not be a + 'properties' element either. </item> <item> The XML file + will now (unless other is specified) be stored in the top + log directory. Earlier, the default directory would be + the current working directory for the erlang node, which + would mostly, but not always, be the top log directory. + </item> <item> The 'hostname' attribute in the + 'testsuite' element would earlier never have the correct + value. This has been corrected. </item> <item> The + 'errors' attribute in the 'testsuite' element would + earlier display the number of failed testcases. This has + been changed and will now always have the value 0, while + the 'failures' attribute will show the number of failed + testcases. </item> <item> A new attribute 'skipped' is + added to the 'testsuite' element. This will display the + number of skipped testcases. These would earlier be + included in the number of failed test cases. </item> + <item> The total number of tests displayed by the 'tests' + attribute in the 'testsuite' element would earlier + include init/end_per_suite and init/end_per_group. This + is no longer the case. The 'tests' attribute will now + only count "real" test cases. </item> <item> Earlier, + auto skipped test cases would have no value in the 'log' + attribute. This is now corrected. </item> <item> A new + attributes 'log' is added to the 'testsuite' element. + </item> <item> A new option named 'url_base' is added for + this hook. If this option is used, a new attribute named + 'url' will be added to the 'testcase' and 'testsuite' + elements. </item> </list></p> + <p> + Own Id: OTP-10589</p> + </item> + </list> + </section> + +</section> + +<section><title>Common_Test 1.6.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The ct:run_test/1 option 'config' only worked with a + single config file, not a list of files. This has been + fixed.</p> + <p> + Own Id: OTP-10495</p> + </item> + <item> + <p> + ct_netconfc:close_session sometimes returned + {error,closed} because the ssh connection was closed + (from the server side) before the rpc-reply was received + by the client. This is normal and can not be helped. It + has been corrected so the return will be 'ok' in this + case. Other error situations will still give + {error,Reason}.</p> + <p> + Own Id: OTP-10510 Aux Id: kunagi-320 [231] </p> + </item> + <item> + <p> + ct_netconfc:close_session sometimes returned + {error,closed} or (if the connection was named) + {error,{process_down,Pid,normal}} because the ssh + connection was closed (from the server side) before the + rpc-reply was received by the client. This is normal and + can not be helped. It has been corrected so the return + will be 'ok' in this situation.</p> + <p> + Own Id: OTP-10570</p> + </item> + <item> + <p> + Fix bug where ct:require of same name with same config + would return name_in_use.</p> + <p> + Own Id: OTP-10572</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + A new test case group search functionality has been + implemented that makes Common Test search automatically + through the group definitions tree (the return value of + groups/0) and create tests for all paths of nested groups + that match the specification. It also allows for + specifying unique paths to sub groups in order to avoid + execution of unwanted tests. This new feature can be used + whenever starting a test run by means of the ct_run + program, the ct:run_test/1 API function, or a Test + Specification. Details can be found in the Test Case + Group Execution section in the Running Tests chapter.</p> + <p> + Own Id: OTP-10466 Aux Id: kunagi-276 [187] </p> + </item> + </list> + </section> + + + <section><title>Known Bugs and Problems</title> + <list> + <item> + <p> + Restore Config data if lost when test case fails.</p> + <p> + Own Id: OTP-10070 Aux Id: kunagi-175 [86] </p> + </item> + <item> + <p> + IO server error in test_server.</p> + <p> + Own Id: OTP-10125 Aux Id: OTP-10101, kunagi-177 [88] </p> + </item> + <item> + <p> + Faulty connection handling in common_test.</p> + <p> + Own Id: OTP-10126 Aux Id: kunagi-178 [89] </p> + </item> + </list> + </section> + +</section> + <section><title>Common_Test 1.6.2.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/common_test/src/ct_netconfc.erl b/lib/common_test/src/ct_netconfc.erl index 294b82bff6..1ccbc86d8f 100644 --- a/lib/common_test/src/ct_netconfc.erl +++ b/lib/common_test/src/ct_netconfc.erl @@ -307,7 +307,7 @@ -type option() :: {ssh,host()} | {port,inet:port_number()} | {user,string()} | {password,string()} | {user_dir,string()} | {timeout,timeout()}. --type host() :: inet:host_name() | inet:ip_address(). +-type host() :: inet:hostname() | inet:ip_address(). -type notification() :: {notification, xml_attributes(), notification_content()}. -type notification_content() :: [event_time()|simple_xml()]. @@ -1073,7 +1073,8 @@ handle_msg({get_event_streams=Op,Streams,Timeout}, From, State) -> SimpleXml = encode_rpc_operation(get,[Filter]), do_send_rpc(Op, SimpleXml, Timeout, From, State). -handle_msg({ssh_cm, _CM, {data, _Ch, _Type, Data}}, State) -> +handle_msg({ssh_cm, CM, {data, Ch, _Type, Data}}, State) -> + ssh_connection:adjust_window(CM,Ch,size(Data)), handle_data(Data, State); handle_msg({ssh_cm, _CM, _SshCloseMsg}, State) -> %% _SshCloseMsg can probably be one of @@ -1805,7 +1806,8 @@ get_tag([]) -> %%% SSH stuff ssh_receive_data() -> receive - {ssh_cm, _CM, {data, _Ch, _Type, Data}} -> + {ssh_cm, CM, {data, Ch, _Type, Data}} -> + ssh_connection:adjust_window(CM,Ch,size(Data)), {ok, Data}; {ssh_cm, _CM, {Closed, _Ch}} = X when Closed == closed; Closed == eof -> {error,X}; diff --git a/lib/common_test/src/ct_slave.erl b/lib/common_test/src/ct_slave.erl index cb05423497..58633b7de6 100644 --- a/lib/common_test/src/ct_slave.erl +++ b/lib/common_test/src/ct_slave.erl @@ -37,7 +37,7 @@ -record(options, {username, password, boot_timeout, init_timeout, startup_timeout, startup_functions, monitor_master, - kill_if_fail, erl_flags}). + kill_if_fail, erl_flags, env}). %%%----------------------------------------------------------------- %%% @spec start(Node) -> Result @@ -85,7 +85,8 @@ start(Host, Node) -> %%% {startup_functions, StartupFunctions} | %%% {monitor_master, Monitor} | %%% {kill_if_fail, KillIfFail} | -%%% {erl_flags, ErlangFlags} +%%% {erl_flags, ErlangFlags} | +%%% {env, [{EnvVar,Value}]} %%% Username = string() %%% Password = string() %%% BootTimeout = integer() @@ -99,6 +100,8 @@ start(Host, Node) -> %%% Monitor = bool() %%% KillIfFail = bool() %%% ErlangFlags = string() +%%% EnvVar = string() +%%% Value = string() %%% Result = {ok, NodeName} | {error, already_started, NodeName} | %%% {error, started_not_connected, NodeName} | %%% {error, boot_timeout, NodeName} | @@ -152,6 +155,9 @@ start(Host, Node) -> %%% <p>Option <code>erlang_flags</code> specifies, which flags will be added %%% to the parameters of the <code>erl</code> executable.</p> %%% +%%% <p>Option <code>env</code> specifies a list of environment variables +%%% that will extended the environment.</p> +%%% %%% <p>Special return values are: %%% <list> %%% <item><code>{error, already_started, NodeName}</code> - if the node with @@ -233,10 +239,12 @@ fetch_options(Options) -> Monitor = get_option_value(monitor_master, Options, false), KillIfFail = get_option_value(kill_if_fail, Options, true), ErlFlags = get_option_value(erl_flags, Options, []), + EnvVars = get_option_value(env, Options, []), #options{username=UserName, password=Password, boot_timeout=BootTimeout, init_timeout=InitTimeout, startup_timeout=StartupTimeout, startup_functions=StartupFunctions, - monitor_master=Monitor, kill_if_fail=KillIfFail, erl_flags=ErlFlags}. + monitor_master=Monitor, kill_if_fail=KillIfFail, + erl_flags=ErlFlags, env=EnvVars}. % send a message when slave node is started % @hidden @@ -306,6 +314,7 @@ do_start(Host, Node, Options) -> true-> spawn_remote_node(Host, Node, Options) end, + BootTimeout = Options#options.boot_timeout, InitTimeout = Options#options.init_timeout, StartupTimeout = Options#options.startup_timeout, @@ -372,9 +381,9 @@ get_cmd(Node, Flags) -> % spawn node locally spawn_local_node(Node, Options) -> - ErlFlags = Options#options.erl_flags, + #options{env=Env,erl_flags=ErlFlags} = Options, Cmd = get_cmd(Node, ErlFlags), - open_port({spawn, Cmd}, [stream]). + open_port({spawn, Cmd}, [stream,{env,Env}]). % start crypto and ssh if not yet started check_for_ssh_running() -> @@ -393,9 +402,10 @@ check_for_ssh_running() -> % spawn node remotely spawn_remote_node(Host, Node, Options) -> - Username = Options#options.username, - Password = Options#options.password, - ErlFlags = Options#options.erl_flags, + #options{username=Username, + password=Password, + erl_flags=ErlFlags, + env=Env} = Options, SSHOptions = case {Username, Password} of {[], []}-> []; @@ -407,8 +417,17 @@ spawn_remote_node(Host, Node, Options) -> check_for_ssh_running(), {ok, SSHConnRef} = ssh:connect(atom_to_list(Host), 22, SSHOptions), {ok, SSHChannelId} = ssh_connection:session_channel(SSHConnRef, infinity), + ssh_setenv(SSHConnRef, SSHChannelId, Env), ssh_connection:exec(SSHConnRef, SSHChannelId, get_cmd(Node, ErlFlags), infinity). + +ssh_setenv(SSHConnRef, SSHChannelId, [{Var, Value} | Vars]) + when is_list(Var), is_list(Value) -> + success = ssh_connection:setenv(SSHConnRef, SSHChannelId, + Var, Value, infinity), + ssh_setenv(SSHConnRef, SSHChannelId, Vars); +ssh_setenv(_SSHConnRef, _SSHChannelId, []) -> ok. + % call functions on a remote Erlang node call_functions(_Node, []) -> ok; diff --git a/lib/common_test/src/cth_conn_log.erl b/lib/common_test/src/cth_conn_log.erl index 3af89db3a5..255f3ec78a 100644 --- a/lib/common_test/src/cth_conn_log.erl +++ b/lib/common_test/src/cth_conn_log.erl @@ -58,7 +58,7 @@ -spec init(Id, HookOpts) -> Result when Id :: term(), - HookOpts :: ct:hook_options(), + HookOpts :: ct_netconfc:hook_options(), Result :: {ok,[{ct_netconfc:conn_mod(), {ct_netconfc:log_type(),[ct_netconfc:key_or_name()]}}]}. init(_Id, HookOpts) -> diff --git a/lib/common_test/src/cth_surefire.erl b/lib/common_test/src/cth_surefire.erl index 76b0f0b5ea..e6eaad8d48 100644 --- a/lib/common_test/src/cth_surefire.erl +++ b/lib/common_test/src/cth_surefire.erl @@ -1,3 +1,22 @@ +%%-------------------------------------------------------------------- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%%-------------------------------------------------------------------- + %%% @doc Common Test Framework functions handling test specifications. %%% %%% <p>This module creates a junit report of the test run if plugged in @@ -27,18 +46,28 @@ -export([terminate/1]). -record(state, { filepath, axis, properties, package, hostname, - curr_suite, curr_suite_ts, curr_group = [], curr_tc, - curr_log_dir, timer, tc_log, + curr_suite, curr_suite_ts, curr_group = [], + curr_log_dir, timer, tc_log, url_base, test_cases = [], test_suites = [] }). --record(testcase, { log, group, classname, name, time, failure, timestamp }). --record(testsuite, { errors, failures, hostname, name, tests, +-record(testcase, { log, url, group, classname, name, time, result, timestamp }). +-record(testsuite, { errors, failures, skipped, hostname, name, tests, time, timestamp, id, package, - properties, testcases }). + properties, testcases, log, url }). + +-define(default_report,"junit_report.xml"). +-define(suite_log,"suite.log.html"). + +%% Number of dirs from log root to testcase log file. +%% ct_run.<node>.<timestamp>/<test_name>/run.<timestamp>/<tc_log>.html +-define(log_depth,3). id(Opts) -> - filename:absname(proplists:get_value(path, Opts, "junit_report.xml")). + case proplists:get_value(path, Opts) of + undefined -> ?default_report; + Path -> filename:absname(Path) + end. init(Path, Opts) -> {ok, Host} = inet:gethostname(), @@ -47,10 +76,24 @@ init(Path, Opts) -> package = proplists:get_value(package,Opts), axis = proplists:get_value(axis,Opts,[]), properties = proplists:get_value(properties,Opts,[]), + url_base = proplists:get_value(url_base,Opts), timer = now() }. pre_init_per_suite(Suite,Config,#state{ test_cases = [] } = State) -> - {Config, init_tc(State#state{ curr_suite = Suite, curr_suite_ts = now() }, + TcLog = proplists:get_value(tc_logfile,Config), + CurrLogDir = filename:dirname(TcLog), + Path = + case State#state.filepath of + ?default_report -> + RootDir = get_test_root(TcLog), + filename:join(RootDir,?default_report); + P -> + P + end, + {Config, init_tc(State#state{ filepath = Path, + curr_suite = Suite, + curr_suite_ts = now(), + curr_log_dir = CurrLogDir}, Config) }; pre_init_per_suite(Suite,Config,State) -> %% Have to close the previous suite @@ -59,7 +102,8 @@ pre_init_per_suite(Suite,Config,State) -> post_init_per_suite(_Suite,Config, Result, State) -> {Result, end_tc(init_per_suite,Config,Result,State)}. -pre_end_per_suite(_Suite,Config,State) -> {Config, init_tc(State, Config)}. +pre_end_per_suite(_Suite,Config,State) -> + {Config, init_tc(State, Config)}. post_end_per_suite(_Suite,Config,Result,State) -> {Result, end_tc(end_per_suite,Config,Result,State)}. @@ -71,13 +115,15 @@ pre_init_per_group(Group,Config,State) -> post_init_per_group(_Group,Config,Result,State) -> {Result, end_tc(init_per_group,Config,Result,State)}. -pre_end_per_group(_Group,Config,State) -> {Config, init_tc(State, Config)}. +pre_end_per_group(_Group,Config,State) -> + {Config, init_tc(State, Config)}. post_end_per_group(_Group,Config,Result,State) -> NewState = end_tc(end_per_group, Config, Result, State), {Result, NewState#state{ curr_group = tl(NewState#state.curr_group)}}. -pre_init_per_testcase(_TC,Config,State) -> {Config, init_tc(State, Config)}. +pre_init_per_testcase(_TC,Config,State) -> + {Config, init_tc(State, Config)}. post_end_per_testcase(TC,Config,Result,State) -> {Result, end_tc(TC,Config, Result,State)}. @@ -88,11 +134,19 @@ on_tc_fail(_TC, Res, State) -> TCs = State#state.test_cases, TC = hd(TCs), NewTC = TC#testcase{ - failure = + result = {fail,lists:flatten(io_lib:format("~p",[Res]))} }, State#state{ test_cases = [NewTC | tl(TCs)]}. -on_tc_skip(Tc,{Type,_Reason} = Res, State) when Type == tc_auto_skip -> +on_tc_skip(Tc,{Type,_Reason} = Res, State0) when Type == tc_auto_skip -> + TcStr = atom_to_list(Tc), + State = + case State0#state.test_cases of + [#testcase{name=TcStr}|TCs] -> + State0#state{test_cases=TCs}; + _ -> + State0 + end, do_tc_skip(Res, end_tc(Tc,[],Res,init_tc(State,[]))); on_tc_skip(_Tc, _Res, State = #state{test_cases = []}) -> State; @@ -103,7 +157,7 @@ do_tc_skip(Res, State) -> TCs = State#state.test_cases, TC = hd(TCs), NewTC = TC#testcase{ - failure = + result = {skipped,lists:flatten(io_lib:format("~p",[Res]))} }, State#state{ test_cases = [NewTC | tl(TCs)]}. @@ -117,33 +171,52 @@ end_tc(Func, Config, Res, State) when is_atom(Func) -> end_tc(atom_to_list(Func), Config, Res, State); end_tc(Name, _Config, _Res, State = #state{ curr_suite = Suite, curr_group = Groups, - timer = TS, tc_log = Log } ) -> + curr_log_dir = CurrLogDir, + timer = TS, + tc_log = Log0, + url_base = UrlBase } ) -> + Log = + case Log0 of + "" -> + LowerSuiteName = string:to_lower(atom_to_list(Suite)), + filename:join(CurrLogDir,LowerSuiteName++"."++Name++".html"); + _ -> + Log0 + end, + Url = make_url(UrlBase,Log), ClassName = atom_to_list(Suite), PGroup = string:join([ atom_to_list(Group)|| Group <- lists:reverse(Groups)],"."), TimeTakes = io_lib:format("~f",[timer:now_diff(now(),TS) / 1000000]), State#state{ test_cases = [#testcase{ log = Log, + url = Url, timestamp = now_to_string(TS), classname = ClassName, group = PGroup, name = Name, time = TimeTakes, - failure = passed }| State#state.test_cases]}. + result = passed }| + State#state.test_cases], + tc_log = ""}. % so old tc_log is not set if next is on_tc_skip close_suite(#state{ test_cases = [] } = State) -> State; -close_suite(#state{ test_cases = TCs } = State) -> - Total = length(TCs), - Succ = length(lists:filter(fun(#testcase{ failure = F }) -> - F == passed - end,TCs)), - Fail = Total - Succ, +close_suite(#state{ test_cases = TCs, url_base = UrlBase } = State) -> + {Total,Fail,Skip} = count_tcs(TCs,0,0,0), TimeTaken = timer:now_diff(now(),State#state.curr_suite_ts) / 1000000, + SuiteLog = filename:join(State#state.curr_log_dir,?suite_log), + SuiteUrl = make_url(UrlBase,SuiteLog), Suite = #testsuite{ name = atom_to_list(State#state.curr_suite), package = State#state.package, + hostname = State#state.hostname, time = io_lib:format("~f",[TimeTaken]), timestamp = now_to_string(State#state.curr_suite_ts), - errors = Fail, tests = Total, - testcases = lists:reverse(TCs) }, + errors = 0, + failures = Fail, + skipped = Skip, + tests = Total, + testcases = lists:reverse(TCs), + log = SuiteLog, + url = SuiteUrl}, State#state{ test_cases = [], test_suites = [Suite | State#state.test_suites]}. @@ -159,14 +232,15 @@ terminate(State) -> -to_xml(#testcase{ group = Group, classname = CL, log = L, name = N, time = T, timestamp = TS, failure = F}) -> +to_xml(#testcase{ group = Group, classname = CL, log = L, url = U, name = N, time = T, timestamp = TS, result = R}) -> ["<testcase ", - [["group=\"",Group,"\""]||Group /= ""]," " + [["group=\"",Group,"\" "]||Group /= ""], "name=\"",N,"\" " "time=\"",T,"\" " - "timestamp=\"",TS,"\" " + "timestamp=\"",TS,"\" ", + [["url=\"",U,"\" "]||U /= undefined], "log=\"",L,"\">", - case F of + case R of passed -> []; {skipped,Reason} -> @@ -176,22 +250,29 @@ to_xml(#testcase{ group = Group, classname = CL, log = L, name = N, time = T, ti ["<failure message=\"Test ",N," in ",CL," failed!\" type=\"crash\">", sanitize(Reason),"</failure>"] end,"</testcase>"]; -to_xml(#testsuite{ package = P, hostname = H, errors = E, time = Time, - timestamp = TS, tests = T, name = N, testcases = Cases }) -> +to_xml(#testsuite{ package = P, hostname = H, errors = E, failures = F, + skipped = S, time = Time, timestamp = TS, tests = T, name = N, + testcases = Cases, log = Log, url = Url }) -> ["<testsuite ", [["package=\"",P,"\" "]||P /= undefined], - [["hostname=\"",P,"\" "]||H /= undefined], - [["name=\"",N,"\" "]||N /= undefined], - [["time=\"",Time,"\" "]||Time /= undefined], - [["timestamp=\"",TS,"\" "]||TS /= undefined], + "hostname=\"",H,"\" " + "name=\"",N,"\" " + "time=\"",Time,"\" " + "timestamp=\"",TS,"\" " "errors=\"",integer_to_list(E),"\" " - "tests=\"",integer_to_list(T),"\">", + "failures=\"",integer_to_list(F),"\" " + "skipped=\"",integer_to_list(S),"\" " + "tests=\"",integer_to_list(T),"\" ", + [["url=\"",Url,"\" "]||Url /= undefined], + "log=\"",Log,"\">", [to_xml(Case) || Case <- Cases], "</testsuite>"]; to_xml(#state{ test_suites = TestSuites, axis = Axis, properties = Props }) -> ["<testsuites>",properties_to_xml(Axis,Props), [to_xml(TestSuite) || TestSuite <- TestSuites],"</testsuites>"]. +properties_to_xml([],[]) -> + []; properties_to_xml(Axis,Props) -> ["<properties>", [["<property name=\"",Name,"\" axis=\"yes\" value=\"",Value,"\" />"] || {Name,Value} <- Axis], @@ -217,3 +298,37 @@ sanitize([]) -> now_to_string(Now) -> {{YY,MM,DD},{HH,Mi,SS}} = calendar:now_to_local_time(Now), io_lib:format("~p-~2..0B-~2..0BT~2..0B:~2..0B:~2..0B",[YY,MM,DD,HH,Mi,SS]). + +make_url(undefined,_) -> + undefined; +make_url(_,[]) -> + undefined; +make_url(UrlBase0,Log) -> + UrlBase = string:strip(UrlBase0,right,$/), + RelativeLog = get_relative_log_url(Log), + string:join([UrlBase,RelativeLog],"/"). + +get_test_root(Log) -> + LogParts = filename:split(Log), + filename:join(lists:sublist(LogParts,1,length(LogParts)-?log_depth)). + +get_relative_log_url(Log) -> + LogParts = filename:split(Log), + Start = length(LogParts)-?log_depth, + Length = ?log_depth+1, + string:join(lists:sublist(LogParts,Start,Length),"/"). + +count_tcs([#testcase{name=ConfCase}|TCs],Ok,F,S) + when ConfCase=="init_per_suite"; + ConfCase=="end_per_suite"; + ConfCase=="init_per_group"; + ConfCase=="end_per_group" -> + count_tcs(TCs,Ok,F,S); +count_tcs([#testcase{result=passed}|TCs],Ok,F,S) -> + count_tcs(TCs,Ok+1,F,S); +count_tcs([#testcase{result={fail,_}}|TCs],Ok,F,S) -> + count_tcs(TCs,Ok,F+1,S); +count_tcs([#testcase{result={skipped,_}}|TCs],Ok,F,S) -> + count_tcs(TCs,Ok,F,S+1); +count_tcs([],Ok,F,S) -> + {Ok+F+S,F,S}. diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile index df816f9a61..d469d03e04 100644 --- a/lib/common_test/test/Makefile +++ b/lib/common_test/test/Makefile @@ -56,7 +56,8 @@ MODULES= \ ct_snmp_SUITE \ ct_group_leader_SUITE \ ct_cover_SUITE \ - ct_groups_search_SUITE + ct_groups_search_SUITE \ + ct_surefire_SUITE ERL_FILES= $(MODULES:%=%.erl) diff --git a/lib/common_test/test/ct_master_SUITE.erl b/lib/common_test/test/ct_master_SUITE.erl index 27243a0067..56a343a96f 100644 --- a/lib/common_test/test/ct_master_SUITE.erl +++ b/lib/common_test/test/ct_master_SUITE.erl @@ -117,14 +117,8 @@ ct_master_test(Config) when is_list(Config) -> reformat(Events, ?eh), PrivDir, []), - find_events(NodeNames, [{tc_start,{master_SUITE,init_per_suite}}, - {tc_start,{master_SUITE,first_testcase}}, - {tc_start,{master_SUITE,second_testcase}}, - {tc_start,{master_SUITE,third_testcase}}, - {tc_start,{master_SUITE,end_per_suite}}], - Events), - - ok. + TestEvents = events_to_check(ct_master_test), + ok = find_events(NodeNames, TestEvents, Events, Config). %%%----------------------------------------------------------------- %%% HELP FUNCTIONS @@ -153,13 +147,15 @@ make_spec(DataDir, FileName, NodeNames, Suites, Config) -> CM = [{config,master,filename:join(DataDir,"master/config.txt")}], + Env = [{"THIS_MUST_BE_SET","yes"}, + {"SO_MUST_THIS","value"}], NS = lists:map( fun(NodeName) -> {init,NodeName,[ {node_start,[{startup_functions,[]}, - {monitor_master,true}]}, - {eval,{erlang,nodes,[]}} - ] + {monitor_master,true}, + {env,Env}]}, + {eval,{erlang,nodes,[]}}] } end, NodeNames), @@ -199,7 +195,6 @@ run_test(_Name, FileName, Config) -> [{FileName,ok}] = ct_test_support:run({ct_master,run,[FileName]}, [{ct_master,basic_html,[true]}], Config), - timer:sleep(5000), [{FileName,ok}] = ct_test_support:run({ct_master,run,[FileName]}, [{ct_master,basic_html,[false]}], Config). @@ -210,28 +205,26 @@ reformat(Events, EH) -> %%%----------------------------------------------------------------- %%% TEST EVENTS %%%----------------------------------------------------------------- -find_events([], _CheckEvents, _) -> - ok; -find_events([NodeName|NodeNames],CheckEvents,AllEvents) -> - find_events(NodeNames, CheckEvents, - remove_events(add_host(NodeName),CheckEvents, AllEvents, [])). - -remove_events(Node,[{Name,Data} | RestChecks], - [{?eh,#event{ name = Name, node = Node, data = Data }}|RestEvs], - Acc) -> - remove_events(Node, RestChecks, RestEvs, Acc); -remove_events(Node, Checks, [Event|RestEvs], Acc) -> - remove_events(Node, Checks, RestEvs, [Event | Acc]); -remove_events(_Node, [], [], Acc) -> - lists:reverse(Acc); -remove_events(Node, Events, [], Acc) -> - test_server:format("Could not find events: ~p in ~p for node ~p", - [Events, lists:reverse(Acc), Node]), - exit(event_not_found). + +find_events(NodeNames, TestEvents, Events, Config) -> + [begin + Node = add_host(Node0), + io:format("Searching for events for node: ~s", [Node]), + ok = ct_test_support:verify_events(TestEvents, Events, Node, Config), + io:nl() + end || Node0 <- NodeNames], + ok. add_host(NodeName) -> {ok, HostName} = inet:gethostname(), list_to_atom(atom_to_list(NodeName)++"@"++HostName). -expected_events(_) -> - []. +events_to_check(_) -> + [{?eh,tc_start,{master_SUITE,first_testcase}}, + {?eh,tc_done,{master_SUITE,first_testcase,ok}}, + {?eh,tc_start,{master_SUITE,second_testcase}}, + {?eh,tc_done,{master_SUITE,second_testcase,ok}}, + {?eh,tc_start,{master_SUITE,third_testcase}}, + {?eh,tc_done,{master_SUITE,third_testcase,ok}}, + {?eh,tc_start,{master_SUITE,env_vars}}, + {?eh,tc_done,{master_SUITE,env_vars,ok}}]. diff --git a/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl b/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl index 032d69ad9f..8a5009ad62 100644 --- a/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl +++ b/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl @@ -39,7 +39,8 @@ init_per_suite(Config) -> end_per_suite(_) -> ok. -all() -> [first_testcase, second_testcase, third_testcase]. +all() -> [first_testcase, second_testcase, third_testcase, + env_vars]. init_per_testcase(_, Config) -> Config. @@ -56,3 +57,9 @@ second_testcase(_)-> third_testcase(_)-> A = 4, A = 2*2. + +env_vars(_) -> + io:format("~p\n", [os:getenv()]), + "yes" = os:getenv("THIS_MUST_BE_SET"), + "value" = os:getenv("SO_MUST_THIS"), + ok. diff --git a/lib/common_test/test/ct_surefire_SUITE.erl b/lib/common_test/test/ct_surefire_SUITE.erl new file mode 100644 index 0000000000..69e98cef48 --- /dev/null +++ b/lib/common_test/test/ct_surefire_SUITE.erl @@ -0,0 +1,351 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_surefire_SUITE +%%% +%%% Description: +%%% Test cth_surefire hook +%%% +%%%------------------------------------------------------------------- +-module(ct_surefire_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-include_lib("xmerl/include/xmerl.hrl"). + +-define(eh, ct_test_support_eh). + +-define(url_base,"http://my.host.com/"). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config1 = ct_test_support:init_per_suite(Config), + Config1. + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [ + default, + absolute_path, + relative_path, + url, + logdir + ]. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%%%----------------------------------------------------------------- +%%% +default(Config) when is_list(Config) -> + run(default,[cth_surefire],Config), + PrivDir = ?config(priv_dir,Config), + XmlRe = filename:join([PrivDir,"*","junit_report.xml"]), + check_xml(default,XmlRe). + +absolute_path(Config) when is_list(Config) -> + PrivDir = ?config(priv_dir,Config), + Path = filename:join(PrivDir,"abspath.xml"), + run(absolute_path,[{cth_surefire,[{path,Path}]}],Config), + check_xml(absolute_path,Path). + +relative_path(Config) when is_list(Config) -> + Path = "relpath.xml", + run(relative_path,[{cth_surefire,[{path,Path}]}],Config), + PrivDir = ?config(priv_dir,Config), + XmlRe = filename:join([PrivDir,"*",Path]), + check_xml(relative_path,XmlRe). + +url(Config) when is_list(Config) -> + Path = "url.xml", + run(url,[{cth_surefire,[{url_base,?url_base}, + {path,Path}]}],Config), + PrivDir = ?config(priv_dir,Config), + XmlRe = filename:join([PrivDir,"*",Path]), + check_xml(url,XmlRe). + +logdir(Config) when is_list(Config) -> + PrivDir = ?config(priv_dir,Config), + LogDir = filename:join(PrivDir,"specific_logdir"), + file:make_dir(LogDir), + Path = "logdir.xml", + run(logdir,[{cth_surefire,[{path,Path}]}],Config,[{logdir,LogDir}]), + PrivDir = ?config(priv_dir,Config), + XmlRe = filename:join([LogDir,"*",Path]), + check_xml(logdir,XmlRe). + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- +run(Case,CTHs,Config) -> + run(Case,CTHs,Config,[]). +run(Case,CTHs,Config,ExtraOpts) -> + DataDir = ?config(data_dir, Config), + Suite = filename:join(DataDir, "surefire_SUITE"), + {Opts,ERPid} = setup([{suite,Suite},{ct_hooks,CTHs},{label,Case}|ExtraOpts], + Config), + ok = execute(Case, Opts, ERPid, Config). + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Opts1 = + case lists:keymember(logdir,1,Test) of + true -> lists:keydelete(logdir,1,Opts0); + false -> Opts0 + end, + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts1 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +execute(Name, Opts, ERPid, Config) -> + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(Name, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(Name), + ct_test_support:verify_events(TestEvents, Events, Config). + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +%%%----------------------------------------------------------------- +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). + +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + test_events(Test) ++ events_to_check(Test, N-1). + +test_events(_) -> + [{?eh,start_logging,'_'}, + {?eh,start_info,{1,1,9}}, + {?eh,tc_start,{surefire_SUITE,init_per_suite}}, + {?eh,tc_done,{surefire_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{surefire_SUITE,tc_ok}}, + {?eh,tc_done,{surefire_SUITE,tc_ok,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{surefire_SUITE,tc_fail}}, + {?eh,tc_done,{surefire_SUITE,tc_fail, + {failed,{error,{test_case_failed,"this test should fail"}}}}}, + {?eh,test_stats,{1,1,{0,0}}}, + {?eh,tc_start,{surefire_SUITE,tc_skip}}, + {?eh,tc_done,{surefire_SUITE,tc_skip,{skipped,"this test is skipped"}}}, + {?eh,test_stats,{1,1,{1,0}}}, + {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, + {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, + {skipped,{require_failed,'_'}}}}, + {?eh,test_stats,{1,1,{1,1}}}, + [{?eh,tc_start,{surefire_SUITE,{init_per_group,g,[]}}}, + {?eh,tc_done,{surefire_SUITE,{init_per_group,g,[]},ok}}, + {?eh,tc_start,{surefire_SUITE,tc_ok}}, + {?eh,tc_done,{surefire_SUITE,tc_ok,ok}}, + {?eh,test_stats,{2,1,{1,1}}}, + {?eh,tc_start,{surefire_SUITE,tc_fail}}, + {?eh,tc_done,{surefire_SUITE,tc_fail, + {failed,{error,{test_case_failed,"this test should fail"}}}}}, + {?eh,test_stats,{2,2,{1,1}}}, + {?eh,tc_start,{surefire_SUITE,tc_skip}}, + {?eh,tc_done,{surefire_SUITE,tc_skip,{skipped,"this test is skipped"}}}, + {?eh,test_stats,{2,2,{2,1}}}, + {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, + {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, + {skipped,{require_failed,'_'}}}}, + {?eh,test_stats,{2,2,{2,2}}}, + {?eh,tc_start,{surefire_SUITE,{end_per_group,g,[]}}}, + {?eh,tc_done,{surefire_SUITE,{end_per_group,g,[]},ok}}], + [{?eh,tc_start,{surefire_SUITE,{init_per_group,g_fail,[]}}}, + {?eh,tc_done,{surefire_SUITE,{init_per_group,g_fail,[]}, + {failed,{error,all_cases_should_be_skipped}}}}, + {?eh,tc_auto_skip,{surefire_SUITE,tc_ok, + {failed, + {surefire_SUITE,init_per_group, + {'EXIT',all_cases_should_be_skipped}}}}}, + {?eh,test_stats,{2,2,{2,3}}}, + {?eh,tc_auto_skip,{surefire_SUITE,end_per_group, + {failed, + {surefire_SUITE,init_per_group, + {'EXIT',all_cases_should_be_skipped}}}}}], + {?eh,tc_start,{surefire_SUITE,end_per_suite}}, + {?eh,tc_done,{surefire_SUITE,end_per_suite,ok}}, + {?eh,stop_logging,[]}]. + + +%%%----------------------------------------------------------------- +%%% Check generated xml log files +check_xml(Case,XmlRe) -> + case filelib:wildcard(XmlRe) of + [] -> + ct:fail("No xml files found with regexp ~p~n", [XmlRe]); + [_] = Xmls when Case==absolute_path -> + do_check_xml(Case,Xmls); + [_,_] = Xmls -> + do_check_xml(Case,Xmls) + end. + +%% Allowed structure: +%% <testsuites> +%% <testsuite> +%% <properties> +%% <property/> +%% ... +%% </properties> +%% <testcase> +%% [<failure/> | <error/> | <skipped/> ] +%% </testcase> +%% ... +%% </testsuite> +%% ... +%% </testsuites> +do_check_xml(Case,[Xml|Xmls]) -> + ct:log("Checking <a href=~p>~s</a>~n",[Xml,Xml]), + {E,_} = xmerl_scan:file(Xml), + Expected = events_to_result(lists:flatten(test_events(Case))), + ParseResult = testsuites(Case,E), + ct:log("Expecting: ~p~n",[[Expected]]), + ct:log("Actual : ~p~n",[ParseResult]), + [Expected] = ParseResult, + do_check_xml(Case,Xmls); +do_check_xml(_,[]) -> + ok. + +%% Scanning the XML to get the same type of result as events_to_result/1 +testsuites(Case,#xmlElement{name=testsuites,content=TS}) -> + %% OTP-10589 - move properties element to <testsuite> + false = lists:keytake(properties,#xmlElement.name,TS), + testsuite(Case,TS). + +testsuite(Case,[#xmlElement{name=testsuite,content=TC,attributes=A}|TS]) -> + {ET,EF,ES} = events_to_numbers(lists:flatten(test_events(Case))), + {T,E,F,S} = get_numbers_from_attrs(A,false,false,false,false), + ct:log("Expecting total:~p, error:~p, failure:~p, skipped:~p~n",[ET,0,EF,ES]), + ct:log("Actual total:~p, error:~p, failure:~p, skipped:~p~n",[T,E,F,S]), + {ET,0,EF,ES} = {T,E,F,S}, + + %% properties should only be there if given a options to hook + false = lists:keytake(properties,#xmlElement.name,TC), + %% system-out and system-err is not used by common_test + false = lists:keytake('system-out',#xmlElement.name,TC), + false = lists:keytake('system-err',#xmlElement.name,TC), + R=testcase(Case,TC), + [R|testsuite(Case,TS)]; +testsuite(_Case,[]) -> + []. + +testcase(url=Case,[#xmlElement{name=testcase,attributes=A,content=C}|TC]) -> + R = failed_or_skipped(C), + case R of + [s] -> + case lists:keyfind(url,#xmlAttribute.name,A) of + false -> ok; + #xmlAttribute{value=UrlAttr} -> + lists:keyfind(url,#xmlAttribute.name,A), + true = lists:prefix(?url_base,UrlAttr) + end; + _ -> + #xmlAttribute{value=UrlAttr} = + lists:keyfind(url,#xmlAttribute.name,A), + true = lists:prefix(?url_base,UrlAttr) + end, + [R|testcase(Case,TC)]; +testcase(Case,[#xmlElement{name=testcase,attributes=A,content=C}|TC]) -> + false = lists:keyfind(url,#xmlAttribute.name,A), + R = failed_or_skipped(C), + [R|testcase(Case,TC)]; +testcase(_Case,[]) -> + []. + +failed_or_skipped([#xmlElement{name=failure}|E]) -> + [f|failed_or_skipped(E)]; +failed_or_skipped([#xmlElement{name=error}|E]) -> + [e|failed_or_skipped(E)]; +failed_or_skipped([#xmlElement{name=skipped}|E]) -> + [s|failed_or_skipped(E)]; +failed_or_skipped([]) -> + []. + +%% Using the expected events to produce the expected result of the XML scanning. +%% The result is a list of test suites: +%% Testsuites = [Testsuite] +%% Testsuite = [Testcase] +%% Testcase = [] | [f] | [s], indicating ok, failed and skipped respectively +events_to_result([{?eh,tc_done,{_Suite,_Case,R}}|E]) -> + [result(R)|events_to_result(E)]; +events_to_result([{?eh,tc_auto_skip,_}|E]) -> + [[s]|events_to_result(E)]; +events_to_result([_|E]) -> + events_to_result(E); +events_to_result([]) -> + []. + +result(ok) ->[]; +result({skipped,_}) -> [s]; +result({failed,_}) -> [f]. + +%% Using the expected events' last test_stats element to produce the +%% expected number of totla, errors, failed and skipped testcases. +events_to_numbers(E) -> + RevE = lists:reverse(E), + {?eh,test_stats,{Ok,F,{US,AS}}} = lists:keyfind(test_stats,2,RevE), + {Ok+F+US+AS,F,US+AS}. + +get_numbers_from_attrs([#xmlAttribute{name=tests,value=X}|A],false,E,F,S) -> + get_numbers_from_attrs(A,list_to_integer(X),E,F,S); +get_numbers_from_attrs([#xmlAttribute{name=errors,value=X}|A],T,false,F,S) -> + get_numbers_from_attrs(A,T,list_to_integer(X),F,S); +get_numbers_from_attrs([#xmlAttribute{name=failures,value=X}|A],T,E,false,S) -> + get_numbers_from_attrs(A,T,E,list_to_integer(X),S); +get_numbers_from_attrs([#xmlAttribute{name=skipped,value=X}|A],T,E,F,false) -> + get_numbers_from_attrs(A,T,E,F,list_to_integer(X)); +get_numbers_from_attrs([_|A],T,E,F,S) -> + get_numbers_from_attrs(A,T,E,F,S); +get_numbers_from_attrs([],T,E,F,S) -> + {T,E,F,S}. diff --git a/lib/common_test/test/ct_surefire_SUITE_data/surefire_SUITE.erl b/lib/common_test/test/ct_surefire_SUITE_data/surefire_SUITE.erl new file mode 100644 index 0000000000..677aee46c5 --- /dev/null +++ b/lib/common_test/test/ct_surefire_SUITE_data/surefire_SUITE.erl @@ -0,0 +1,92 @@ +%%-------------------------------------------------------------------- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%%---------------------------------------------------------------------- +%% File: surefire_SUITE.erl +%% +%% Description: +%% This file contains the test cases for cth_surefire. +%% +%% @author Support +%% @doc Test of surefire support in common_test +%% @end +%%---------------------------------------------------------------------- +%%---------------------------------------------------------------------- +-module(surefire_SUITE). +-include_lib("common_test/include/ct.hrl"). + +-compile(export_all). + +%% Default timetrap timeout (set in init_per_testcase). +-define(default_timeout, ?t:minutes(1)). + +all() -> + testcases() ++ [{group,g},{group,g_fail}]. + +groups() -> + [{g,testcases()}, + {g_fail,[tc_ok]}]. + +testcases() -> + [tc_ok, + tc_fail, + tc_skip, + tc_autoskip_require]. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + Config. + +init_per_group(g_fail, _Config) -> + exit(all_cases_should_be_skipped); +init_per_group(_, Config) -> + Config. + +end_per_group(_Group, Config) -> + Config. + +init_per_testcase(_Case, Config) -> + Dog = test_server:timetrap(?default_timeout), + [{watchdog, Dog}|Config]. + +end_per_testcase(_Case, Config) -> + Dog=?config(watchdog, Config), + test_server:timetrap_cancel(Dog), + ok. + +%%%----------------------------------------------------------------- +%%% Test cases +break(_Config) -> + test_server:break(""), + ok. + +tc_ok(_Config) -> + ok. + +tc_fail(_Config) -> + ct:fail("this test should fail"). + +tc_skip(_Config) -> + {skip,"this test is skipped"}. + +tc_autoskip_require() -> + [{require,whatever}]. +tc_autoskip_require(Config) -> + ct:fail("this test should never be executed - it should be autoskipped"). diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl index 901e53c71d..e5e2e68fcb 100644 --- a/lib/common_test/test/ct_test_support.erl +++ b/lib/common_test/test/ct_test_support.erl @@ -32,7 +32,7 @@ run/2, run/3, run/4, get_opts/1, wait_for_ct_stop/1]). -export([handle_event/2, start_event_receiver/1, get_events/2, - verify_events/3, reformat/2, log_events/4, + verify_events/3, verify_events/4, reformat/2, log_events/4, join_abs_dirs/2]). -export([ct_test_halt/1]). @@ -370,6 +370,14 @@ verify_events(TEvs, Evs, Config) -> ok end. +verify_events(TEvs, Evs, Node, Config) -> + case catch verify_events1(TEvs, Evs, Node, Config) of + {'EXIT',Reason} -> + Reason; + _ -> + ok + end. + verify_events1([TestEv|_], [{TEH,#event{name=stop_logging,node=Node,data=_}}|_], Node, _) when element(1,TestEv) == TEH, element(2,TestEv) =/= stop_logging -> test_server:format("Failed to find ~p in the list of events!~n", [TestEv]), diff --git a/lib/common_test/vsn.mk b/lib/common_test/vsn.mk index 5c9fdfc47e..f9bb22867e 100644 --- a/lib/common_test/vsn.mk +++ b/lib/common_test/vsn.mk @@ -1 +1 @@ -COMMON_TEST_VSN = 1.6.2.1 +COMMON_TEST_VSN = 1.6.3 |