From f309f03b406b9bc30abcc4daf3154e6475363c96 Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Tue, 13 Nov 2012 14:12:41 +0100 Subject: [common_test] Fix cth_surefire The following corrections/changes are done in the cth_surefire hook: * Earlier there would always be a 'properties' element under the 'testsuites' element. This would exist even if there were now 'property' element inside it. This has been changed so if there are no 'property' elements to display, then there will not be a 'properties' element either. * The XML file will now (unless other is specified) be stored in the top log directory. Earlier, the default directory would be the current working directory for the erlang node, which would mostly, but not always, be the top log directory. * The 'hostname' attribute in the 'testsuite' element would earlier never have the correct value. This has been corrected. * The 'errors' attribute in the 'testsuite' element would earlier display the number of failed testcases. This has been changed and will now always have the value 0, while the 'failures' attribute will show the number of failed testcases. * A new attribute 'skipped' is added to the 'testsuite' element. This will display the number of skipped testcases. These would earlier be included in the number of failed test cases. * The total number of tests displayed by the 'tests' attribute in the 'testsuite' element would earlier include init/end_per_suite and init/end_per_group. This is no longer the case. The 'tests' attribute will now only count "real" test cases. * Earlier, auto skipped test cases would have no value in the 'log' attribute. This is now corrected. * A new attributes 'log' is added to the 'testsuite' element. * A new option named 'url_base' is added for this hook. If this option is used, a new attribute named 'url' will be added to the 'testcase' and 'testsuite' elements. Tests are added for the ct_surefire hook. --- lib/common_test/src/cth_surefire.erl | 183 +++++++++-- lib/common_test/test/Makefile | 3 +- lib/common_test/test/ct_surefire_SUITE.erl | 351 +++++++++++++++++++++ .../test/ct_surefire_SUITE_data/surefire_SUITE.erl | 92 ++++++ 4 files changed, 594 insertions(+), 35 deletions(-) create mode 100644 lib/common_test/test/ct_surefire_SUITE.erl create mode 100644 lib/common_test/test/ct_surefire_SUITE_data/surefire_SUITE.erl diff --git a/lib/common_test/src/cth_surefire.erl b/lib/common_test/src/cth_surefire.erl index 76b0f0b5ea..e6eaad8d48 100644 --- a/lib/common_test/src/cth_surefire.erl +++ b/lib/common_test/src/cth_surefire.erl @@ -1,3 +1,22 @@ +%%-------------------------------------------------------------------- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%%-------------------------------------------------------------------- + %%% @doc Common Test Framework functions handling test specifications. %%% %%%

This module creates a junit report of the test run if plugged in @@ -27,18 +46,28 @@ -export([terminate/1]). -record(state, { filepath, axis, properties, package, hostname, - curr_suite, curr_suite_ts, curr_group = [], curr_tc, - curr_log_dir, timer, tc_log, + curr_suite, curr_suite_ts, curr_group = [], + curr_log_dir, timer, tc_log, url_base, test_cases = [], test_suites = [] }). --record(testcase, { log, group, classname, name, time, failure, timestamp }). --record(testsuite, { errors, failures, hostname, name, tests, +-record(testcase, { log, url, group, classname, name, time, result, timestamp }). +-record(testsuite, { errors, failures, skipped, hostname, name, tests, time, timestamp, id, package, - properties, testcases }). + properties, testcases, log, url }). + +-define(default_report,"junit_report.xml"). +-define(suite_log,"suite.log.html"). + +%% Number of dirs from log root to testcase log file. +%% ct_run..//run./.html +-define(log_depth,3). id(Opts) -> - filename:absname(proplists:get_value(path, Opts, "junit_report.xml")). + case proplists:get_value(path, Opts) of + undefined -> ?default_report; + Path -> filename:absname(Path) + end. init(Path, Opts) -> {ok, Host} = inet:gethostname(), @@ -47,10 +76,24 @@ init(Path, Opts) -> package = proplists:get_value(package,Opts), axis = proplists:get_value(axis,Opts,[]), properties = proplists:get_value(properties,Opts,[]), + url_base = proplists:get_value(url_base,Opts), timer = now() }. pre_init_per_suite(Suite,Config,#state{ test_cases = [] } = State) -> - {Config, init_tc(State#state{ curr_suite = Suite, curr_suite_ts = now() }, + TcLog = proplists:get_value(tc_logfile,Config), + CurrLogDir = filename:dirname(TcLog), + Path = + case State#state.filepath of + ?default_report -> + RootDir = get_test_root(TcLog), + filename:join(RootDir,?default_report); + P -> + P + end, + {Config, init_tc(State#state{ filepath = Path, + curr_suite = Suite, + curr_suite_ts = now(), + curr_log_dir = CurrLogDir}, Config) }; pre_init_per_suite(Suite,Config,State) -> %% Have to close the previous suite @@ -59,7 +102,8 @@ pre_init_per_suite(Suite,Config,State) -> post_init_per_suite(_Suite,Config, Result, State) -> {Result, end_tc(init_per_suite,Config,Result,State)}. -pre_end_per_suite(_Suite,Config,State) -> {Config, init_tc(State, Config)}. +pre_end_per_suite(_Suite,Config,State) -> + {Config, init_tc(State, Config)}. post_end_per_suite(_Suite,Config,Result,State) -> {Result, end_tc(end_per_suite,Config,Result,State)}. @@ -71,13 +115,15 @@ pre_init_per_group(Group,Config,State) -> post_init_per_group(_Group,Config,Result,State) -> {Result, end_tc(init_per_group,Config,Result,State)}. -pre_end_per_group(_Group,Config,State) -> {Config, init_tc(State, Config)}. +pre_end_per_group(_Group,Config,State) -> + {Config, init_tc(State, Config)}. post_end_per_group(_Group,Config,Result,State) -> NewState = end_tc(end_per_group, Config, Result, State), {Result, NewState#state{ curr_group = tl(NewState#state.curr_group)}}. -pre_init_per_testcase(_TC,Config,State) -> {Config, init_tc(State, Config)}. +pre_init_per_testcase(_TC,Config,State) -> + {Config, init_tc(State, Config)}. post_end_per_testcase(TC,Config,Result,State) -> {Result, end_tc(TC,Config, Result,State)}. @@ -88,11 +134,19 @@ on_tc_fail(_TC, Res, State) -> TCs = State#state.test_cases, TC = hd(TCs), NewTC = TC#testcase{ - failure = + result = {fail,lists:flatten(io_lib:format("~p",[Res]))} }, State#state{ test_cases = [NewTC | tl(TCs)]}. -on_tc_skip(Tc,{Type,_Reason} = Res, State) when Type == tc_auto_skip -> +on_tc_skip(Tc,{Type,_Reason} = Res, State0) when Type == tc_auto_skip -> + TcStr = atom_to_list(Tc), + State = + case State0#state.test_cases of + [#testcase{name=TcStr}|TCs] -> + State0#state{test_cases=TCs}; + _ -> + State0 + end, do_tc_skip(Res, end_tc(Tc,[],Res,init_tc(State,[]))); on_tc_skip(_Tc, _Res, State = #state{test_cases = []}) -> State; @@ -103,7 +157,7 @@ do_tc_skip(Res, State) -> TCs = State#state.test_cases, TC = hd(TCs), NewTC = TC#testcase{ - failure = + result = {skipped,lists:flatten(io_lib:format("~p",[Res]))} }, State#state{ test_cases = [NewTC | tl(TCs)]}. @@ -117,33 +171,52 @@ end_tc(Func, Config, Res, State) when is_atom(Func) -> end_tc(atom_to_list(Func), Config, Res, State); end_tc(Name, _Config, _Res, State = #state{ curr_suite = Suite, curr_group = Groups, - timer = TS, tc_log = Log } ) -> + curr_log_dir = CurrLogDir, + timer = TS, + tc_log = Log0, + url_base = UrlBase } ) -> + Log = + case Log0 of + "" -> + LowerSuiteName = string:to_lower(atom_to_list(Suite)), + filename:join(CurrLogDir,LowerSuiteName++"."++Name++".html"); + _ -> + Log0 + end, + Url = make_url(UrlBase,Log), ClassName = atom_to_list(Suite), PGroup = string:join([ atom_to_list(Group)|| Group <- lists:reverse(Groups)],"."), TimeTakes = io_lib:format("~f",[timer:now_diff(now(),TS) / 1000000]), State#state{ test_cases = [#testcase{ log = Log, + url = Url, timestamp = now_to_string(TS), classname = ClassName, group = PGroup, name = Name, time = TimeTakes, - failure = passed }| State#state.test_cases]}. + result = passed }| + State#state.test_cases], + tc_log = ""}. % so old tc_log is not set if next is on_tc_skip close_suite(#state{ test_cases = [] } = State) -> State; -close_suite(#state{ test_cases = TCs } = State) -> - Total = length(TCs), - Succ = length(lists:filter(fun(#testcase{ failure = F }) -> - F == passed - end,TCs)), - Fail = Total - Succ, +close_suite(#state{ test_cases = TCs, url_base = UrlBase } = State) -> + {Total,Fail,Skip} = count_tcs(TCs,0,0,0), TimeTaken = timer:now_diff(now(),State#state.curr_suite_ts) / 1000000, + SuiteLog = filename:join(State#state.curr_log_dir,?suite_log), + SuiteUrl = make_url(UrlBase,SuiteLog), Suite = #testsuite{ name = atom_to_list(State#state.curr_suite), package = State#state.package, + hostname = State#state.hostname, time = io_lib:format("~f",[TimeTaken]), timestamp = now_to_string(State#state.curr_suite_ts), - errors = Fail, tests = Total, - testcases = lists:reverse(TCs) }, + errors = 0, + failures = Fail, + skipped = Skip, + tests = Total, + testcases = lists:reverse(TCs), + log = SuiteLog, + url = SuiteUrl}, State#state{ test_cases = [], test_suites = [Suite | State#state.test_suites]}. @@ -159,14 +232,15 @@ terminate(State) -> -to_xml(#testcase{ group = Group, classname = CL, log = L, name = N, time = T, timestamp = TS, failure = F}) -> +to_xml(#testcase{ group = Group, classname = CL, log = L, url = U, name = N, time = T, timestamp = TS, result = R}) -> ["", - case F of + case R of passed -> []; {skipped,Reason} -> @@ -176,22 +250,29 @@ to_xml(#testcase{ group = Group, classname = CL, log = L, name = N, time = T, ti ["", sanitize(Reason),""] end,""]; -to_xml(#testsuite{ package = P, hostname = H, errors = E, time = Time, - timestamp = TS, tests = T, name = N, testcases = Cases }) -> +to_xml(#testsuite{ package = P, hostname = H, errors = E, failures = F, + skipped = S, time = Time, timestamp = TS, tests = T, name = N, + testcases = Cases, log = Log, url = Url }) -> ["", + "failures=\"",integer_to_list(F),"\" " + "skipped=\"",integer_to_list(S),"\" " + "tests=\"",integer_to_list(T),"\" ", + [["url=\"",Url,"\" "]||Url /= undefined], + "log=\"",Log,"\">", [to_xml(Case) || Case <- Cases], ""]; to_xml(#state{ test_suites = TestSuites, axis = Axis, properties = Props }) -> ["",properties_to_xml(Axis,Props), [to_xml(TestSuite) || TestSuite <- TestSuites],""]. +properties_to_xml([],[]) -> + []; properties_to_xml(Axis,Props) -> ["", [[""] || {Name,Value} <- Axis], @@ -217,3 +298,37 @@ sanitize([]) -> now_to_string(Now) -> {{YY,MM,DD},{HH,Mi,SS}} = calendar:now_to_local_time(Now), io_lib:format("~p-~2..0B-~2..0BT~2..0B:~2..0B:~2..0B",[YY,MM,DD,HH,Mi,SS]). + +make_url(undefined,_) -> + undefined; +make_url(_,[]) -> + undefined; +make_url(UrlBase0,Log) -> + UrlBase = string:strip(UrlBase0,right,$/), + RelativeLog = get_relative_log_url(Log), + string:join([UrlBase,RelativeLog],"/"). + +get_test_root(Log) -> + LogParts = filename:split(Log), + filename:join(lists:sublist(LogParts,1,length(LogParts)-?log_depth)). + +get_relative_log_url(Log) -> + LogParts = filename:split(Log), + Start = length(LogParts)-?log_depth, + Length = ?log_depth+1, + string:join(lists:sublist(LogParts,Start,Length),"/"). + +count_tcs([#testcase{name=ConfCase}|TCs],Ok,F,S) + when ConfCase=="init_per_suite"; + ConfCase=="end_per_suite"; + ConfCase=="init_per_group"; + ConfCase=="end_per_group" -> + count_tcs(TCs,Ok,F,S); +count_tcs([#testcase{result=passed}|TCs],Ok,F,S) -> + count_tcs(TCs,Ok+1,F,S); +count_tcs([#testcase{result={fail,_}}|TCs],Ok,F,S) -> + count_tcs(TCs,Ok,F+1,S); +count_tcs([#testcase{result={skipped,_}}|TCs],Ok,F,S) -> + count_tcs(TCs,Ok,F,S+1); +count_tcs([],Ok,F,S) -> + {Ok+F+S,F,S}. diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile index 374fd8a824..8cdc4ebed7 100644 --- a/lib/common_test/test/Makefile +++ b/lib/common_test/test/Makefile @@ -52,7 +52,8 @@ MODULES= \ ct_auto_compile_SUITE \ ct_verbosity_SUITE \ ct_shell_SUITE \ - ct_groups_search_SUITE + ct_groups_search_SUITE \ + ct_surefire_SUITE ERL_FILES= $(MODULES:%=%.erl) diff --git a/lib/common_test/test/ct_surefire_SUITE.erl b/lib/common_test/test/ct_surefire_SUITE.erl new file mode 100644 index 0000000000..69e98cef48 --- /dev/null +++ b/lib/common_test/test/ct_surefire_SUITE.erl @@ -0,0 +1,351 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_surefire_SUITE +%%% +%%% Description: +%%% Test cth_surefire hook +%%% +%%%------------------------------------------------------------------- +-module(ct_surefire_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-include_lib("xmerl/include/xmerl.hrl"). + +-define(eh, ct_test_support_eh). + +-define(url_base,"http://my.host.com/"). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config1 = ct_test_support:init_per_suite(Config), + Config1. + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [ + default, + absolute_path, + relative_path, + url, + logdir + ]. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%%%----------------------------------------------------------------- +%%% +default(Config) when is_list(Config) -> + run(default,[cth_surefire],Config), + PrivDir = ?config(priv_dir,Config), + XmlRe = filename:join([PrivDir,"*","junit_report.xml"]), + check_xml(default,XmlRe). + +absolute_path(Config) when is_list(Config) -> + PrivDir = ?config(priv_dir,Config), + Path = filename:join(PrivDir,"abspath.xml"), + run(absolute_path,[{cth_surefire,[{path,Path}]}],Config), + check_xml(absolute_path,Path). + +relative_path(Config) when is_list(Config) -> + Path = "relpath.xml", + run(relative_path,[{cth_surefire,[{path,Path}]}],Config), + PrivDir = ?config(priv_dir,Config), + XmlRe = filename:join([PrivDir,"*",Path]), + check_xml(relative_path,XmlRe). + +url(Config) when is_list(Config) -> + Path = "url.xml", + run(url,[{cth_surefire,[{url_base,?url_base}, + {path,Path}]}],Config), + PrivDir = ?config(priv_dir,Config), + XmlRe = filename:join([PrivDir,"*",Path]), + check_xml(url,XmlRe). + +logdir(Config) when is_list(Config) -> + PrivDir = ?config(priv_dir,Config), + LogDir = filename:join(PrivDir,"specific_logdir"), + file:make_dir(LogDir), + Path = "logdir.xml", + run(logdir,[{cth_surefire,[{path,Path}]}],Config,[{logdir,LogDir}]), + PrivDir = ?config(priv_dir,Config), + XmlRe = filename:join([LogDir,"*",Path]), + check_xml(logdir,XmlRe). + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- +run(Case,CTHs,Config) -> + run(Case,CTHs,Config,[]). +run(Case,CTHs,Config,ExtraOpts) -> + DataDir = ?config(data_dir, Config), + Suite = filename:join(DataDir, "surefire_SUITE"), + {Opts,ERPid} = setup([{suite,Suite},{ct_hooks,CTHs},{label,Case}|ExtraOpts], + Config), + ok = execute(Case, Opts, ERPid, Config). + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Opts1 = + case lists:keymember(logdir,1,Test) of + true -> lists:keydelete(logdir,1,Opts0); + false -> Opts0 + end, + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts1 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +execute(Name, Opts, ERPid, Config) -> + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(Name, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(Name), + ct_test_support:verify_events(TestEvents, Events, Config). + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +%%%----------------------------------------------------------------- +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). + +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + test_events(Test) ++ events_to_check(Test, N-1). + +test_events(_) -> + [{?eh,start_logging,'_'}, + {?eh,start_info,{1,1,9}}, + {?eh,tc_start,{surefire_SUITE,init_per_suite}}, + {?eh,tc_done,{surefire_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{surefire_SUITE,tc_ok}}, + {?eh,tc_done,{surefire_SUITE,tc_ok,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{surefire_SUITE,tc_fail}}, + {?eh,tc_done,{surefire_SUITE,tc_fail, + {failed,{error,{test_case_failed,"this test should fail"}}}}}, + {?eh,test_stats,{1,1,{0,0}}}, + {?eh,tc_start,{surefire_SUITE,tc_skip}}, + {?eh,tc_done,{surefire_SUITE,tc_skip,{skipped,"this test is skipped"}}}, + {?eh,test_stats,{1,1,{1,0}}}, + {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, + {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, + {skipped,{require_failed,'_'}}}}, + {?eh,test_stats,{1,1,{1,1}}}, + [{?eh,tc_start,{surefire_SUITE,{init_per_group,g,[]}}}, + {?eh,tc_done,{surefire_SUITE,{init_per_group,g,[]},ok}}, + {?eh,tc_start,{surefire_SUITE,tc_ok}}, + {?eh,tc_done,{surefire_SUITE,tc_ok,ok}}, + {?eh,test_stats,{2,1,{1,1}}}, + {?eh,tc_start,{surefire_SUITE,tc_fail}}, + {?eh,tc_done,{surefire_SUITE,tc_fail, + {failed,{error,{test_case_failed,"this test should fail"}}}}}, + {?eh,test_stats,{2,2,{1,1}}}, + {?eh,tc_start,{surefire_SUITE,tc_skip}}, + {?eh,tc_done,{surefire_SUITE,tc_skip,{skipped,"this test is skipped"}}}, + {?eh,test_stats,{2,2,{2,1}}}, + {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, + {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, + {skipped,{require_failed,'_'}}}}, + {?eh,test_stats,{2,2,{2,2}}}, + {?eh,tc_start,{surefire_SUITE,{end_per_group,g,[]}}}, + {?eh,tc_done,{surefire_SUITE,{end_per_group,g,[]},ok}}], + [{?eh,tc_start,{surefire_SUITE,{init_per_group,g_fail,[]}}}, + {?eh,tc_done,{surefire_SUITE,{init_per_group,g_fail,[]}, + {failed,{error,all_cases_should_be_skipped}}}}, + {?eh,tc_auto_skip,{surefire_SUITE,tc_ok, + {failed, + {surefire_SUITE,init_per_group, + {'EXIT',all_cases_should_be_skipped}}}}}, + {?eh,test_stats,{2,2,{2,3}}}, + {?eh,tc_auto_skip,{surefire_SUITE,end_per_group, + {failed, + {surefire_SUITE,init_per_group, + {'EXIT',all_cases_should_be_skipped}}}}}], + {?eh,tc_start,{surefire_SUITE,end_per_suite}}, + {?eh,tc_done,{surefire_SUITE,end_per_suite,ok}}, + {?eh,stop_logging,[]}]. + + +%%%----------------------------------------------------------------- +%%% Check generated xml log files +check_xml(Case,XmlRe) -> + case filelib:wildcard(XmlRe) of + [] -> + ct:fail("No xml files found with regexp ~p~n", [XmlRe]); + [_] = Xmls when Case==absolute_path -> + do_check_xml(Case,Xmls); + [_,_] = Xmls -> + do_check_xml(Case,Xmls) + end. + +%% Allowed structure: +%% +%% +%% +%% +%% ... +%% +%% +%% [ | | ] +%% +%% ... +%% +%% ... +%% +do_check_xml(Case,[Xml|Xmls]) -> + ct:log("Checking ~s~n",[Xml,Xml]), + {E,_} = xmerl_scan:file(Xml), + Expected = events_to_result(lists:flatten(test_events(Case))), + ParseResult = testsuites(Case,E), + ct:log("Expecting: ~p~n",[[Expected]]), + ct:log("Actual : ~p~n",[ParseResult]), + [Expected] = ParseResult, + do_check_xml(Case,Xmls); +do_check_xml(_,[]) -> + ok. + +%% Scanning the XML to get the same type of result as events_to_result/1 +testsuites(Case,#xmlElement{name=testsuites,content=TS}) -> + %% OTP-10589 - move properties element to + false = lists:keytake(properties,#xmlElement.name,TS), + testsuite(Case,TS). + +testsuite(Case,[#xmlElement{name=testsuite,content=TC,attributes=A}|TS]) -> + {ET,EF,ES} = events_to_numbers(lists:flatten(test_events(Case))), + {T,E,F,S} = get_numbers_from_attrs(A,false,false,false,false), + ct:log("Expecting total:~p, error:~p, failure:~p, skipped:~p~n",[ET,0,EF,ES]), + ct:log("Actual total:~p, error:~p, failure:~p, skipped:~p~n",[T,E,F,S]), + {ET,0,EF,ES} = {T,E,F,S}, + + %% properties should only be there if given a options to hook + false = lists:keytake(properties,#xmlElement.name,TC), + %% system-out and system-err is not used by common_test + false = lists:keytake('system-out',#xmlElement.name,TC), + false = lists:keytake('system-err',#xmlElement.name,TC), + R=testcase(Case,TC), + [R|testsuite(Case,TS)]; +testsuite(_Case,[]) -> + []. + +testcase(url=Case,[#xmlElement{name=testcase,attributes=A,content=C}|TC]) -> + R = failed_or_skipped(C), + case R of + [s] -> + case lists:keyfind(url,#xmlAttribute.name,A) of + false -> ok; + #xmlAttribute{value=UrlAttr} -> + lists:keyfind(url,#xmlAttribute.name,A), + true = lists:prefix(?url_base,UrlAttr) + end; + _ -> + #xmlAttribute{value=UrlAttr} = + lists:keyfind(url,#xmlAttribute.name,A), + true = lists:prefix(?url_base,UrlAttr) + end, + [R|testcase(Case,TC)]; +testcase(Case,[#xmlElement{name=testcase,attributes=A,content=C}|TC]) -> + false = lists:keyfind(url,#xmlAttribute.name,A), + R = failed_or_skipped(C), + [R|testcase(Case,TC)]; +testcase(_Case,[]) -> + []. + +failed_or_skipped([#xmlElement{name=failure}|E]) -> + [f|failed_or_skipped(E)]; +failed_or_skipped([#xmlElement{name=error}|E]) -> + [e|failed_or_skipped(E)]; +failed_or_skipped([#xmlElement{name=skipped}|E]) -> + [s|failed_or_skipped(E)]; +failed_or_skipped([]) -> + []. + +%% Using the expected events to produce the expected result of the XML scanning. +%% The result is a list of test suites: +%% Testsuites = [Testsuite] +%% Testsuite = [Testcase] +%% Testcase = [] | [f] | [s], indicating ok, failed and skipped respectively +events_to_result([{?eh,tc_done,{_Suite,_Case,R}}|E]) -> + [result(R)|events_to_result(E)]; +events_to_result([{?eh,tc_auto_skip,_}|E]) -> + [[s]|events_to_result(E)]; +events_to_result([_|E]) -> + events_to_result(E); +events_to_result([]) -> + []. + +result(ok) ->[]; +result({skipped,_}) -> [s]; +result({failed,_}) -> [f]. + +%% Using the expected events' last test_stats element to produce the +%% expected number of totla, errors, failed and skipped testcases. +events_to_numbers(E) -> + RevE = lists:reverse(E), + {?eh,test_stats,{Ok,F,{US,AS}}} = lists:keyfind(test_stats,2,RevE), + {Ok+F+US+AS,F,US+AS}. + +get_numbers_from_attrs([#xmlAttribute{name=tests,value=X}|A],false,E,F,S) -> + get_numbers_from_attrs(A,list_to_integer(X),E,F,S); +get_numbers_from_attrs([#xmlAttribute{name=errors,value=X}|A],T,false,F,S) -> + get_numbers_from_attrs(A,T,list_to_integer(X),F,S); +get_numbers_from_attrs([#xmlAttribute{name=failures,value=X}|A],T,E,false,S) -> + get_numbers_from_attrs(A,T,E,list_to_integer(X),S); +get_numbers_from_attrs([#xmlAttribute{name=skipped,value=X}|A],T,E,F,false) -> + get_numbers_from_attrs(A,T,E,F,list_to_integer(X)); +get_numbers_from_attrs([_|A],T,E,F,S) -> + get_numbers_from_attrs(A,T,E,F,S); +get_numbers_from_attrs([],T,E,F,S) -> + {T,E,F,S}. diff --git a/lib/common_test/test/ct_surefire_SUITE_data/surefire_SUITE.erl b/lib/common_test/test/ct_surefire_SUITE_data/surefire_SUITE.erl new file mode 100644 index 0000000000..677aee46c5 --- /dev/null +++ b/lib/common_test/test/ct_surefire_SUITE_data/surefire_SUITE.erl @@ -0,0 +1,92 @@ +%%-------------------------------------------------------------------- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%%---------------------------------------------------------------------- +%% File: surefire_SUITE.erl +%% +%% Description: +%% This file contains the test cases for cth_surefire. +%% +%% @author Support +%% @doc Test of surefire support in common_test +%% @end +%%---------------------------------------------------------------------- +%%---------------------------------------------------------------------- +-module(surefire_SUITE). +-include_lib("common_test/include/ct.hrl"). + +-compile(export_all). + +%% Default timetrap timeout (set in init_per_testcase). +-define(default_timeout, ?t:minutes(1)). + +all() -> + testcases() ++ [{group,g},{group,g_fail}]. + +groups() -> + [{g,testcases()}, + {g_fail,[tc_ok]}]. + +testcases() -> + [tc_ok, + tc_fail, + tc_skip, + tc_autoskip_require]. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + Config. + +init_per_group(g_fail, _Config) -> + exit(all_cases_should_be_skipped); +init_per_group(_, Config) -> + Config. + +end_per_group(_Group, Config) -> + Config. + +init_per_testcase(_Case, Config) -> + Dog = test_server:timetrap(?default_timeout), + [{watchdog, Dog}|Config]. + +end_per_testcase(_Case, Config) -> + Dog=?config(watchdog, Config), + test_server:timetrap_cancel(Dog), + ok. + +%%%----------------------------------------------------------------- +%%% Test cases +break(_Config) -> + test_server:break(""), + ok. + +tc_ok(_Config) -> + ok. + +tc_fail(_Config) -> + ct:fail("this test should fail"). + +tc_skip(_Config) -> + {skip,"this test is skipped"}. + +tc_autoskip_require() -> + [{require,whatever}]. +tc_autoskip_require(Config) -> + ct:fail("this test should never be executed - it should be autoskipped"). -- cgit v1.2.3 From 987b281e597f615f07d7ac70f2a57cc42e36454f Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Wed, 12 Dec 2012 12:31:12 +0100 Subject: [common_test] Add documentation of new url_base option to cth_surefire --- lib/common_test/doc/src/ct_hooks_chapter.xml | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/lib/common_test/doc/src/ct_hooks_chapter.xml b/lib/common_test/doc/src/ct_hooks_chapter.xml index 86237f5fc1..27d56fd47d 100644 --- a/lib/common_test/doc/src/ct_hooks_chapter.xml +++ b/lib/common_test/doc/src/ct_hooks_chapter.xml @@ -457,12 +457,26 @@ terminate(State) -> cth_surefire no - Captures all test results and outputs them as surefire XML into - a file. The file which is created is by default called junit_report.xml. - The name can be by setting the path option for this hook. e.g. +

Captures all test results and outputs them as surefire + XML into a file. The file which is created is by default + called junit_report.xml. The file name can be changed by + setting the path option for this hook, e.g.

+ -ct_hooks cth_surefire [{path,"/tmp/report.xml"}] - Surefire XML can forinstance be used by Jenkins to display test - results. + +

If the url_base option is set, an additional + attribute named url will be added to each + testsuite and testcase XML element. The value will + be a constructed from the url_base and a relative path + to the test suite or test case log respectively, e.g.

+ + -ct_hooks cth_surefire [{url_base,"http://myserver.com/"}] +

will give a url attribute value similar to

+ + "http://myserver.com/ct_run.ct@myhost.2012-12-12_11.19.39/x86_64-unknown-linux-gnu.my_test.logs/run.2012-12-12_11.19.39/suite.log.html" + +

Surefire XML can for instance be used by Jenkins to display test + results.

-- cgit v1.2.3 From 3c05d3389f8b45e5b9f13a874de53ec521981a41 Mon Sep 17 00:00:00 2001 From: Erlang/OTP Date: Wed, 12 Dec 2012 17:24:24 +0100 Subject: Prepare release --- lib/common_test/doc/src/notes.xml | 50 +++++++++++++++++++++++++++++++++++++++ lib/common_test/vsn.mk | 2 +- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/lib/common_test/doc/src/notes.xml b/lib/common_test/doc/src/notes.xml index 7e33b71de1..8c3b13951d 100644 --- a/lib/common_test/doc/src/notes.xml +++ b/lib/common_test/doc/src/notes.xml @@ -32,6 +32,56 @@ notes.xml +
Common_Test 1.6.3.1 + +
Known Bugs and Problems + + +

+ The following corrections/changes are done in the + cth_surefire hook:

+

+ Earlier there would always be a + 'properties' element under the 'testsuites' element. This + would exist even if there were no 'property' element + inside it. This has been changed so if there are no + 'property' elements to display, then there will not be a + 'properties' element either. The XML file + will now (unless other is specified) be stored in the top + log directory. Earlier, the default directory would be + the current working directory for the erlang node, which + would mostly, but not always, be the top log directory. + The 'hostname' attribute in the + 'testsuite' element would earlier never have the correct + value. This has been corrected. The + 'errors' attribute in the 'testsuite' element would + earlier display the number of failed testcases. This has + been changed and will now always have the value 0, while + the 'failures' attribute will show the number of failed + testcases. A new attribute 'skipped' is + added to the 'testsuite' element. This will display the + number of skipped testcases. These would earlier be + included in the number of failed test cases. + The total number of tests displayed by the 'tests' + attribute in the 'testsuite' element would earlier + include init/end_per_suite and init/end_per_group. This + is no longer the case. The 'tests' attribute will now + only count "real" test cases. Earlier, + auto skipped test cases would have no value in the 'log' + attribute. This is now corrected. A new + attributes 'log' is added to the 'testsuite' element. + A new option named 'url_base' is added for + this hook. If this option is used, a new attribute named + 'url' will be added to the 'testcase' and 'testsuite' + elements.

+

+ Own Id: OTP-10589

+ + +
+ +
+
Common_Test 1.6.3
Fixed Bugs and Malfunctions diff --git a/lib/common_test/vsn.mk b/lib/common_test/vsn.mk index f9bb22867e..6869c08636 100644 --- a/lib/common_test/vsn.mk +++ b/lib/common_test/vsn.mk @@ -1 +1 @@ -COMMON_TEST_VSN = 1.6.3 +COMMON_TEST_VSN = 1.6.3.1 -- cgit v1.2.3