diff options
Diffstat (limited to 'lib/common_test/test')
24 files changed, 3621 insertions, 92 deletions
diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile index 7691920993..d469d03e04 100644 --- a/lib/common_test/test/Makefile +++ b/lib/common_test/test/Makefile @@ -53,7 +53,11 @@ MODULES= \ ct_verbosity_SUITE \ ct_shell_SUITE \ ct_system_error_SUITE \ - ct_snmp_SUITE + ct_snmp_SUITE \ + ct_group_leader_SUITE \ + ct_cover_SUITE \ + ct_groups_search_SUITE \ + ct_surefire_SUITE ERL_FILES= $(MODULES:%=%.erl) @@ -107,7 +111,7 @@ release_spec: opt release_tests_spec: $(INSTALL_DIR) "$(RELSYSDIR)" $(INSTALL_DATA) $(ERL_FILES) $(COVERFILE) "$(RELSYSDIR)" - $(INSTALL_DATA) common_test.spec "$(RELSYSDIR)" + $(INSTALL_DATA) common_test.spec common_test.cover "$(RELSYSDIR)" chmod -R u+w "$(RELSYSDIR)" @tar cf - *_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -) diff --git a/lib/common_test/test/common_test.cover b/lib/common_test/test/common_test.cover new file mode 100644 index 0000000000..3aa49623e7 --- /dev/null +++ b/lib/common_test/test/common_test.cover @@ -0,0 +1,10 @@ +%% -*- erlang -*- +{incl_app,common_test,details}. +{cross,common_test,[{test_server,[erl2html2, + test_server, + test_server_ctrl, + test_server_gl, + test_server_h, + test_server_io, + test_server_node, + test_server_sup]}]}. diff --git a/lib/common_test/test/ct_config_SUITE.erl b/lib/common_test/test/ct_config_SUITE.erl index 0b1abae757..d92be9ec6e 100644 --- a/lib/common_test/test/ct_config_SUITE.erl +++ b/lib/common_test/test/ct_config_SUITE.erl @@ -88,8 +88,8 @@ require(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), run_test(config_static_SUITE, Config, - [{config, filename:join(DataDir, "config/shadow.txt")}, - {config, filename:join(DataDir, "config/config.txt")}], + [{config, [filename:join(DataDir, "config/shadow.txt"), + filename:join(DataDir, "config/config.txt")]}], ["config_static_SUITE"]). install_config(Config) when is_list(Config) -> @@ -174,6 +174,7 @@ run_test(Name, Config, CTConfig, SuiteNames)-> Joiner = fun(Suite) -> filename:join(DataDir, "config/test/"++Suite) end, Suites = lists:map(Joiner, SuiteNames), {Opts,ERPid} = setup_env({suite,Suites}, Config, CTConfig), + ok = ct_test_support:run(Opts, Config), TestEvents = ct_test_support:get_events(ERPid, Config), ct_test_support:log_events(Name, @@ -251,6 +252,7 @@ expected_events(config_static_SUITE)-> ?sok(test_alias_tclocal_nested,{14,0,{2,1}}), ?sok(test_alias_tclocal_nested_backward_compat,{15,0,{2,1}}), ?sok(test_alias_tclocal_nested_backward_compat_subvals,{16,0,{2,1}}), + ?sok(test_config_same_name_already_in_use,{17,0,{2,1}}), {?eh,tc_start,{config_static_SUITE,end_per_suite}}, {?eh,tc_done,{config_static_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, diff --git a/lib/common_test/test/ct_config_SUITE_data/config/test/config_static_SUITE.erl b/lib/common_test/test/ct_config_SUITE_data/config/test/config_static_SUITE.erl index 2e1ad651e8..19f1dab4af 100644 --- a/lib/common_test/test/ct_config_SUITE_data/config/test/config_static_SUITE.erl +++ b/lib/common_test/test/ct_config_SUITE_data/config/test/config_static_SUITE.erl @@ -64,7 +64,8 @@ all() -> [test_get_config_simple, test_get_config_nested, test_shadow_all,test_element,test_shadow_all_element, test_internal_deep, test_alias_tclocal_nested, test_alias_tclocal_nested_backward_compat, - test_alias_tclocal_nested_backward_compat_subvals + test_alias_tclocal_nested_backward_compat_subvals, + test_config_same_name_already_in_use ]. init_per_testcase(_,Config) -> @@ -124,6 +125,13 @@ test_config_name_already_in_use2(_) -> ct:fail("Test should've been skipped, you shouldn't see this!"), ok. + +test_config_same_name_already_in_use() -> + []. +test_config_same_name_already_in_use(_) -> + ok = ct:require(x2,{gen_cfg,c}), + ok = ct:require(x2,{gen_cfg,c}). + %% test aliases test_alias_tclocal() -> [{require,newalias,gen_cfg}]. diff --git a/lib/common_test/test/ct_config_info_SUITE.erl b/lib/common_test/test/ct_config_info_SUITE.erl index 40da377ee5..10fe8286dd 100644 --- a/lib/common_test/test/ct_config_info_SUITE.erl +++ b/lib/common_test/test/ct_config_info_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2011. All Rights Reserved. +%% Copyright Ericsson AB 2009-2012. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -123,8 +123,7 @@ test_events(config_info) -> {?eh,tc_done,{config_info_1_SUITE,init_per_suite,ok}}, [{?eh,tc_start,{config_info_1_SUITE,{init_per_group,g1,[]}}}, - {?eh,tc_done,{config_info_1_SUITE, - {init_per_group,unknown,[]}, + {?eh,tc_done,{config_info_1_SUITE,{init_per_group,g1,[]}, {failed,{timetrap_timeout,350}}}}, {?eh,tc_auto_skip,{config_info_1_SUITE,t11, {failed,{config_info_1_SUITE,init_per_group,{timetrap_timeout,350}}}}}, @@ -136,14 +135,12 @@ test_events(config_info) -> {?eh,tc_done,{config_info_1_SUITE,{init_per_group,g2,[]},ok}}, {?eh,tc_done,{config_info_1_SUITE,t21,ok}}, {?eh,tc_start,{config_info_1_SUITE,{end_per_group,g2,[]}}}, - {?eh,tc_done,{config_info_1_SUITE, - {end_per_group,unknown,[]}, + {?eh,tc_done,{config_info_1_SUITE,{end_per_group,g2,[]}, {failed,{timetrap_timeout,450}}}}], [{?eh,tc_start,{config_info_1_SUITE,{init_per_group,g3,[]}}}, {?eh,tc_done,{config_info_1_SUITE,{init_per_group,g3,[]},ok}}, [{?eh,tc_start,{config_info_1_SUITE,{init_per_group,g4,[]}}}, - {?eh,tc_done,{config_info_1_SUITE, - {init_per_group,unknown,[]}, + {?eh,tc_done,{config_info_1_SUITE,{init_per_group,g4,[]}, {failed,{timetrap_timeout,400}}}}, {?eh,tc_auto_skip,{config_info_1_SUITE,t41, {failed,{config_info_1_SUITE,init_per_group, @@ -164,8 +161,7 @@ test_events(config_info) -> {?eh,tc_done,{config_info_1_SUITE,{init_per_group,g5,[]},ok}}, {?eh,tc_done,{config_info_1_SUITE,t51,ok}}, {?eh,tc_start,{config_info_1_SUITE,{end_per_group,g5,[]}}}, - {?eh,tc_done,{config_info_1_SUITE, - {end_per_group,unknown,[]}, + {?eh,tc_done,{config_info_1_SUITE,{end_per_group,g5,[]}, {failed,{timetrap_timeout,400}}}}], {?eh,tc_start,{config_info_1_SUITE,{end_per_group,g3,[]}}}, {?eh,tc_done,{config_info_1_SUITE,{end_per_group,g3,[]},ok}}], diff --git a/lib/common_test/test/ct_cover_SUITE.erl b/lib/common_test/test/ct_cover_SUITE.erl new file mode 100644 index 0000000000..cb49dc423f --- /dev/null +++ b/lib/common_test/test/ct_cover_SUITE.erl @@ -0,0 +1,312 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_cover_SUITE +%%% +%%% Description: +%%% Test code cover analysis support +%%% +%%%------------------------------------------------------------------- +-module(ct_cover_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). +-define(suite, cover_SUITE). +-define(mod, cover_test_mod). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + case test_server:is_cover() of + true -> + {skip,"Test server is running cover already - skipping"}; + false -> + ct_test_support:init_per_suite(Config) + end. + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + Node = fullname(existing_node), + case lists:member(Node,nodes()) of + true -> rpc:call(Node,erlang,halt,[]); + false -> ok + end, + ct_test_support:end_per_testcase(TestCase, Config). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [ + default, + cover_stop_true, + cover_stop_false, + slave, + slave_start_slave, + cover_node_option, + ct_cover_add_remove_nodes, + otp_9956, + cross + ]. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%% Check that cover is collected from test node +%% Also check that cover is by default stopped after test is completed +default(Config) -> + {ok,Events} = run_test(default,Config), + false = check_cover(Config), + check_calls(Events,1), + ok. + +%% Check that cover is stopped when cover_stop option is set to true +cover_stop_true(Config) -> + {ok,_Events} = run_test(cover_stop_true,[{cover_stop,true}],Config), + false = check_cover(Config). + +%% Check that cover is not stopped when cover_stop option is set to false +cover_stop_false(Config) -> + {ok,_Events} = run_test(cover_stop_false,[{cover_stop,false}],Config), + {true,[],[?mod]} = check_cover(Config), + CTNode = proplists:get_value(ct_node, Config), + ok = rpc:call(CTNode,cover,stop,[]), + false = check_cover(Config), + ok. + +%% Let test node start a slave node - check that cover is collected +%% from both nodes +slave(Config) -> + {ok,Events} = run_test(slave,slave,[],Config), + check_calls(Events,2), + ok. + +%% Let test node start a slave node which in turn starts another slave +%% node - check that cover is collected from all three nodes +slave_start_slave(Config) -> + {ok,Events} = run_test(slave_start_slave,slave_start_slave,[],Config), + check_calls(Events,3), + ok. + +%% Start a slave node before test starts - the node is listed in cover +%% spec file. +%% Check that cover is collected from test node and slave node. +cover_node_option(Config) -> + {ok, HostStr}=inet:gethostname(), + Host = list_to_atom(HostStr), + DataDir = ?config(data_dir,Config), + {ok,Node} = ct_slave:start(Host,existing_node, + [{erl_flags,"-pa " ++ DataDir}]), + false = check_cover(Node), + CoverSpec = default_cover_file_content() ++ [{nodes,[Node]}], + CoverFile = create_cover_file(cover_node_option,CoverSpec,Config), + {ok,Events} = run_test(cover_node_option,cover_node_option, + [{cover,CoverFile}],Config), + check_calls(Events,2), + {ok,Node} = ct_slave:stop(existing_node), + ok. + +%% Test ct_cover:add_nodes/1 and ct_cover:remove_nodes/1 +%% Check that cover is collected from added node +ct_cover_add_remove_nodes(Config) -> + {ok, HostStr}=inet:gethostname(), + Host = list_to_atom(HostStr), + DataDir = ?config(data_dir,Config), + {ok,Node} = ct_slave:start(Host,existing_node, + [{erl_flags,"-pa " ++ DataDir}]), + false = check_cover(Node), + {ok,Events} = run_test(ct_cover_add_remove_nodes,ct_cover_add_remove_nodes, + [],Config), + check_calls(Events,2), + {ok,Node} = ct_slave:stop(existing_node), + ok. + +%% Test that the test suite itself can be cover compiled and that +%% data_dir is set correctly (OTP-9956) +otp_9956(Config) -> + CoverFile = create_cover_file(otp_9956,[{incl_mods,[?suite]}],Config), + {ok,Events} = run_test(otp_9956,otp_9956,[{cover,CoverFile}],Config), + check_calls(Events,{?suite,otp_9956,1},1), + ok. + +%% Test cross cover mechanism +cross(Config) -> + {ok,Events1} = run_test(cross1,Config), + check_calls(Events1,1), + + CoverFile2 = create_cover_file(cross1,[{cross,[{cross1,[?mod]}]}],Config), + {ok,Events2} = run_test(cross2,[{cover,CoverFile2}],Config), + check_calls(Events2,1), + + %% Get the log dirs for each test and run cross cover analyse + [D11,D12] = lists:sort(get_run_dirs(Events1)), + [D21,D22] = lists:sort(get_run_dirs(Events2)), + + ct_cover:cross_cover_analyse(details,[{cross1,D11},{cross2,D21}]), + ct_cover:cross_cover_analyse(details,[{cross1,D12},{cross2,D22}]), + + %% Get the cross cover logs and read for each test + [C11,C12,C21,C22] = + [filename:join(D,"cross_cover.html") || D <- [D11,D12,D21,D22]], + + {ok,CrossData} = file:read_file(C11), + {ok,CrossData} = file:read_file(C12), + + {ok,Def} = file:read_file(C21), + {ok,Def} = file:read_file(C22), + + %% A simple test: just check that the test module exists in the + %% log from cross1 test, and that it does not exist in the log + %% from cross2 test. + TestMod = list_to_binary(atom_to_list(?mod)), + {_,_} = binary:match(CrossData,TestMod), + nomatch = binary:match(Def,TestMod), + {_,_} = binary:match(Def, + <<"No cross cover modules exist for this application">>), + + ok. + + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- +run_test(Label,Config) -> + run_test(Label,[],Config). +run_test(Label,ExtraOpts,Config) -> + run_test(Label,default,ExtraOpts,Config). +run_test(Label,Testcase,ExtraOpts,Config) -> + DataDir = ?config(data_dir, Config), + Suite = filename:join(DataDir, ?suite), + CoverFile = + case proplists:get_value(cover,ExtraOpts) of + undefined -> + create_default_cover_file(Label,Config); + CF -> + CF + end, + RestOpts = lists:keydelete(cover,1,ExtraOpts), + {Opts,ERPid} = setup([{suite,Suite},{testcase,Testcase}, + {cover,CoverFile},{label,Label}] ++ RestOpts, Config), + execute(Label, Testcase, Opts, ERPid, Config). + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +execute(Name, Testcase, Opts, ERPid, Config) -> + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(Name, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + TestEvents = events_to_check(Testcase), + R = ct_test_support:verify_events(TestEvents, Events, Config), + {R,Events}. + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). + +events_to_check(Testcase) -> + OneTest = + [{?eh,start_logging,{'DEF','RUNDIR'}}] ++ + [{?eh,tc_done,{?suite,Testcase,ok}}] ++ + [{?eh,stop_logging,[]}], + + %% 2 tests (ct:run_test + script_start) is default + OneTest ++ OneTest. + +check_cover(Config) when is_list(Config) -> + CTNode = proplists:get_value(ct_node, Config), + check_cover(CTNode); +check_cover(Node) when is_atom(Node) -> + case rpc:call(Node,test_server,is_cover,[]) of + true -> + {true, + rpc:call(Node,cover,which_nodes,[]), + rpc:call(Node,cover,modules,[])}; + false -> + false + end. + +%% Get the log dir "run.<timestamp>" for all (both!) tests +get_run_dirs(Events) -> + [filename:dirname(TCLog) || + {ct_test_support_eh, + {event,tc_logfile,_Node, + {{?suite,init_per_suite},TCLog}}} <- Events]. + +%% Check that each coverlog includes N calls to ?mod:foo/0 +check_calls(Events,N) -> + check_calls(Events,{?mod,foo,0},N). +check_calls(Events,MFA,N) -> + CoverLogs = [filename:join(D,"all.coverdata") || D <- get_run_dirs(Events)], + do_check_logs(CoverLogs,MFA,N). + +do_check_logs([CoverLog|CoverLogs],{Mod,_,_} = MFA,N) -> + {ok,_} = cover:start(), + ok = cover:import(CoverLog), + {ok,Calls} = cover:analyse(Mod,calls,function), + ok = cover:stop(), + {MFA,N} = lists:keyfind(MFA,1,Calls), + do_check_logs(CoverLogs,MFA,N); +do_check_logs([],_,_) -> + ok. + +fullname(Name) -> + {ok,Host} = inet:gethostname(), + list_to_atom(atom_to_list(Name) ++ "@" ++ Host). + +default_cover_file_content() -> + [{incl_mods,[?mod]}]. + +create_default_cover_file(Filename,Config) -> + create_cover_file(Filename,default_cover_file_content(),Config). + +create_cover_file(Filename,Terms,Config) -> + PrivDir = ?config(priv_dir,Config), + File = filename:join(PrivDir,Filename) ++ ".cover", + {ok,Fd} = file:open(File,[write]), + lists:foreach(fun(Term) -> + file:write(Fd,io_lib:format("~p.~n",[Term])) + end,Terms), + ok = file:close(Fd), + File. diff --git a/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl new file mode 100644 index 0000000000..fdc3323f0a --- /dev/null +++ b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl @@ -0,0 +1,156 @@ +%%-------------------------------------------------------------------- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%%---------------------------------------------------------------------- +%% File: cover_SUITE.erl +%% +%% Description: +%% This file contains the test cases for the code coverage support +%% +%% @author Support +%% @doc Test of code coverage support in common_test +%% @end +%%---------------------------------------------------------------------- +%%---------------------------------------------------------------------- +-module(cover_SUITE). +-include_lib("common_test/include/ct.hrl"). + +-compile(export_all). + +%% Default timetrap timeout (set in init_per_testcase). +-define(default_timeout, ?t:minutes(1)). + +suite() -> + []. + +all() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + Config. + +init_per_testcase(_Case, Config) -> + Dog = test_server:timetrap(?default_timeout), + [{watchdog, Dog}|Config]. + +end_per_testcase(Case, Config) -> + %% try apply(?MODULE,Case,[cleanup,Config]) + %% catch error:undef -> ok + %% end, + + kill_slaves(Case,nodes()), + Dog=?config(watchdog, Config), + test_server:timetrap_cancel(Dog), + ok. + +%%%----------------------------------------------------------------- +%%% Test cases +break(_Config) -> + test_server:break(""), + ok. + +default(Config) -> + cover_compiled = code:which(cover_test_mod), + cover_test_mod:foo(), + ok. + +slave(Config) -> + cover_compiled = code:which(cover_test_mod), + cover_test_mod:foo(), + N1 = nodename(slave,1), + {ok,Node} = ct_slave:start(N1), + cover_compiled = rpc:call(Node,code,which,[cover_test_mod]), + rpc:call(Node,cover_test_mod,foo,[]), + {ok,Node} = ct_slave:stop(N1), + ok. + +slave_start_slave(Config) -> + cover_compiled = code:which(cover_test_mod), + cover_test_mod:foo(), + N1 = nodename(slave_start_slave,1), + N2 = nodename(slave_start_slave,2), + {ok,Node} = ct_slave:start(N1), + cover_compiled = rpc:call(Node,code,which,[cover_test_mod]), + rpc:call(Node,cover_test_mod,foo,[]), + {ok,Node2} = rpc:call(Node,ct_slave,start,[N2]), + rpc:call(Node2,cover_test_mod,foo,[]), + {ok,Node2} = rpc:call(Node,ct_slave,stop,[N2]), + {ok,Node} = ct_slave:stop(N1), + ok. + +cover_node_option(Config) -> + cover_compiled = code:which(cover_test_mod), + cover_test_mod:foo(), + Node = fullname(existing_node), + cover_compiled = rpc:call(Node,code,which,[cover_test_mod]), + rpc:call(Node,cover_test_mod,foo,[]), + ok. + +ct_cover_add_remove_nodes(Config) -> + cover_compiled = code:which(cover_test_mod), + cover_test_mod:foo(), + Node = fullname(existing_node), + Beam = rpc:call(Node,code,which,[cover_test_mod]), + false = (Beam == cover_compiled), + + rpc:call(Node,cover_test_mod,foo,[]), % should not be collected + {ok,[Node]} = ct_cover:add_nodes([Node]), + cover_compiled = rpc:call(Node,code,which,[cover_test_mod]), + rpc:call(Node,cover_test_mod,foo,[]), % should be collected + ok = ct_cover:remove_nodes([Node]), + rpc:call(Node,cover_test_mod,foo,[]), % should not be collected + + Beam = rpc:call(Node,code,which,[cover_test_mod]), + + ok. + +otp_9956(Config) -> + cover_compiled = code:which(?MODULE), + DataDir = ?config(data_dir,Config), + absolute = filename:pathtype(DataDir), + true = filelib:is_dir(DataDir), + ok. + + +%%%----------------------------------------------------------------- +%%% Internal +nodename(Case,N) -> + list_to_atom(nodeprefix(Case) ++ integer_to_list(N)). + +nodeprefix(Case) -> + atom_to_list(?MODULE) ++ "_" ++ atom_to_list(Case) ++ "_node". + + +fullname(Name) -> + {ok,Host} = inet:gethostname(), + list_to_atom(atom_to_list(Name) ++ "@" ++ Host). + +kill_slaves(Case, [Node|Nodes]) -> + Prefix = nodeprefix(Case), + case lists:prefix(Prefix,atom_to_list(Node)) of + true -> + rpc:call(Node,erlang,halt,[]); + _ -> + ok + end, + kill_slaves(Case,Nodes); +kill_slaves(_,[]) -> + ok. diff --git a/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE_data/.gitignore b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE_data/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE_data/.gitignore diff --git a/lib/common_test/test/ct_cover_SUITE_data/cover_test_mod.erl b/lib/common_test/test/ct_cover_SUITE_data/cover_test_mod.erl new file mode 100644 index 0000000000..d4f69452c3 --- /dev/null +++ b/lib/common_test/test/ct_cover_SUITE_data/cover_test_mod.erl @@ -0,0 +1,4 @@ +-module(cover_test_mod). +-compile(export_all). +foo() -> + ok. diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index 338e76264e..6d90b29f41 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -61,7 +61,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [cfg_error, lib_error, no_compile, timetrap_end_conf, timetrap_normal, timetrap_extended, timetrap_parallel, - timetrap_fun, misc_errors]. + timetrap_fun, timetrap_fun_group, misc_errors]. groups() -> []. @@ -251,6 +251,24 @@ timetrap_fun(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% +timetrap_fun_group(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + Join = fun(D, S) -> filename:join(D, "error/test/"++S) end, + Suites = [Join(DataDir, "timetrap_8_SUITE")], + {Opts,ERPid} = setup([{suite,Suites}], Config), + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(timetrap_fun_group, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(timetrap_fun_group), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + +%%%----------------------------------------------------------------- +%%% misc_errors(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), Join = fun(D, S) -> filename:join(D, "error/test/"++S) end, @@ -429,8 +447,7 @@ test_events(cfg_error) -> {'EXIT',{init_per_group_fails,g1}}}}}}], [{?eh,tc_start,{cfg_error_8_SUITE,{init_per_group,g2,[]}}}, - {?eh,tc_done,{cfg_error_8_SUITE, - {init_per_group,unknown,[]}, + {?eh,tc_done,{cfg_error_8_SUITE,{init_per_group,g2,[]}, {failed,{timetrap_timeout,2000}}}}, {?eh,tc_auto_skip,{cfg_error_8_SUITE,tc1, {failed,{cfg_error_8_SUITE,init_per_group, @@ -500,7 +517,7 @@ test_events(cfg_error) -> {?eh,tc_done,{cfg_error_8_SUITE,tc1,ok}}, {?eh,test_stats,{9,0,{0,14}}}, {?eh,tc_start,{cfg_error_8_SUITE,{end_per_group,g12,[]}}}, - {?eh,tc_done,{cfg_error_8_SUITE,{end_per_group,unknown,[]}, + {?eh,tc_done,{cfg_error_8_SUITE,{end_per_group,g12,[]}, {failed,{timetrap_timeout,2000}}}}], {?eh,tc_start,{cfg_error_8_SUITE,end_per_suite}}, @@ -971,11 +988,423 @@ test_events(timetrap_fun) -> {?eh,stop_logging,[]} ]; +test_events(timetrap_fun_group) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,58}}, + {?eh,tc_start,{timetrap_8_SUITE,init_per_suite}}, + {?eh,tc_done,{timetrap_8_SUITE,init_per_suite,ok}}, + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g0,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g0,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{0,1,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{0,2,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g0,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g0,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g1,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g1,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{0,3,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{0,4,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g1,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g1,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g2,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g2,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc1}}, + {?eh,tc_done,{timetrap_8_SUITE,tc1, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{0,5,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{0,6,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g2,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g2,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g3,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g3,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc4}}, + {?eh,tc_done,{timetrap_8_SUITE,tc4, + {failed,{timetrap_timeout,{'$approx',2000}}}}}, + {?eh,test_stats,{0,7,{0,0}}}, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g1,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g1,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{0,8,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{0,9,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g1,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g1,[]},ok}}], + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g2,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g2,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc1}}, + {?eh,tc_done,{timetrap_8_SUITE,tc1, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{0,10,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{0,11,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g2,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g2,[]},ok}}], + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g3,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g3,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g4,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g4,[]}, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{0,11,{0,1}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{0,11,{0,2}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g5,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g5,[]}, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{0,11,{0,3}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{0,11,{0,4}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g6,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g6,[]}, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}, + {?eh,test_stats,{0,11,{0,5}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}, + {?eh,test_stats,{0,11,{0,6}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g7,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g7,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{1,11,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g7,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g7,[]}, + {user_timetrap_error,{kaboom,'_'}}}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g8,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g8,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{2,11,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g8,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g8,[]}, + {failed,{timetrap_timeout,{'$approx',500}}}}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g9,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g9,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{3,11,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,test_stats,{3,12,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g9,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g9,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g10,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g10,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,test_stats,{3,13,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{4,13,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g10,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g10,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g11,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g11,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc3}}, + {?eh,tc_done,{timetrap_8_SUITE,tc3, + {failed,{timetrap_timeout,{'$approx',4000}}}}}, + {?eh,test_stats,{4,14,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,15,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g11,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g11,[]},ok}}], + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg0,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg0,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{4,16,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,17,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg0,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg0,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg1,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg1,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{4,18,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,19,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg1,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg1,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg2,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg2,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc1}}, + {?eh,tc_done,{timetrap_8_SUITE,tc1, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{4,20,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,21,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg2,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg2,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg3,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg3,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc4}}, + {?eh,tc_done,{timetrap_8_SUITE,tc4, + {failed,{timetrap_timeout,{'$approx',2000}}}}}, + {?eh,test_stats,{4,22,{0,6}}}, + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg1,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg1,[parallel]}, + ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{4,23,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,24,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg1,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg1,[parallel]}, + ok}}]}, + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg2,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg2,[parallel]}, + ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc1}}, + {?eh,tc_done,{timetrap_8_SUITE,tc1, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{4,25,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,26,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg2,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg2,[parallel]}, + ok}}]}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg3,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg3,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg4,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg4,[parallel]}, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{4,26,{0,7}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{4,26,{0,8}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg5,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg5,[parallel]}, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{4,26,{0,9}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{4,26,{0,10}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg6,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg6,[parallel]}, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}, + {?eh,test_stats,{4,26,{0,11}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}, + {?eh,test_stats,{4,26,{0,12}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg7,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg7,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{5,26,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg7,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg7,[parallel]}, + {user_timetrap_error,{kaboom,'_'}}}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg8,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg8,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{6,26,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg8,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg8,[parallel]}, + {failed,{timetrap_timeout,{'$approx',500}}}}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg9,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg9,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {user_timetrap_error,{kaboom,'_'}}}}, + %% Due to parallelism only checking final test stat in group + {?eh,test_stats,{7,27,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg9,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg9,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg10,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg10,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + %% Due to parallelism only checking final test stat in group + {?eh,test_stats,{8,28,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg10,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg10,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg11,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg11,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc3}}, + {?eh,tc_done,{timetrap_8_SUITE,tc3, + {failed,{timetrap_timeout,{'$approx',4000}}}}}, + {?eh,test_stats,{8,29,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{8,30,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg11,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg11,[parallel]},ok}}]}, + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,sg1,[sequence]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,sg1,[sequence]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{9,30,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,test_stats,{9,31,{0,12}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc1, + {failed,{timetrap_8_SUITE,tc0}}}}, + {?eh,test_stats,{9,31,{0,13}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,tc0}}}}, + {?eh,test_stats,{9,31,{0,14}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,sg1,[sequence]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,sg1,[sequence]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,sg2,[sequence]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,sg2,[sequence]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{10,31,{0,14}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{10,32,{0,14}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc1, + {failed,{timetrap_8_SUITE,tc0}}}}, + {?eh,test_stats,{10,32,{0,15}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,tc0}}}}, + {?eh,test_stats,{10,32,{0,16}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,sg2,[sequence]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,sg2,[sequence]},ok}}], + + {?eh,tc_start,{timetrap_8_SUITE,end_per_suite}}, + {?eh,tc_done,{timetrap_8_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + test_events(misc_errors) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, - {?eh,start_info,{1,1,7}}, + {?eh,start_info,{1,1,9}}, {?eh,tc_start,{misc_error_1_SUITE,ct_fail_1}}, {?eh,tc_done,{misc_error_1_SUITE,ct_fail_1, {failed,{error,{test_case_failed,{error,this_is_expected}}}}}}, @@ -1002,7 +1431,12 @@ test_events(misc_errors) -> {?eh,tc_start,{misc_error_1_SUITE,killed_by_signal_2}}, {?eh,tc_done,{misc_error_1_SUITE,killed_by_signal_2, {failed,testcase_aborted_or_killed}}}, - {?eh,test_stats,{0,7,{0,0}}}, + {parallel, + [{?eh,tc_start,{misc_error_1_SUITE,p1}}, + {?eh,tc_done,{misc_error_1_SUITE,p1,ok}}, + {?eh,tc_start,{misc_error_1_SUITE,p2}}, + {?eh,tc_done,{misc_error_1_SUITE,p2,ok}}]}, + {?eh,test_stats,{2,7,{0,0}}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} ]. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/misc_error_1_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/misc_error_1_SUITE.erl index 99c3ed05ec..61f3fa7e59 100644 --- a/lib/common_test/test/ct_error_SUITE_data/error/test/misc_error_1_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/misc_error_1_SUITE.erl @@ -96,7 +96,7 @@ end_per_testcase(_TestCase, _Config) -> %% N = integer() | forever %%-------------------------------------------------------------------- groups() -> - []. + [{p,[parallel],[p1,p2]}]. %%-------------------------------------------------------------------- %% Function: all() -> GroupsAndTestCases | {skip,Reason} @@ -107,7 +107,8 @@ groups() -> %%-------------------------------------------------------------------- all() -> [ct_fail_1, ct_fail_2, ct_fail_3, ts_fail_1, ts_fail_2, - killed_by_signal_1, killed_by_signal_2]. + killed_by_signal_1, killed_by_signal_2, + {group,p}]. ct_fail_1(_) -> ct:fail({error,this_is_expected}), @@ -152,3 +153,10 @@ killed_by_signal_2(_) -> end), ct:sleep(1000), exit(this_should_not_be_seen). + +p1(_) -> + {error,parallel_group} = ct:abort_current_testcase(aborted), + ok. + +p2(_) -> + receive after 1000 -> ok end. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_8_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_8_SUITE.erl new file mode 100644 index 0000000000..ff138f38b5 --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_8_SUITE.erl @@ -0,0 +1,258 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(timetrap_8_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +-define(TO, 4). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{timetrap_utils,timetrap_val,[{seconds,?TO}]}}]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_group(G6, Config) when G6==g6; G6==pg6 -> + ct:sleep({seconds,1}), + Config; +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_group(G7or8, _Config) when G7or8==g7; G7or8==pg7; G7or8==g8; G7or8==pg8 -> + ct:sleep({seconds,5}), + ok; +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_testcase(_, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_testcase(_, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + [ + {g0,[],[tc0,tc2]}, % group override suite and tc overrides group + {g1,[],[tc0,tc2]}, % group override suite and tc overrides group + {g2,[],[tc1,tc2]}, % tc override group + {g3,[],[tc4,{group,g1},{group,g2}]}, % subgroup override group + {g4,[],[tc0,tc2]}, % exit during init_per_group + {g5,[],[tc0,tc2]}, % exit during init_per_group + {g6,[],[tc0,tc2]}, % timeout during init_per_group + {g7,[],[tc5]}, % exit during end_per_group + {g8,[],[tc5]}, % timeout during end_per_group + {g9,[],[tc5,tc0]}, % exit during testcase + {g10,[],[tc0,tc5]}, % exit during testcase + {g11,[],[tc3,tc2]}, % suite is valid if nothing else is specified + {pg0,[parallel],[tc0,tc2]}, % group override suite and tc overrides group + {pg1,[parallel],[tc0,tc2]}, % group override suite and tc overrides group + {pg2,[parallel],[tc1,tc2]}, % tc override group + {pg3,[parallel],[tc4,{group,pg1},{group,pg2}]}, % subgroup override group + {pg4,[parallel],[tc0,tc2]}, % exit during init_per_group + {pg5,[parallel],[tc0,tc2]}, % exit during init_per_group + {pg6,[parallel],[tc0,tc2]}, % timeout during init_per_group + {pg7,[parallel],[tc5]}, % exit during end_per_group + {pg8,[parallel],[tc5]}, % timeout during end_per_group + {pg9,[parallel],[tc5,tc0]}, % exit during testcase + {pg10,[parallel],[tc0,tc5]},% exit during testcase + {pg11,[parallel],[tc3,tc2]},% suite is valid if nothing else is specified + {sg1,[sequence],[tc5,tc0,tc1,tc2]}, % exit during sequencial testcase + {sg2,[sequence],[tc5,tc0,tc1,tc2]}].% timeout during sequencial testcase + +group(g0) -> + [{timetrap,{timetrap_utils,timetrap_val,[{seconds,1}]}}]; +group(g1) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(1000) end}]; +group(g2) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(3000) end}]; +group(g3) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(2000) end}]; +group(g4) -> + [{timetrap,{timetrap_utils,timetrap_exit,[kaboom]}}]; +group(g5) -> + [{timetrap,fun() -> exit(kaboom) end}]; +group(g6) -> + [{timetrap,{timetrap_utils,timetrap_val,[500]}}]; +group(g7) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(g8) -> + [{timetrap,{timetrap_utils,timetrap_val,[500]}}]; +group(g9) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(g10) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(g11) -> + []; +group(pg0) -> + [{timetrap,{timetrap_utils,timetrap_val,[{seconds,1}]}}]; +group(pg1) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(1000) end}]; +group(pg2) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(3000) end}]; +group(pg3) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(2000) end}]; +group(pg4) -> + [{timetrap,{timetrap_utils,timetrap_exit,[kaboom]}}]; +group(pg5) -> + [{timetrap,fun() -> exit(kaboom) end}]; +group(pg6) -> + [{timetrap,{timetrap_utils,timetrap_val,[500]}}]; +group(pg7) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(pg8) -> + [{timetrap,{timetrap_utils,timetrap_val,[500]}}]; +group(pg9) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(pg10) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(pg11) -> + []; +group(sg1) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(sg2) -> + [{timetrap,{timetrap_utils,timetrap_val,[{seconds,1}]}}]. + + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [ + {group,g0}, + {group,g1}, + {group,g2}, + {group,g3}, + {group,g4}, + {group,g5}, + {group,g6}, + {group,g7}, + {group,g8}, + {group,g9}, + {group,g10}, + {group,g11}, + {group,pg0}, + {group,pg1}, + {group,pg2}, + {group,pg3}, + {group,pg4}, + {group,pg5}, + {group,pg6}, + {group,pg7}, + {group,pg8}, + {group,pg9}, + {group,pg10}, + {group,pg11}, + {group,sg1}, + {group,sg2}]. + + + +tc0(_) -> + ct:comment("TO set by group"), + ct:sleep({seconds,5}), + ok. + +tc1() -> + [{timetrap,{timetrap_utils,timetrap_val,[1000]}}]. +tc1(_) -> + ct:comment("TO after 1 sec"), + ct:sleep({seconds,2}), + ok. + +tc2() -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(500) end}]. +tc2(_) -> + ct:comment("TO after 0.5 sec"), + ct:sleep({seconds,2}), + ok. + +tc3(_) -> + ct:comment(io_lib:format("TO after ~w sec", [?TO])), + ct:sleep({seconds,5}), + ok. + +tc4(_) -> + ct:comment("TO set by group"), + ct:sleep({seconds,5}), + ok. + +tc5(_) -> + ct:comment("No TO in this testcase, maybe later"), + ok. diff --git a/lib/common_test/test/ct_group_leader_SUITE.erl b/lib/common_test/test/ct_group_leader_SUITE.erl new file mode 100644 index 0000000000..cde3061d6a --- /dev/null +++ b/lib/common_test/test/ct_group_leader_SUITE.erl @@ -0,0 +1,181 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_system_error_SUITE +%%% +%%% Description: +%%% +%%% Test the group leader functionality in the test_server application. +%%%------------------------------------------------------------------- +-module(ct_group_leader_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config1 = ct_test_support:init_per_suite(Config), + Config1. + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [ + basic + ]. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%%%----------------------------------------------------------------- +%%% +basic(Config) -> + TC = basic, + DataDir = ?config(data_dir, Config), + Suite = filename:join(DataDir, "group_leader_SUITE"), + {Opts,ERPid} = setup([{suite,Suite},{label,TC}], Config), + SuiteLog = execute(TC, Opts, ERPid, Config), + {ok,Data} = file:read_file(SuiteLog), + Lines = binary:split(Data, <<"\n">>, [global]), + {ok,RE} = re:compile("(\\S+):(\\S+)$"), + Cases0 = [begin + {match,[M,F]} = re:run(Case, RE, [{capture,all_but_first,list}]), + {list_to_atom(M),list_to_atom(F)} + end || <<"=case ",Case/binary>> <- Lines], + Cases = [MF || {_,F}=MF <- Cases0, + F =/= init_per_suite, + F =/= end_per_suite, + F =/= init_per_group, + F =/= end_per_group], + io:format("~p\n", [Cases]), + [] = verify_cases(events_to_check(TC), Cases, false), + ok. + +verify_cases([{parallel,P}|Ts], Cases0, Par) -> + Cases = verify_cases(P, Cases0, true), + verify_cases(Ts, Cases, Par); +verify_cases([{?eh,tc_done,{M,F,_}}|Ts], Cases0, false) -> + [{M,F}|Cases] = Cases0, + verify_cases(Ts, Cases, false); +verify_cases([{?eh,tc_done,{M,F,_}}|Ts], Cases0, true) -> + case lists:member({M,F}, Cases0) of + true -> + Cases = Cases0 -- [{M,F}], + verify_cases(Ts, Cases, true); + false -> + io:format("~p not found\n", [{M,F}]), + ?t:fail() + end; +verify_cases([{?eh,_,_}|Ts], Cases, Par) -> + verify_cases(Ts, Cases, Par); +verify_cases([], Cases, _) -> + Cases; +verify_cases([List|Ts], Cases0, Par) when is_list(List) -> + Cases = verify_cases(List, Cases0, false), + verify_cases(Ts, Cases, Par). + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +execute(Name, Opts, ERPid, Config) -> + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(Name, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(Name), + ok = ct_test_support:verify_events(TestEvents, Events, Config), + {event,tc_logfile,_,{_,File}} = + lists:keyfind(tc_logfile, 2, [Ev || {?eh,Ev} <- Events]), + LogDir = filename:dirname(File), + filename:join(LogDir, "suite.log"). + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +%%%----------------------------------------------------------------- + +events_to_check(_Test) -> + [{?eh,tc_done,{group_leader_SUITE,tc1,ok}}, + {parallel,[{?eh,tc_start,{group_leader_SUITE,p1}}, + {?eh,tc_done,{group_leader_SUITE,p1,ok}}, + {?eh,tc_start,{group_leader_SUITE,p2}}, + {?eh,tc_done,{group_leader_SUITE,p2,ok}}]}, + {?eh,tc_done,{group_leader_SUITE,p_restart_my_io_server,ok}}, + {?eh,tc_done,{group_leader_SUITE,p3,ok}}, + {parallel,[ + {?eh,tc_start,{group_leader_SUITE,p10}}, + {?eh,tc_start,{group_leader_SUITE,p11}}, + {?eh,tc_done,{group_leader_SUITE,p10,ok}}, + {?eh,tc_done,{group_leader_SUITE,p11,ok}}, + [{?eh,tc_done,{group_leader_SUITE,s1,ok}}, + {?eh,tc_done,{group_leader_SUITE,s2,ok}}, + {?eh,tc_done,{group_leader_SUITE,s3,ok}}], + {?eh,tc_start,{group_leader_SUITE,p12}}, + {?eh,tc_done,{group_leader_SUITE,p12,ok}}, + [{?eh,tc_done,{group_leader_SUITE,s4,ok}}, + {?eh,tc_done,{group_leader_SUITE,s5,ok}}], + {?eh,tc_start,{group_leader_SUITE,p13}}, + {?eh,tc_done,{group_leader_SUITE,p13,ok}} ]}, + {?eh,tc_done,{group_leader_SUITE,cap1,ok}}, + {?eh,tc_done,{group_leader_SUITE,cap2,ok}}, + {parallel,[{?eh,tc_start,{group_leader_SUITE,cap1}}, + {?eh,tc_done,{group_leader_SUITE,cap1,ok}}, + {?eh,tc_start,{group_leader_SUITE,cap2}}, + {?eh,tc_done,{group_leader_SUITE,cap2,ok}}]}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]. diff --git a/lib/common_test/test/ct_group_leader_SUITE_data/group_leader_SUITE.erl b/lib/common_test/test/ct_group_leader_SUITE_data/group_leader_SUITE.erl new file mode 100644 index 0000000000..3f1844b4ae --- /dev/null +++ b/lib/common_test/test/ct_group_leader_SUITE_data/group_leader_SUITE.erl @@ -0,0 +1,252 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(group_leader_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% @spec suite() -> Info +%% Info = [tuple()] +%% @end +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{seconds,10}}]. + +%%-------------------------------------------------------------------- +%% @spec init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + start_my_io_server(), + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%% @end +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + my_io_server ! die, + ok. + +%%-------------------------------------------------------------------- +%% @spec init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% @end +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} | {fail,Reason} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +end_per_testcase(_TestCase, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%% @end +%%-------------------------------------------------------------------- +groups() -> + [{p,[parallel],[p1,p2]}, + {p_restart,[parallel],[p_restart_my_io_server]}, + {seq,[],[s1,s2,s3]}, + {seq2,[],[s4,s5]}, + {seq_in_par,[parallel],[p10,p11,{group,seq},p12,{group,seq2},p13]}, + {capture_io,[parallel],[cap1,cap2]}]. + +%%-------------------------------------------------------------------- +%% @spec all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +all() -> + [tc1,{group,p},{group,p_restart},p3, + {group,seq_in_par}, + cap1,cap2, + {group,capture_io}]. + +tc1(_C) -> + ok. + +p1(_) -> + %% OTP-10101: + %% + %% External apps/processes started by init_per_suite (common operation), + %% will inherit the group leader of the init_per_suite process, i.e. the + %% test_server test case control process (executing run_test_case_msgloop/7). + %% If, later, a parallel test case triggers the external app to print with + %% e.g. io:format() (also common operation), the calling process will hang! + %% The reason for this is that a parallel test case has a dedicated IO + %% server process, other than the central test case control process. The + %% latter process is not executing run_test_case_msgloop/7 and will not + %% respond to IO messages. The process is still group leader for the + %% external app, however, which is wrong. It's the IO process for the + %% parallel test case that should be group leader - but only for the + %% particular invokation, since other parallel test cases could be + %% invoking the external app too. + print("hej\n"). + +p2(_) -> + print("hopp\n"). + +p_restart_my_io_server(_) -> + %% Restart the IO server and change its group leader. This used + %% to set to the group leader to a process that would soon die. + Ref = erlang:monitor(process, my_io_server), + my_io_server ! die, + receive + {'DOWN',Ref,_,_,_} -> + start_my_io_server() + end. + +p3(_) -> + %% OTP-10125. This would crash since the group leader process + %% for the my_io_server had died. + print("hoppsan\n"). + +print(String) -> + my_io_server ! {print,self(),String}, + receive + {printed,String} -> + ok + end. + +start_my_io_server() -> + Parent = self(), + Pid = spawn(fun() -> my_io_server(Parent) end), + receive + {Pid,started} -> + io:format("~p\n", [process_info(Pid)]), + ok + end. + +my_io_server(Parent) -> + register(my_io_server, self()), + Parent ! {self(),started}, + my_io_server_loop(). + +my_io_server_loop() -> + receive + {print,From,String} -> + io:put_chars(String), + From ! {printed,String}, + my_io_server_loop(); + die -> + ok + end. + +p10(_) -> + receive after 1 -> ok end. + +p11(_) -> + ok. + +p12(_) -> + ok. + +p13(_) -> + ok. + +s1(_) -> + ok. + +s2(_) -> + ok. + +s3(_) -> + ok. + +s4(_) -> + ok. + +s5(_) -> + ok. + +cap1(_) -> + ct:capture_start(), + IO = gen_io(cap1, 10, []), + ct:capture_stop(), + IO = ct:capture_get(), + ok. + +cap2(_) -> + ct:capture_start(), + {Pid,Ref} = spawn_monitor(fun() -> + exit(gen_io(cap2, 42, [])) + end), + receive + {'DOWN',Ref,process,Pid,IO} -> + ct:capture_stop(), + IO = ct:capture_get(), + ok + end. + +gen_io(_, 0, Acc) -> + lists:reverse(Acc); +gen_io(Label, N, Acc) -> + S = lists:flatten(io_lib:format("~s: ~p\n", [Label,N])), + io:put_chars(S), + gen_io(Label, N-1, [S|Acc]). diff --git a/lib/common_test/test/ct_groups_search_SUITE.erl b/lib/common_test/test/ct_groups_search_SUITE.erl new file mode 100644 index 0000000000..6b1c1f4634 --- /dev/null +++ b/lib/common_test/test/ct_groups_search_SUITE.erl @@ -0,0 +1,1245 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: +%%% +%%% Description: +%%% +%%% +%%% The suites used for the test are located in the data directory. +%%% +%%% The group(s) and case(s) are specified according to this: +%%% +%%% Tests = ct_groups:find_groups(Mod, GroupPaths, TestCases, GroupDef) +%%% +%%% GroupPaths = GroupPath | [GroupPath] +%%% GroupPath = atom() | [atom()] +%%% +%%% CT will find all paths that include GroupPath. GroupPath can be a +%%% single group, or a list of groups along the path to TestCases. +%%% If GroupPath is the latter, the last group in the list must be +%%% the "terminating" group in the path, or it will be impossible to +%%% execute test cases in higher level groups *only*, as in this case: +%%% groups() -> [{g1,[],[tc1,{g2,[],[tc2]}]}]. +%%% Compare: find_groups(x, g1, all, groups()), and +%%% find_groups(x, [[g1]], all, groups()) +%%% +%%% Some examples: +%%% +%%% GroupPaths = g1, means find all paths with g1 included +%%% GroupPaths = [g1], -''- +%%% GroupPaths = [g1,g2], search twice - once for g1 and once for g2 +%%% GroupPaths = [[g1,g2]], find cases under group g1 and sub group g2 +%%% GroupPaths = [[g1,g2],[g1,g3]], find cases for g1-g2 AND g1-g3 +%%% +%%% TestCases = all | atom() | [atom()] +%%% +%%%------------------------------------------------------------------- + +-module(ct_groups_search_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/src/ct_util.hrl"). + + +-define(eh, ct_test_support_eh). + +-define(M1, groups_search_dummy_1_SUITE). +-define(M2, groups_search_dummy_2_SUITE). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +init_per_suite(Config) -> + DataDir = proplists:get_value(data_dir, Config), + code:add_patha(DataDir), + M1Erl = filename:join(DataDir, atom_to_list(?M1)++".erl"), + M2Erl = filename:join(DataDir, atom_to_list(?M2)++".erl"), + {ok,?M1} = compile:file(M1Erl, [{outdir,DataDir}]), + {ok,?M2} = compile:file(M2Erl, [{outdir,DataDir}]), + {module,?M1} = code:load_file(?M1), + {module,?M2} = code:load_file(?M2), + + Config1 = ct_test_support:init_per_suite(Config), + Config1. + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +groups() -> + [ + {find_groups,[],[all_groups, + testcases_in_all_groups, + all_in_top_group1, + all_in_top_group2, + all_in_sub_group1, + all_in_sub_group2, + testcase_in_top_group1, + testcase_in_top_group2, + testcase_in_sub_group1, + testcase_in_sub_group2, + testcase_in_top_groups1, + testcase_in_top_groups2, + testcase_in_top_groups3, + testcase_in_top_groups4, + testcase_in_top_groups5, + testcase_in_top_groups6, + testcase_in_top_groups7, + testcase_in_sub_groups1, + testcase_in_sub_groups2, + testcase_in_sub_groups3, + testcase_in_sub_groups4, + testcase_in_sub_groups5, + testcase_in_sub_groups6, + testcase_in_sub_groups7, + testcase_in_sub_groups8, + testcase_in_sub_groups9, + testcase_in_sub_groups10, + testcase_in_sub_groups11, + testcase_in_sub_groups12, + testcase_in_sub_groups13, + bad_testcase_in_sub_groups1]}, + + {run_groups,[sequence],[run_groups_with_options, + run_groups_with_testspec]} + ]. + +all() -> + [{group,find_groups,[parallel]}, + {group,run_groups}]. + + + +%%-------------------------------------------------------------------- +%% TEST CASES CHECKING RETURN VALUE ONLY +%%-------------------------------------------------------------------- + +all_groups(_) -> + GPath = all, TCs = all, + + Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()), + + Top1 = ct_groups:find_groups(?M1, top1, TCs, groups1()), + Top2 = ct_groups:find_groups(?M1, top2, TCs, groups1()), + + All = Top1 ++ Top2 ++ [{conf,[{name,sub2}], + {?M1,init_per_group}, + [{?M1,sub2_tc1},{?M1,sub2_tc2}], + {?M1,end_per_group}}], + + All = Found, + + {?M1,GPath,TCs,Top1++Top2}. + +%%%----------------------------------------------------------------- +%%% +testcases_in_all_groups(_) -> + GPath = all, TCs = [tc3,sub_tc2], + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [Top1 = + {conf,[{name,top1}],{?M2,init_per_group}, + [{?M2,tc3}, + {conf,[{name,sub11}], + {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}], + {?M2,end_per_group}}, + {conf,[{name,sub12}], + {?M2,init_per_group}, + [{?M2,tc3},{?M2,sub_tc2}, + {conf,[{name,sub121}], + {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + Top2 = + {conf,[{name,top2}],{?M2,init_per_group}, + [{?M2,tc3}, + {conf,[{name,sub21}], + {?M2,init_per_group}, + [{?M2,tc3},{?M2,sub_tc2}, + {conf,[{name,sub2xx}], + {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub22}], + {?M2,init_per_group}, + [{?M2,tc3},{?M2,sub_tc2}, + {conf,[{name,sub221}], + {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}], + {?M2,end_per_group}}, + {conf,[{name,sub2xx}], + {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub21}], + {?M2,init_per_group}, + [{?M2,tc3},{?M2,sub_tc2}, + {conf,[{name,sub2xx}], + {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}],{?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub22}], + {?M2,init_per_group}, + [{?M2,tc3},{?M2,sub_tc2}, + {conf,[{name,sub221}], + {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}],{?M2,end_per_group}}, + {conf,[{name,sub2xx}], + {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}],{?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub221}], + {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}],{?M2,end_per_group}}, + + {conf,[{name,sub2xx}], + {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}],{?M2,end_per_group}}] + + = Found, + + {?M2,GPath,TCs,[Top1,Top2]}. + +%%%----------------------------------------------------------------- +%%% +all_in_top_group1(_) -> + GPath= top1, TCs = all, + + Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()), + + [{conf,[{name,top1}], + {?M1,init_per_group}, + [{?M1,top1_tc1},{?M1,top1_tc2}, + {conf,[{name,sub1}], + {?M1,init_per_group}, + [{?M1,sub1_tc1},{?M1,sub1_tc2}], + {?M1,end_per_group}}], + {?M1,end_per_group}}] = Found, + + {?M1,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +all_in_top_group2(_) -> + GPath= top2, TCs = all, + + Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()), + + [{conf,[{name,top2}], + {?M1,init_per_group}, + [{conf,[{name,sub2}], + {?M1,init_per_group}, + [{?M1,sub2_tc1},{?M1,sub2_tc2}], + {?M1,end_per_group}}, + {?M1,top2_tc1},{?M1,top2_tc2}], + {?M1,end_per_group}}] = Found, + + {?M1,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +all_in_sub_group1(_) -> + GPath = sub1, TCs = all, + + Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()), + + [{conf,[{name,top1}], + {?M1,init_per_group}, + [{conf,[{name,sub1}], + {?M1,init_per_group}, + [{?M1,sub1_tc1},{?M1,sub1_tc2}], + {?M1,end_per_group}}], + {?M1,end_per_group}}] = Found, + + {?M1,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +all_in_sub_group2(_) -> + GPath = sub2, TCs = all, + + Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()), + + [Top2 = + {conf,[{name,top2}], + {?M1,init_per_group}, + [{conf,[{name,sub2}], + {?M1,init_per_group}, + [{?M1,sub2_tc1},{?M1,sub2_tc2}], + {?M1,end_per_group}}], + {?M1,end_per_group}}, + + {conf,[{name,sub2}], + {?M1,init_per_group}, + [{?M1,sub2_tc1},{?M1,sub2_tc2}], + {?M1,end_per_group}}] = Found, + + {?M1,GPath,TCs,Top2}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_top_group1(_) -> + GPath = top1, TCs = [top1_tc2], + + Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()), + + [{conf,[{name,top1}], + {?M1,init_per_group}, + [{?M1,top1_tc2}], + {?M1,end_per_group}}] = Found, + + {?M1,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_top_group2(_) -> + GPath = top2, TCs = [top2_tc2], + + Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()), + + [{conf,[{name,top2}], + {?M1,init_per_group}, + [{?M1,top2_tc2}], + {?M1,end_per_group}}] = Found, + + {?M1,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_group1(_) -> + GPath = sub1, TCs = [sub1_tc2], + + Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()), + + [{conf,[{name,top1}], + {?M1,init_per_group}, + [{conf,[{name,sub1}], + {?M1,init_per_group}, + [{?M1,sub1_tc2}], + {?M1,end_per_group}}], + {?M1,end_per_group}}] = Found, + + {?M1,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_group2(_) -> + GPath = sub2, TCs = [sub2_tc2], + + Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()), + + [Top2 = + {conf,[{name,top2}], + {?M1,init_per_group}, + [{conf,[{name,sub2}], + {?M1,init_per_group}, + [{?M1,sub2_tc2}], + {?M1,end_per_group}}], + {?M1,end_per_group}}, + + {conf,[{name,sub2}], + {?M1,init_per_group}, + [{?M1,sub2_tc2}], + {?M1,end_per_group}}] = Found, + + {?M1,GPath,TCs,Top2}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_top_groups1(_) -> + GPath = [top1,top2], TCs = all, + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{?M2,top1_tc1},{?M2,top_tc2},{?M2,tc3}, + {conf,[{name,sub11}], + {?M2,init_per_group}, + [{?M2,sub11_tc1},{?M2,sub_tc2},{?M2,tc3}], + {?M2,end_per_group}}, + {conf,[{name,sub12}], + {?M2,init_per_group}, + [{?M2,sub12_tc1},{?M2,sub_tc2},{?M2,tc3}, + {conf,[{name,sub121}], + {?M2,init_per_group}, + [{?M2,sub121_tc1},{?M2,sub_tc2},{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,top2}], + {?M2,init_per_group}, + [{conf,[{name,sub21}], + {?M2,init_per_group}, + [{?M2,sub21_tc1},{?M2,sub_tc2},{?M2,tc3}, + {conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,sub2xx_tc1},{?M2,sub_tc2},{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + {?M2,top2_tc1},{?M2,top_tc2},{?M2,tc3}, + {conf,[{name,sub22}], + {?M2,init_per_group}, + [{conf,[{name,sub221}], + {?M2,init_per_group}, + [{?M2,sub221_tc1},{?M2,sub_tc2},{?M2,tc3}], + {?M2,end_per_group}}, + {?M2,sub22_tc1},{?M2,sub_tc2},{?M2,tc3}, + {conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,sub2xx_tc1},{?M2,sub_tc2},{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_top_groups2(_) -> + GPath = [top1,top2], TCs = tc3, + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{?M2,tc3}, + {conf,[{name,sub11}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}, + {conf,[{name,sub12}], + {?M2,init_per_group}, + [{?M2,tc3}, + {conf,[{name,sub121}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,top2}], + {?M2,init_per_group}, + [{?M2,tc3}, + {conf,[{name,sub21}], + {?M2,init_per_group}, + [{?M2,tc3}, + {conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub22}], + {?M2,init_per_group}, + [{?M2,tc3}, + {conf,[{name,sub221}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}, + {conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_top_groups3(_) -> + GPath = [top1,top2], TCs = top1_tc1, + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{?M2,top1_tc1}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_top_groups4(_) -> + GPath = [top1,top2], TCs = sub2xx_tc1, + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top2}], + {?M2,init_per_group}, + [{conf,[{name,sub21}], + {?M2,init_per_group}, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,sub2xx_tc1}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + {conf,[{name,sub22}], + {?M2,init_per_group}, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,sub2xx_tc1}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_top_groups5(_) -> + GPath = [top1,top2], TCs = [sub21_tc1,sub22_tc1], + + Found = ct_groups:find_groups(?M2, [top1,top2], [sub21_tc1,sub22_tc1], + groups2()), + + [{conf,[{name,top2}], + {?M2,init_per_group}, + [{conf,[{name,sub21}], + {?M2,init_per_group}, + [{?M2,sub21_tc1}], + {?M2,end_per_group}}, + {conf,[{name,sub22}], + {?M2,init_per_group}, + [{?M2,sub22_tc1}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_top_groups6(_) -> + GPath = [[top1],[top2]], TCs = tc3, + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}, + {conf,[{name,top2}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_top_groups7(_) -> + GPath = [[top1],[top2]], TCs = all, + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{?M2,top1_tc1}, + {?M2,top_tc2}, + {?M2,tc3}], + {?M2,end_per_group}}, + {conf,[{name,top2}], + {?M2,init_per_group}, + [{?M2,top2_tc1}, + {?M2,top_tc2}, + {?M2,tc3}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups1(_) -> + GPath = [sub121], TCs = tc3, + + Found = ct_groups:find_groups(?M2, sub121, tc3, groups2()), + Found = ct_groups:find_groups(?M2, [sub121], tc3, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{conf,[{name,sub12}], + {?M2,init_per_group}, + [{conf,[{name,sub121}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups2(_) -> + GPath = sub12, TCs = tc3, + + Found = ct_groups:find_groups(?M2, sub12, tc3, groups2()), + Found = ct_groups:find_groups(?M2, [sub12], tc3, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{conf,[{name,sub12}], + {?M2,init_per_group}, + [{?M2,tc3}, + {conf,[{name,sub121}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + FoundX = ct_groups:find_groups(?M2, [[sub12]], tc3, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{conf,[{name,sub12}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = FoundX, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups3(_) -> + GPath = [sub121,sub221], TCs = all, + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [Top1 = + {conf,[{name,top1}], + {?M2,init_per_group}, + [{conf,[{name,sub12}], + {?M2,init_per_group}, + [{conf,[{name,sub121}], + {?M2,init_per_group}, + [{?M2,sub121_tc1}, + {?M2,sub_tc2}, + {?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + Top2 = + {conf,[{name,top2}], + {?M2,init_per_group}, + [{conf,[{name,sub22}], + {?M2,init_per_group}, + [{conf,[{name,sub221}], + {?M2,init_per_group}, + [{?M2,sub221_tc1}, + {?M2,sub_tc2}, + {?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub22}], + {?M2,init_per_group}, + [{conf,[{name,sub221}], + {?M2,init_per_group}, + [{?M2,sub221_tc1}, + {?M2,sub_tc2}, + {?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub221}], + {?M2,init_per_group}, + [{?M2,sub221_tc1}, + {?M2,sub_tc2}, + {?M2,tc3}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,[Top1,Top2]}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups4(_) -> + GPath = [top1,sub21], TCs = sub_tc2, + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [Top1 = + {conf,[{name,top1}], + {?M2,init_per_group}, + [{conf,[{name,sub11}], + {?M2,init_per_group}, + [{?M2,sub_tc2}], + {?M2,end_per_group}}, + {conf,[{name,sub12}], + {?M2,init_per_group}, + [{?M2,sub_tc2}, + {conf,[{name,sub121}], + {?M2,init_per_group}, + [{?M2,sub_tc2}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + Top2 = + {conf,[{name,top2}], + {?M2,init_per_group}, + [{conf,[{name,sub21}], + {?M2,init_per_group}, + [{?M2,sub_tc2}, + {conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,sub_tc2}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub21}], + {?M2,init_per_group}, + [{?M2,sub_tc2}, + {conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,sub_tc2}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,[Top1,Top2]}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups5(_) -> + GPath = [[top1,sub12]], TCs = sub12_tc1, + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{conf,[{name,sub12}], + {?M2,init_per_group}, + [{?M2,sub12_tc1}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups6(_) -> + GPath = [[top1,sub12]], TCs = [sub_tc2], + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{conf,[{name,sub12}], + {?M2,init_per_group}, + [{?M2,sub_tc2}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups7(_) -> + GPath = [[top1,sub12]], TCs = [sub12_tc1,sub_tc2], + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{conf,[{name,sub12}], + {?M2,init_per_group}, + [{?M2,sub12_tc1}, + {?M2,sub_tc2}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups8(_) -> + GPath = [[top2,sub22]], TCs = [sub22_tc1,sub_tc2], + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top2}], + {?M2,init_per_group}, + [{conf,[{name,sub22}], + {?M2,init_per_group}, + [{?M2,sub22_tc1}, + {?M2,sub_tc2}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups9(_) -> + GPath = [[sub2xx]], TCs = tc3, + + Found = ct_groups:find_groups(?M2, sub2xx, tc3, groups2()), + Found = ct_groups:find_groups(?M2, [[sub2xx]], tc3, groups2()), + + [Top2 = + {conf,[{name,top2}], + {?M2,init_per_group}, + [{conf,[{name,sub21}], + {?M2,init_per_group}, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + {conf,[{name,sub22}], + {?M2,init_per_group}, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub21}], + {?M2,init_per_group}, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub22}], + {?M2,init_per_group}, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Top2}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups10(_) -> + GPath = [[sub22,sub2xx]], TCs = tc3, + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [Top2 = + {conf,[{name,top2}], + {?M2,init_per_group}, + [{conf,[{name,sub22}], + {?M2,init_per_group}, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub22}], + {?M2,init_per_group}, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Top2}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups11(_) -> + GPath = [[top1,sub12,sub121]], TCs = all, + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{conf,[{name,sub12}], + {?M2,init_per_group}, + [{conf,[{name,sub121}], + {?M2,init_per_group}, + [{?M2,sub121_tc1}, + {?M2,sub_tc2}, + {?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups12(_) -> + GPath = [[top2,sub2xx]], TCs = [sub2xx_tc1,tc3], + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top2}], + {?M2,init_per_group}, + [{conf,[{name,sub21}], + {?M2,init_per_group}, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,sub2xx_tc1}, + {?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + {conf,[{name,sub22}], + {?M2,init_per_group}, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,sub2xx_tc1}, + {?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups13(_) -> + GPath = [[top2,sub22,sub2xx]], TCs = [top2_tc1,sub2xx_tc1,tc3], + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top2}], + {?M2,init_per_group}, + [{conf,[{name,sub22}], + {?M2,init_per_group}, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,sub2xx_tc1}, + {?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +bad_testcase_in_sub_groups1(_) -> + GPath = [sub2xx], TCs = [top2_tc1], + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +bad_testcase_in_sub_groups2(_) -> + GPath = [sub12,sub2xx], TCs = [top1_tc1,top2_tc1], + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% CASES EXECUTING THE TESTS +%%%----------------------------------------------------------------- + +run_groups_with_options(Config) -> + DataDir = ?config(data_dir, Config), + + {M1All,M1Rest,M2All,M2Rest} = get_all_groups_and_cases(Config), + + M1AllGrs = lists:flatmap(fun({Path,_,_}) when is_atom(hd(Path)) -> Path; + ({Path,_,_}) when is_list(hd(Path)) -> Path; + ({Path,_,_}) -> [Path] + end, M1All), + + %% ct:pal("NOW RUNNING M1 TEST: ~p", [M1All]), + + {OptsM11,ERPidM11} = setup([{dir,DataDir},{suite,?M1}, + {group,M1AllGrs},{label,m1_all_cases}], Config), + M1AllGrInfo = {M1AllGrs,lists:flatten([Found || {_,_,Found} <- M1All])}, + ok = execute(m1_all_cases, M1AllGrInfo, OptsM11, ERPidM11, Config), + + lists:foldl( + fun({GrPath,TCs,Found}, N) -> + TestName = list_to_atom("m1_spec_cases_" ++ integer_to_list(N)), + %% ct:pal("NOW RUNNING M1 TEST ~p: ~p + ~p", + %% [TestName,GrPath,TCs]), + {OptsM12,ERPidM12} = setup([{dir,DataDir},{suite,?M1}, + {group,GrPath},{testcase,TCs}, + {label,TestName}], Config), + ok = execute(TestName, {GrPath,TCs,Found}, + OptsM12, ERPidM12, Config), + N+1 + end, 1, M1Rest), + + %% ct:pal("NOW RUNNING M2 TEST: ~p", [M2All]), + + M2AllGrs = lists:flatmap(fun({Path,_,_}) when is_atom(hd(Path)) -> Path; + ({Path,_,_}) when is_list(hd(Path)) -> Path; + ({Path,_,_}) -> [Path] + end, M2All), + + + {OptsM21,ERPidM21} = setup([{dir,DataDir},{suite,?M2}, + {group,M2AllGrs},{testcase,all}, + {label,m2_all_cases}], Config), + M2AllGrInfo = {M2AllGrs,lists:flatten([Found || {_,_,Found} <- M2All])}, + ok = execute(m2_all_cases, M2AllGrInfo, OptsM21, ERPidM21, Config), + + lists:foldl( + fun({GrPath,TCs,Found}, N) -> + TestName = list_to_atom("m2_spec_cases_" ++ integer_to_list(N)), + %% ct:pal("NOW RUNNING M2 TEST ~p: ~p + ~p", [TestName,GrPath,TCs]), + {OptsM22,ERPidM22} = setup([{dir,DataDir},{suite,?M2}, + {group,GrPath},{testcase,TCs}, + {label,TestName}], Config), + ok = execute(TestName, {GrPath,TCs,Found}, + OptsM22, ERPidM22, Config), + N+1 + end, 1, M2Rest), + ok. + + +%%%----------------------------------------------------------------- +%%% +run_groups_with_testspec(Config) -> + Name = run_groups_with_testspec, + DataDir = ?config(data_dir, Config), + PrivDir = ?config(priv_dir, Config), + + {M1All,M1Rest,M2All,M2Rest} = get_all_groups_and_cases(Config), + + M1AllGrs = lists:flatmap(fun({Path,_,_}) when is_atom(hd(Path)) -> Path; + ({Path,_,_}) when is_list(hd(Path)) -> Path; + ({Path,_,_}) -> [Path] + end, M1All), + M1AllTerm = {groups,DataDir,?M1,M1AllGrs}, + + M1RestTerms = lists:map( + fun({GrPath,TCs,_}) -> + {groups,DataDir,?M1,GrPath,{cases,TCs}} + end, M1Rest), + + M2AllGrs = lists:flatmap(fun({Path,_,_}) when is_atom(hd(Path)) -> Path; + ({Path,_,_}) when is_list(hd(Path)) -> Path; + ({Path,_,_}) -> [Path] + end, M2All), + M2AllTerm = {groups,DataDir,?M2,M2AllGrs,{cases,all}}, + + M2RestTerms = lists:map( + fun({GrPath,TCs,_}) -> + {groups,DataDir,?M2,GrPath,{cases,TCs}} + end, M2Rest), + + GroupTerms = lists:flatten([M1AllTerm, + M1RestTerms, + M2AllTerm, + M2RestTerms]), + + TestSpec = [{merge_tests,false}, + {label,Name}] ++ GroupTerms, + + ct:pal("Here's the test spec:~n~p", [TestSpec]), + + TestSpecName = ct_test_support:write_testspec(TestSpec, PrivDir, + "groups_search_spec"), + + {Opts,ERPid} = setup([{spec,TestSpecName}], Config), + GroupInfo = + [{M1AllTerm,lists:flatten([Found || {_,_,Found} <- M1All])} | + M1Rest] ++ + [{M2AllTerm,lists:flatten([Found || {_,_,Found} <- M2All])} | + M2Rest], + ok = execute(Name, GroupInfo, Opts, ERPid, Config). + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- + +groups1() -> + [{top1,[],[top1_tc1,top1_tc2,{sub1,[],[sub1_tc1,sub1_tc2]}]}, + {top2,[],[{group,sub2},top2_tc1,top2_tc2]}, + {sub2,[],[sub2_tc1,sub2_tc2]}]. + +groups2() -> + [{top1,[],[top1_tc1,top_tc2,tc3, + {sub11,[],[sub11_tc1,sub_tc2,tc3]}, + {sub12,[],[sub12_tc1,sub_tc2,tc3, + {sub121,[],[sub121_tc1,sub_tc2,tc3]}]}]}, + {top2,[],[{group,sub21},top2_tc1,top_tc2,tc3,{group,sub22}]}, + {sub21,[],[sub21_tc1,sub_tc2,tc3,{group,sub2xx}]}, + {sub22,[],[{group,sub221},sub22_tc1,sub_tc2,tc3,{group,sub2xx}]}, + {sub221,[],[sub221_tc1,sub_tc2,tc3]}, + {sub2xx,[],[sub2xx_tc1,sub_tc2,tc3]}]. + +get_all_groups_and_cases(Config) -> + {value,{_,_,FindGrTCs}} = lists:keysearch(find_groups, 1, groups()), + + MGTFs = [apply(?MODULE, TC, [Config]) || TC <- FindGrTCs], + + ct:pal("Extracted data from ~p test cases", [length(MGTFs)]), + + lists:foldr(fun({M,Gs,TCs,F}, + {M11,M12,M21,M22}) -> + case {M,Gs,TCs} of + {?M1,all,_} -> {M11,[{Gs,TCs,F}|M12],M21,M22}; + {?M1,_,all} -> {[{Gs,all,F}|M11],M12,M21,M22}; + {?M1,_,_} -> {M11,[{Gs,TCs,F}|M12],M21,M22}; + {?M2,all,_} -> {M11,M12,M21,[{Gs,TCs,F}|M22]}; + {?M2,_,all} -> {M11,M12,[{Gs,all,F}|M21],M22}; + {?M2,_,_} -> {M11,M12,M21,[{Gs,TCs,F}|M22]} + end + end, {[],[],[],[]}, MGTFs). + +%%%----------------------------------------------------------------- + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +execute(Name, TestParams, Opts, ERPid, Config) -> + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + Events1 = reformat(Events, ?eh), + ct_test_support:log_events(Name, + Events1, + ?config(priv_dir, Config), + Opts), + verify_events(Name, TestParams, Events1). + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +verify_events(Name, Params, Events) -> + %% 2 tests (ct:run_test + script_start) is default + verify_events(Name, Params, Events, 2). + +verify_events(_, _, _, 0) -> + ok; +verify_events(Name, Params, Events, N) -> + test_events(Name, Params, Events), + verify_events(Name, Params, Events, N-1). + +%%%----------------------------------------------------------------- +%%% check run_groups_with_options + +test_events(TestName, {GrPath,Found}, Events) -> + test_events(TestName, {GrPath,all,Found}, Events); + +test_events(TestName, {GrPath,TCs,Found}, Events) + when TestName /= run_groups_with_testspec -> + try check_events(Events, flatten_tests(Found)) of + ok -> ok + catch + throw:Reason -> + ct:pal("Test failed for ~p with group path ~p and cases ~p" + "~nReason: ~p", [TestName,GrPath,TCs,Reason]), + throw(failed) + end; + +%%%----------------------------------------------------------------- +%%% check run_groups_with_testspec + +test_events(run_groups_with_testspec, Params, Events) -> + AllFound = lists:flatmap(fun({_All,Found}) when is_tuple(Found) -> + [Found]; + ({_All,Found}) -> + Found; + ({_Gr,_TCs,Found}) when is_tuple(Found) -> + [Found]; + ({_Gr,_TCs,Found}) -> + Found + end, Params), + try check_events(Events, flatten_tests(AllFound)) of + ok -> ok + catch + throw:Reason -> + ct:pal("Test failed for run_groups_with_testspec." + "~nReason: ~p", [Reason]), + throw(failed) + end. + +flatten_tests({conf,[{name,G}|_],{Mod,_I},Tests,_E}) -> + lists:flatten([{group,Mod,G} | flatten_tests(Tests)]); +flatten_tests([{conf,[{name,G}|_],{Mod,_I},Tests,_E} | Confs]) -> + lists:flatten([{group,Mod,G} | flatten_tests(Tests)]) ++ + lists:flatten(flatten_tests(Confs)); +flatten_tests([{_Mod,_TC} = Case | Tests]) -> + lists:flatten([Case | flatten_tests(Tests)]); +flatten_tests([]) -> + []. + +check_events([{_,tc_start,{Mod,{init_per_group,G,_}}} | Evs], + [{group,Mod,G} | Check]) -> + check_events(Evs, Check); +check_events([{_,tc_start,{Mod,TC}} | Evs], + [{Mod,TC} | Check]) when is_atom(TC) -> + check_events(Evs, Check); +check_events([{_,tc_start,{Mod,{init_per_group,G,_}}} | _Evs], Check) -> + ct:pal("CHECK FAILED!~nGroup ~p in ~p not found in ~p.", + [G,Mod,Check]), + throw({test_not_found,{Mod,G}}); +check_events([{_,tc_start,{Mod,TC}} | _Evs], Check) + when is_atom(TC), TC /= init_per_suite, TC /= end_per_suite -> + ct:pal("CHECK FAILED!~nCase ~p in ~p not found in ~p.", + [TC,Mod,Check]), + throw({test_not_found,{Mod,TC}}); +check_events([Group | Evs], Check) when is_list(Group) -> + Check1 = check_events(Group, Check), + check_events(Evs, Check1); +check_events(_, []) -> + ok; +check_events([Elem | Evs], Check) when is_tuple(Elem) -> + check_events(Evs, Check); +check_events([], Check = [_|_]) -> + ct:pal("CHECK FAILED!~nTests remain: ~p", [Check]), + throw({tests_remain,Check}); +check_events([Wut | _],_) -> + throw({unexpected,Wut}). + diff --git a/lib/common_test/test/ct_groups_search_SUITE_data/groups_search_dummy_1_SUITE.erl b/lib/common_test/test/ct_groups_search_SUITE_data/groups_search_dummy_1_SUITE.erl new file mode 100644 index 0000000000..1c5b572f92 --- /dev/null +++ b/lib/common_test/test/ct_groups_search_SUITE_data/groups_search_dummy_1_SUITE.erl @@ -0,0 +1,83 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(groups_search_dummy_1_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + + +all() -> + [{group,top1}, + {group,top2}]. + +groups() -> + [{top1,[],[top1_tc1,top1_tc2,{sub1,[],[sub1_tc1,sub1_tc2]}]}, + {top2,[],[{group,sub2},top2_tc1,top2_tc2]}, + {sub2,[],[sub2_tc1,sub2_tc2]}]. + +%%%----------------------------------------------------------------- +%%% CONFIG FUNCS +%%%----------------------------------------------------------------- + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +init_per_group(_, Config) -> + Config. + +end_per_group(_, _Config) -> + ok. + +%%%----------------------------------------------------------------- +%%% TEST CASES +%%%----------------------------------------------------------------- + +top1_tc1(_) -> + ok. + +top1_tc2(_) -> + ok. + +sub1_tc1(_) -> + ok. + +sub1_tc2(_) -> + ok. + +top2_tc1(_) -> + ok. + +top2_tc2(_) -> + ok. + +sub2_tc1(_) -> + ok. + +sub2_tc2(_) -> + ok. diff --git a/lib/common_test/test/ct_groups_search_SUITE_data/groups_search_dummy_2_SUITE.erl b/lib/common_test/test/ct_groups_search_SUITE_data/groups_search_dummy_2_SUITE.erl new file mode 100644 index 0000000000..060012de29 --- /dev/null +++ b/lib/common_test/test/ct_groups_search_SUITE_data/groups_search_dummy_2_SUITE.erl @@ -0,0 +1,102 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(groups_search_dummy_2_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + + +all() -> + [{group,top1}, + {group,top2}]. + +groups() -> + [{top1,[],[top1_tc1,top_tc2,tc3, + {sub11,[],[sub11_tc1,sub_tc2,tc3]}, + {sub12,[],[sub12_tc1,sub_tc2,tc3, + {sub121,[],[sub121_tc1,sub_tc2,tc3]}]}]}, + + {top2,[],[{group,sub21},top2_tc1,top_tc2,tc3,{group,sub22}]}, + {sub21,[],[sub21_tc1,sub_tc2,tc3,{group,sub2xx}]}, + {sub22,[],[{group,sub221},sub22_tc1,sub_tc2,tc3,{group,sub2xx}]}, + {sub221,[],[sub221_tc1,sub_tc2,tc3]}, + {sub2xx,[],[sub2xx_tc1,sub_tc2,tc3]}]. + +%%%----------------------------------------------------------------- +%%% CONFIG FUNCS +%%%----------------------------------------------------------------- + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +init_per_group(_, Config) -> + Config. + +end_per_group(_, _Config) -> + ok. + +%%%------------------------------------------------------------------ +%%% TEST CASES +%%%------------------------------------------------------------------ + +top1_tc1(_) -> + ok. + +top_tc2(_) -> + ok. + +tc3(_) -> + ok. + +sub_tc2(_) -> + ok. + +sub11_tc1(_) -> + ok. + +sub12_tc1(_) -> + ok. + +sub121_tc1(_) -> + ok. + +top2_tc1(_) -> + ok. + +sub21_tc1(_) -> + ok. + +sub22_tc1(_) -> + ok. + +sub221_tc1(_) -> + ok. + +sub2xx_tc1(_) -> + ok. diff --git a/lib/common_test/test/ct_master_SUITE.erl b/lib/common_test/test/ct_master_SUITE.erl index 27243a0067..56a343a96f 100644 --- a/lib/common_test/test/ct_master_SUITE.erl +++ b/lib/common_test/test/ct_master_SUITE.erl @@ -117,14 +117,8 @@ ct_master_test(Config) when is_list(Config) -> reformat(Events, ?eh), PrivDir, []), - find_events(NodeNames, [{tc_start,{master_SUITE,init_per_suite}}, - {tc_start,{master_SUITE,first_testcase}}, - {tc_start,{master_SUITE,second_testcase}}, - {tc_start,{master_SUITE,third_testcase}}, - {tc_start,{master_SUITE,end_per_suite}}], - Events), - - ok. + TestEvents = events_to_check(ct_master_test), + ok = find_events(NodeNames, TestEvents, Events, Config). %%%----------------------------------------------------------------- %%% HELP FUNCTIONS @@ -153,13 +147,15 @@ make_spec(DataDir, FileName, NodeNames, Suites, Config) -> CM = [{config,master,filename:join(DataDir,"master/config.txt")}], + Env = [{"THIS_MUST_BE_SET","yes"}, + {"SO_MUST_THIS","value"}], NS = lists:map( fun(NodeName) -> {init,NodeName,[ {node_start,[{startup_functions,[]}, - {monitor_master,true}]}, - {eval,{erlang,nodes,[]}} - ] + {monitor_master,true}, + {env,Env}]}, + {eval,{erlang,nodes,[]}}] } end, NodeNames), @@ -199,7 +195,6 @@ run_test(_Name, FileName, Config) -> [{FileName,ok}] = ct_test_support:run({ct_master,run,[FileName]}, [{ct_master,basic_html,[true]}], Config), - timer:sleep(5000), [{FileName,ok}] = ct_test_support:run({ct_master,run,[FileName]}, [{ct_master,basic_html,[false]}], Config). @@ -210,28 +205,26 @@ reformat(Events, EH) -> %%%----------------------------------------------------------------- %%% TEST EVENTS %%%----------------------------------------------------------------- -find_events([], _CheckEvents, _) -> - ok; -find_events([NodeName|NodeNames],CheckEvents,AllEvents) -> - find_events(NodeNames, CheckEvents, - remove_events(add_host(NodeName),CheckEvents, AllEvents, [])). - -remove_events(Node,[{Name,Data} | RestChecks], - [{?eh,#event{ name = Name, node = Node, data = Data }}|RestEvs], - Acc) -> - remove_events(Node, RestChecks, RestEvs, Acc); -remove_events(Node, Checks, [Event|RestEvs], Acc) -> - remove_events(Node, Checks, RestEvs, [Event | Acc]); -remove_events(_Node, [], [], Acc) -> - lists:reverse(Acc); -remove_events(Node, Events, [], Acc) -> - test_server:format("Could not find events: ~p in ~p for node ~p", - [Events, lists:reverse(Acc), Node]), - exit(event_not_found). + +find_events(NodeNames, TestEvents, Events, Config) -> + [begin + Node = add_host(Node0), + io:format("Searching for events for node: ~s", [Node]), + ok = ct_test_support:verify_events(TestEvents, Events, Node, Config), + io:nl() + end || Node0 <- NodeNames], + ok. add_host(NodeName) -> {ok, HostName} = inet:gethostname(), list_to_atom(atom_to_list(NodeName)++"@"++HostName). -expected_events(_) -> - []. +events_to_check(_) -> + [{?eh,tc_start,{master_SUITE,first_testcase}}, + {?eh,tc_done,{master_SUITE,first_testcase,ok}}, + {?eh,tc_start,{master_SUITE,second_testcase}}, + {?eh,tc_done,{master_SUITE,second_testcase,ok}}, + {?eh,tc_start,{master_SUITE,third_testcase}}, + {?eh,tc_done,{master_SUITE,third_testcase,ok}}, + {?eh,tc_start,{master_SUITE,env_vars}}, + {?eh,tc_done,{master_SUITE,env_vars,ok}}]. diff --git a/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl b/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl index 032d69ad9f..8a5009ad62 100644 --- a/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl +++ b/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl @@ -39,7 +39,8 @@ init_per_suite(Config) -> end_per_suite(_) -> ok. -all() -> [first_testcase, second_testcase, third_testcase]. +all() -> [first_testcase, second_testcase, third_testcase, + env_vars]. init_per_testcase(_, Config) -> Config. @@ -56,3 +57,9 @@ second_testcase(_)-> third_testcase(_)-> A = 4, A = 2*2. + +env_vars(_) -> + io:format("~p\n", [os:getenv()]), + "yes" = os:getenv("THIS_MUST_BE_SET"), + "value" = os:getenv("SO_MUST_THIS"), + ok. diff --git a/lib/common_test/test/ct_netconfc_SUITE.erl b/lib/common_test/test/ct_netconfc_SUITE.erl index e6e8d5b09c..3042a924fe 100644 --- a/lib/common_test/test/ct_netconfc_SUITE.erl +++ b/lib/common_test/test/ct_netconfc_SUITE.erl @@ -44,7 +44,12 @@ %%-------------------------------------------------------------------- init_per_suite(Config) -> Config1 = ct_test_support:init_per_suite(Config), - Config1. + case application:load(crypto) of + {error,Reason} -> + {skip, Reason}; + _ -> + Config1 + end. end_per_suite(Config) -> ct_test_support:end_per_suite(Config). @@ -108,7 +113,7 @@ reformat(Events, EH) -> %%%----------------------------------------------------------------- %%% TEST EVENTS %%%----------------------------------------------------------------- -events_to_check(Test,Config) -> +events_to_check(default,Config) -> {module,_} = code:load_abs(filename:join(?config(data_dir,Config), netconfc1_SUITE)), TCs = netconfc1_SUITE:all(), diff --git a/lib/common_test/test/ct_surefire_SUITE.erl b/lib/common_test/test/ct_surefire_SUITE.erl new file mode 100644 index 0000000000..69e98cef48 --- /dev/null +++ b/lib/common_test/test/ct_surefire_SUITE.erl @@ -0,0 +1,351 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_surefire_SUITE +%%% +%%% Description: +%%% Test cth_surefire hook +%%% +%%%------------------------------------------------------------------- +-module(ct_surefire_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-include_lib("xmerl/include/xmerl.hrl"). + +-define(eh, ct_test_support_eh). + +-define(url_base,"http://my.host.com/"). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config1 = ct_test_support:init_per_suite(Config), + Config1. + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [ + default, + absolute_path, + relative_path, + url, + logdir + ]. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%%%----------------------------------------------------------------- +%%% +default(Config) when is_list(Config) -> + run(default,[cth_surefire],Config), + PrivDir = ?config(priv_dir,Config), + XmlRe = filename:join([PrivDir,"*","junit_report.xml"]), + check_xml(default,XmlRe). + +absolute_path(Config) when is_list(Config) -> + PrivDir = ?config(priv_dir,Config), + Path = filename:join(PrivDir,"abspath.xml"), + run(absolute_path,[{cth_surefire,[{path,Path}]}],Config), + check_xml(absolute_path,Path). + +relative_path(Config) when is_list(Config) -> + Path = "relpath.xml", + run(relative_path,[{cth_surefire,[{path,Path}]}],Config), + PrivDir = ?config(priv_dir,Config), + XmlRe = filename:join([PrivDir,"*",Path]), + check_xml(relative_path,XmlRe). + +url(Config) when is_list(Config) -> + Path = "url.xml", + run(url,[{cth_surefire,[{url_base,?url_base}, + {path,Path}]}],Config), + PrivDir = ?config(priv_dir,Config), + XmlRe = filename:join([PrivDir,"*",Path]), + check_xml(url,XmlRe). + +logdir(Config) when is_list(Config) -> + PrivDir = ?config(priv_dir,Config), + LogDir = filename:join(PrivDir,"specific_logdir"), + file:make_dir(LogDir), + Path = "logdir.xml", + run(logdir,[{cth_surefire,[{path,Path}]}],Config,[{logdir,LogDir}]), + PrivDir = ?config(priv_dir,Config), + XmlRe = filename:join([LogDir,"*",Path]), + check_xml(logdir,XmlRe). + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- +run(Case,CTHs,Config) -> + run(Case,CTHs,Config,[]). +run(Case,CTHs,Config,ExtraOpts) -> + DataDir = ?config(data_dir, Config), + Suite = filename:join(DataDir, "surefire_SUITE"), + {Opts,ERPid} = setup([{suite,Suite},{ct_hooks,CTHs},{label,Case}|ExtraOpts], + Config), + ok = execute(Case, Opts, ERPid, Config). + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Opts1 = + case lists:keymember(logdir,1,Test) of + true -> lists:keydelete(logdir,1,Opts0); + false -> Opts0 + end, + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts1 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +execute(Name, Opts, ERPid, Config) -> + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(Name, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(Name), + ct_test_support:verify_events(TestEvents, Events, Config). + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +%%%----------------------------------------------------------------- +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). + +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + test_events(Test) ++ events_to_check(Test, N-1). + +test_events(_) -> + [{?eh,start_logging,'_'}, + {?eh,start_info,{1,1,9}}, + {?eh,tc_start,{surefire_SUITE,init_per_suite}}, + {?eh,tc_done,{surefire_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{surefire_SUITE,tc_ok}}, + {?eh,tc_done,{surefire_SUITE,tc_ok,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{surefire_SUITE,tc_fail}}, + {?eh,tc_done,{surefire_SUITE,tc_fail, + {failed,{error,{test_case_failed,"this test should fail"}}}}}, + {?eh,test_stats,{1,1,{0,0}}}, + {?eh,tc_start,{surefire_SUITE,tc_skip}}, + {?eh,tc_done,{surefire_SUITE,tc_skip,{skipped,"this test is skipped"}}}, + {?eh,test_stats,{1,1,{1,0}}}, + {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, + {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, + {skipped,{require_failed,'_'}}}}, + {?eh,test_stats,{1,1,{1,1}}}, + [{?eh,tc_start,{surefire_SUITE,{init_per_group,g,[]}}}, + {?eh,tc_done,{surefire_SUITE,{init_per_group,g,[]},ok}}, + {?eh,tc_start,{surefire_SUITE,tc_ok}}, + {?eh,tc_done,{surefire_SUITE,tc_ok,ok}}, + {?eh,test_stats,{2,1,{1,1}}}, + {?eh,tc_start,{surefire_SUITE,tc_fail}}, + {?eh,tc_done,{surefire_SUITE,tc_fail, + {failed,{error,{test_case_failed,"this test should fail"}}}}}, + {?eh,test_stats,{2,2,{1,1}}}, + {?eh,tc_start,{surefire_SUITE,tc_skip}}, + {?eh,tc_done,{surefire_SUITE,tc_skip,{skipped,"this test is skipped"}}}, + {?eh,test_stats,{2,2,{2,1}}}, + {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, + {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, + {skipped,{require_failed,'_'}}}}, + {?eh,test_stats,{2,2,{2,2}}}, + {?eh,tc_start,{surefire_SUITE,{end_per_group,g,[]}}}, + {?eh,tc_done,{surefire_SUITE,{end_per_group,g,[]},ok}}], + [{?eh,tc_start,{surefire_SUITE,{init_per_group,g_fail,[]}}}, + {?eh,tc_done,{surefire_SUITE,{init_per_group,g_fail,[]}, + {failed,{error,all_cases_should_be_skipped}}}}, + {?eh,tc_auto_skip,{surefire_SUITE,tc_ok, + {failed, + {surefire_SUITE,init_per_group, + {'EXIT',all_cases_should_be_skipped}}}}}, + {?eh,test_stats,{2,2,{2,3}}}, + {?eh,tc_auto_skip,{surefire_SUITE,end_per_group, + {failed, + {surefire_SUITE,init_per_group, + {'EXIT',all_cases_should_be_skipped}}}}}], + {?eh,tc_start,{surefire_SUITE,end_per_suite}}, + {?eh,tc_done,{surefire_SUITE,end_per_suite,ok}}, + {?eh,stop_logging,[]}]. + + +%%%----------------------------------------------------------------- +%%% Check generated xml log files +check_xml(Case,XmlRe) -> + case filelib:wildcard(XmlRe) of + [] -> + ct:fail("No xml files found with regexp ~p~n", [XmlRe]); + [_] = Xmls when Case==absolute_path -> + do_check_xml(Case,Xmls); + [_,_] = Xmls -> + do_check_xml(Case,Xmls) + end. + +%% Allowed structure: +%% <testsuites> +%% <testsuite> +%% <properties> +%% <property/> +%% ... +%% </properties> +%% <testcase> +%% [<failure/> | <error/> | <skipped/> ] +%% </testcase> +%% ... +%% </testsuite> +%% ... +%% </testsuites> +do_check_xml(Case,[Xml|Xmls]) -> + ct:log("Checking <a href=~p>~s</a>~n",[Xml,Xml]), + {E,_} = xmerl_scan:file(Xml), + Expected = events_to_result(lists:flatten(test_events(Case))), + ParseResult = testsuites(Case,E), + ct:log("Expecting: ~p~n",[[Expected]]), + ct:log("Actual : ~p~n",[ParseResult]), + [Expected] = ParseResult, + do_check_xml(Case,Xmls); +do_check_xml(_,[]) -> + ok. + +%% Scanning the XML to get the same type of result as events_to_result/1 +testsuites(Case,#xmlElement{name=testsuites,content=TS}) -> + %% OTP-10589 - move properties element to <testsuite> + false = lists:keytake(properties,#xmlElement.name,TS), + testsuite(Case,TS). + +testsuite(Case,[#xmlElement{name=testsuite,content=TC,attributes=A}|TS]) -> + {ET,EF,ES} = events_to_numbers(lists:flatten(test_events(Case))), + {T,E,F,S} = get_numbers_from_attrs(A,false,false,false,false), + ct:log("Expecting total:~p, error:~p, failure:~p, skipped:~p~n",[ET,0,EF,ES]), + ct:log("Actual total:~p, error:~p, failure:~p, skipped:~p~n",[T,E,F,S]), + {ET,0,EF,ES} = {T,E,F,S}, + + %% properties should only be there if given a options to hook + false = lists:keytake(properties,#xmlElement.name,TC), + %% system-out and system-err is not used by common_test + false = lists:keytake('system-out',#xmlElement.name,TC), + false = lists:keytake('system-err',#xmlElement.name,TC), + R=testcase(Case,TC), + [R|testsuite(Case,TS)]; +testsuite(_Case,[]) -> + []. + +testcase(url=Case,[#xmlElement{name=testcase,attributes=A,content=C}|TC]) -> + R = failed_or_skipped(C), + case R of + [s] -> + case lists:keyfind(url,#xmlAttribute.name,A) of + false -> ok; + #xmlAttribute{value=UrlAttr} -> + lists:keyfind(url,#xmlAttribute.name,A), + true = lists:prefix(?url_base,UrlAttr) + end; + _ -> + #xmlAttribute{value=UrlAttr} = + lists:keyfind(url,#xmlAttribute.name,A), + true = lists:prefix(?url_base,UrlAttr) + end, + [R|testcase(Case,TC)]; +testcase(Case,[#xmlElement{name=testcase,attributes=A,content=C}|TC]) -> + false = lists:keyfind(url,#xmlAttribute.name,A), + R = failed_or_skipped(C), + [R|testcase(Case,TC)]; +testcase(_Case,[]) -> + []. + +failed_or_skipped([#xmlElement{name=failure}|E]) -> + [f|failed_or_skipped(E)]; +failed_or_skipped([#xmlElement{name=error}|E]) -> + [e|failed_or_skipped(E)]; +failed_or_skipped([#xmlElement{name=skipped}|E]) -> + [s|failed_or_skipped(E)]; +failed_or_skipped([]) -> + []. + +%% Using the expected events to produce the expected result of the XML scanning. +%% The result is a list of test suites: +%% Testsuites = [Testsuite] +%% Testsuite = [Testcase] +%% Testcase = [] | [f] | [s], indicating ok, failed and skipped respectively +events_to_result([{?eh,tc_done,{_Suite,_Case,R}}|E]) -> + [result(R)|events_to_result(E)]; +events_to_result([{?eh,tc_auto_skip,_}|E]) -> + [[s]|events_to_result(E)]; +events_to_result([_|E]) -> + events_to_result(E); +events_to_result([]) -> + []. + +result(ok) ->[]; +result({skipped,_}) -> [s]; +result({failed,_}) -> [f]. + +%% Using the expected events' last test_stats element to produce the +%% expected number of totla, errors, failed and skipped testcases. +events_to_numbers(E) -> + RevE = lists:reverse(E), + {?eh,test_stats,{Ok,F,{US,AS}}} = lists:keyfind(test_stats,2,RevE), + {Ok+F+US+AS,F,US+AS}. + +get_numbers_from_attrs([#xmlAttribute{name=tests,value=X}|A],false,E,F,S) -> + get_numbers_from_attrs(A,list_to_integer(X),E,F,S); +get_numbers_from_attrs([#xmlAttribute{name=errors,value=X}|A],T,false,F,S) -> + get_numbers_from_attrs(A,T,list_to_integer(X),F,S); +get_numbers_from_attrs([#xmlAttribute{name=failures,value=X}|A],T,E,false,S) -> + get_numbers_from_attrs(A,T,E,list_to_integer(X),S); +get_numbers_from_attrs([#xmlAttribute{name=skipped,value=X}|A],T,E,F,false) -> + get_numbers_from_attrs(A,T,E,F,list_to_integer(X)); +get_numbers_from_attrs([_|A],T,E,F,S) -> + get_numbers_from_attrs(A,T,E,F,S); +get_numbers_from_attrs([],T,E,F,S) -> + {T,E,F,S}. diff --git a/lib/common_test/test/ct_surefire_SUITE_data/surefire_SUITE.erl b/lib/common_test/test/ct_surefire_SUITE_data/surefire_SUITE.erl new file mode 100644 index 0000000000..677aee46c5 --- /dev/null +++ b/lib/common_test/test/ct_surefire_SUITE_data/surefire_SUITE.erl @@ -0,0 +1,92 @@ +%%-------------------------------------------------------------------- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%%---------------------------------------------------------------------- +%% File: surefire_SUITE.erl +%% +%% Description: +%% This file contains the test cases for cth_surefire. +%% +%% @author Support +%% @doc Test of surefire support in common_test +%% @end +%%---------------------------------------------------------------------- +%%---------------------------------------------------------------------- +-module(surefire_SUITE). +-include_lib("common_test/include/ct.hrl"). + +-compile(export_all). + +%% Default timetrap timeout (set in init_per_testcase). +-define(default_timeout, ?t:minutes(1)). + +all() -> + testcases() ++ [{group,g},{group,g_fail}]. + +groups() -> + [{g,testcases()}, + {g_fail,[tc_ok]}]. + +testcases() -> + [tc_ok, + tc_fail, + tc_skip, + tc_autoskip_require]. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + Config. + +init_per_group(g_fail, _Config) -> + exit(all_cases_should_be_skipped); +init_per_group(_, Config) -> + Config. + +end_per_group(_Group, Config) -> + Config. + +init_per_testcase(_Case, Config) -> + Dog = test_server:timetrap(?default_timeout), + [{watchdog, Dog}|Config]. + +end_per_testcase(_Case, Config) -> + Dog=?config(watchdog, Config), + test_server:timetrap_cancel(Dog), + ok. + +%%%----------------------------------------------------------------- +%%% Test cases +break(_Config) -> + test_server:break(""), + ok. + +tc_ok(_Config) -> + ok. + +tc_fail(_Config) -> + ct:fail("this test should fail"). + +tc_skip(_Config) -> + {skip,"this test is skipped"}. + +tc_autoskip_require() -> + [{require,whatever}]. +tc_autoskip_require(Config) -> + ct:fail("this test should never be executed - it should be autoskipped"). diff --git a/lib/common_test/test/ct_system_error_SUITE.erl b/lib/common_test/test/ct_system_error_SUITE.erl index f00f470c33..f2d6ef4b1b 100644 --- a/lib/common_test/test/ct_system_error_SUITE.erl +++ b/lib/common_test/test/ct_system_error_SUITE.erl @@ -87,7 +87,7 @@ test_server_failing_logs(Config) -> crash_test_server(Config) -> DataDir = ?config(data_dir, Config), - Root = ?config(priv_dir, Config), + Root = proplists:get_value(logdir, ct_test_support:get_opts(Config)), [$@|Host] = lists:dropwhile(fun(C) -> C =/= $@ end, atom_to_list(node())), diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl index 80cca4a1cc..fc572aa82f 100644 --- a/lib/common_test/test/ct_test_support.erl +++ b/lib/common_test/test/ct_test_support.erl @@ -32,7 +32,7 @@ run/2, run/3, run/4, get_opts/1, wait_for_ct_stop/1]). -export([handle_event/2, start_event_receiver/1, get_events/2, - verify_events/3, reformat/2, log_events/4, + verify_events/3, verify_events/4, reformat/2, log_events/4, join_abs_dirs/2]). -export([ct_test_halt/1]). @@ -117,8 +117,7 @@ end_per_suite(Config) -> CTNode = proplists:get_value(ct_node, Config), PrivDir = proplists:get_value(priv_dir, Config), true = rpc:call(CTNode, code, del_path, [filename:join(PrivDir,"")]), - cover:stop(CTNode), - slave:stop(CTNode), + slave_stop(CTNode), ok. %%%----------------------------------------------------------------- @@ -149,8 +148,7 @@ end_per_testcase(_TestCase, Config) -> case wait_for_ct_stop(CTNode) of %% Common test was not stopped to we restart node. false -> - cover:stop(CTNode), - slave:stop(CTNode), + slave_stop(CTNode), start_slave(Config,proplists:get_value(trace_level,Config)), {fail, "Could not stop common_test"}; true -> @@ -364,6 +362,14 @@ verify_events(TEvs, Evs, Config) -> ok end. +verify_events(TEvs, Evs, Node, Config) -> + case catch verify_events1(TEvs, Evs, Node, Config) of + {'EXIT',Reason} -> + Reason; + _ -> + ok + end. + verify_events1([TestEv|_], [{TEH,#event{name=stop_logging,node=Node,data=_}}|_], Node, _) when element(1,TestEv) == TEH, element(2,TestEv) =/= stop_logging -> test_server:format("Failed to find ~p in the list of events!~n", [TestEv]), @@ -612,8 +618,11 @@ locate({parallel,TEvs}, Node, Evs, Config) -> fun({EH,#event{name=tc_auto_skip, node=EvNode, data={Mod,end_per_group,Reason}}}) when - EH == TEH, EvNode == Node, Mod == M, Reason == R -> - false; + EH == TEH, EvNode == Node, Mod == M -> + case match_data(R, Reason) of + match -> false; + _ -> true + end; ({EH,#event{name=stop_logging, node=EvNode,data=_}}) when EH == TEH, EvNode == Node -> @@ -627,23 +636,12 @@ locate({parallel,TEvs}, Node, Evs, Config) -> [_AutoSkip | RemEvs2] -> {Done,RemEvs2,length(RemEvs2)} end; - %% match other event than test case - (TEv={TEH,N,D}, Acc) when D == '_' -> - case [E || E={EH,#event{name=Name, - node=EvNode, - data=_}} <- Evs1, - EH == TEH, EvNode == Node, Name == N] of - [] -> - exit({unmatched,TEv}); - _ -> - test_server:format("Found ~p!", [TEv]), - Acc - end; (TEv={TEH,N,D}, Acc) -> case [E || E={EH,#event{name=Name, node=EvNode, data=Data}} <- Evs1, - EH == TEH, EvNode == Node, Name == N, Data == D] of + EH == TEH, EvNode == Node, Name == N, + match == match_data(D,Data)] of [] -> exit({unmatched,TEv}); _ -> @@ -1002,33 +1000,39 @@ locate({TEH,Name,Data}, Node, [{TEH,#event{name=Name, data = EvData, node = Node}}|Evs], Config) -> - try match_data(Data, EvData) of + case match_data(Data, EvData) of match -> - {Config,Evs} - catch _:_ -> + {Config,Evs}; + _ -> nomatch end; locate({_TEH,_Name,_Data}, _Node, [_|_Evs], _Config) -> nomatch. -match_data(D,D) -> +match_data(Data, EvData) -> + try do_match_data(Data, EvData) + catch _:_ -> + nomatch + end. + +do_match_data(D,D) -> match; -match_data('_',_) -> +do_match_data('_',_) -> match; -match_data(Fun,Data) when is_function(Fun) -> +do_match_data(Fun,Data) when is_function(Fun) -> Fun(Data); -match_data('$proplist',Proplist) -> - match_data( +do_match_data('$proplist',Proplist) -> + do_match_data( fun(List) -> lists:foreach(fun({_,_}) -> ok end,List) end,Proplist); -match_data([H1|MatchT],[H2|ValT]) -> - match_data(H1,H2), - match_data(MatchT,ValT); -match_data(Tuple1,Tuple2) when is_tuple(Tuple1),is_tuple(Tuple2) -> - match_data(tuple_to_list(Tuple1),tuple_to_list(Tuple2)); -match_data([],[]) -> +do_match_data([H1|MatchT],[H2|ValT]) -> + do_match_data(H1,H2), + do_match_data(MatchT,ValT); +do_match_data(Tuple1,Tuple2) when is_tuple(Tuple1),is_tuple(Tuple2) -> + do_match_data(tuple_to_list(Tuple1),tuple_to_list(Tuple2)); +do_match_data([],[]) -> match. result_match({SkipOrFail,{ErrorInd,{Why,'_'}}}, @@ -1043,6 +1047,9 @@ result_match({failed,{timetrap_timeout,{'$approx',Num}}}, Value =< trunc(Num+0.02*Num) -> true; true -> false end; +result_match({user_timetrap_error,{Why,'_'}}, + {user_timetrap_error,{Why,_Stack}}) -> + true; result_match(Result, Result) -> true; result_match(_, _) -> @@ -1259,3 +1266,22 @@ rm_files([F | Fs]) -> rm_files([]) -> ok. +%%%----------------------------------------------------------------- +%%% +slave_stop(Node) -> + Cover = test_server:is_cover(), + if Cover-> cover:flush(Node); + true -> ok + end, + erlang:monitor_node(Node, true), + slave:stop(Node), + receive + {nodedown, Node} -> + if Cover -> cover:stop(Node); + true -> ok + end + after 5000 -> + erlang:monitor_node(Node, false), + receive {nodedown, Node} -> ok after 0 -> ok end %flush + end, + ok. |