diff options
Diffstat (limited to 'lib/common_test/test')
68 files changed, 5543 insertions, 203 deletions
diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile index 374fd8a824..bd746f87a7 100644 --- a/lib/common_test/test/Makefile +++ b/lib/common_test/test/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2008-2012. All Rights Reserved. +# Copyright Ericsson AB 2008-2013. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -40,6 +40,7 @@ MODULES= \ ct_repeat_1_SUITE \ ct_testspec_1_SUITE \ ct_testspec_2_SUITE \ + ct_testspec_3_SUITE \ ct_skip_SUITE \ ct_error_SUITE \ ct_test_server_if_1_SUITE \ @@ -52,7 +53,12 @@ MODULES= \ ct_auto_compile_SUITE \ ct_verbosity_SUITE \ ct_shell_SUITE \ - ct_groups_search_SUITE + ct_system_error_SUITE \ + ct_snmp_SUITE \ + ct_group_leader_SUITE \ + ct_cover_SUITE \ + ct_groups_search_SUITE \ + ct_surefire_SUITE ERL_FILES= $(MODULES:%=%.erl) @@ -106,7 +112,7 @@ release_spec: opt release_tests_spec: $(INSTALL_DIR) "$(RELSYSDIR)" $(INSTALL_DATA) $(ERL_FILES) $(COVERFILE) "$(RELSYSDIR)" - $(INSTALL_DATA) common_test.spec "$(RELSYSDIR)" + $(INSTALL_DATA) common_test.spec common_test.cover "$(RELSYSDIR)" chmod -R u+w "$(RELSYSDIR)" @tar cf - *_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -) diff --git a/lib/common_test/test/common_test.cover b/lib/common_test/test/common_test.cover new file mode 100644 index 0000000000..3aa49623e7 --- /dev/null +++ b/lib/common_test/test/common_test.cover @@ -0,0 +1,10 @@ +%% -*- erlang -*- +{incl_app,common_test,details}. +{cross,common_test,[{test_server,[erl2html2, + test_server, + test_server_ctrl, + test_server_gl, + test_server_h, + test_server_io, + test_server_node, + test_server_sup]}]}. diff --git a/lib/common_test/test/ct_config_info_SUITE.erl b/lib/common_test/test/ct_config_info_SUITE.erl index 40da377ee5..10fe8286dd 100644 --- a/lib/common_test/test/ct_config_info_SUITE.erl +++ b/lib/common_test/test/ct_config_info_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2011. All Rights Reserved. +%% Copyright Ericsson AB 2009-2012. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -123,8 +123,7 @@ test_events(config_info) -> {?eh,tc_done,{config_info_1_SUITE,init_per_suite,ok}}, [{?eh,tc_start,{config_info_1_SUITE,{init_per_group,g1,[]}}}, - {?eh,tc_done,{config_info_1_SUITE, - {init_per_group,unknown,[]}, + {?eh,tc_done,{config_info_1_SUITE,{init_per_group,g1,[]}, {failed,{timetrap_timeout,350}}}}, {?eh,tc_auto_skip,{config_info_1_SUITE,t11, {failed,{config_info_1_SUITE,init_per_group,{timetrap_timeout,350}}}}}, @@ -136,14 +135,12 @@ test_events(config_info) -> {?eh,tc_done,{config_info_1_SUITE,{init_per_group,g2,[]},ok}}, {?eh,tc_done,{config_info_1_SUITE,t21,ok}}, {?eh,tc_start,{config_info_1_SUITE,{end_per_group,g2,[]}}}, - {?eh,tc_done,{config_info_1_SUITE, - {end_per_group,unknown,[]}, + {?eh,tc_done,{config_info_1_SUITE,{end_per_group,g2,[]}, {failed,{timetrap_timeout,450}}}}], [{?eh,tc_start,{config_info_1_SUITE,{init_per_group,g3,[]}}}, {?eh,tc_done,{config_info_1_SUITE,{init_per_group,g3,[]},ok}}, [{?eh,tc_start,{config_info_1_SUITE,{init_per_group,g4,[]}}}, - {?eh,tc_done,{config_info_1_SUITE, - {init_per_group,unknown,[]}, + {?eh,tc_done,{config_info_1_SUITE,{init_per_group,g4,[]}, {failed,{timetrap_timeout,400}}}}, {?eh,tc_auto_skip,{config_info_1_SUITE,t41, {failed,{config_info_1_SUITE,init_per_group, @@ -164,8 +161,7 @@ test_events(config_info) -> {?eh,tc_done,{config_info_1_SUITE,{init_per_group,g5,[]},ok}}, {?eh,tc_done,{config_info_1_SUITE,t51,ok}}, {?eh,tc_start,{config_info_1_SUITE,{end_per_group,g5,[]}}}, - {?eh,tc_done,{config_info_1_SUITE, - {end_per_group,unknown,[]}, + {?eh,tc_done,{config_info_1_SUITE,{end_per_group,g5,[]}, {failed,{timetrap_timeout,400}}}}], {?eh,tc_start,{config_info_1_SUITE,{end_per_group,g3,[]}}}, {?eh,tc_done,{config_info_1_SUITE,{end_per_group,g3,[]},ok}}], diff --git a/lib/common_test/test/ct_cover_SUITE.erl b/lib/common_test/test/ct_cover_SUITE.erl new file mode 100644 index 0000000000..ec2680f664 --- /dev/null +++ b/lib/common_test/test/ct_cover_SUITE.erl @@ -0,0 +1,321 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012-2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_cover_SUITE +%%% +%%% Description: +%%% Test code cover analysis support +%%% +%%%------------------------------------------------------------------- +-module(ct_cover_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). +-define(suite, cover_SUITE). +-define(mod, cover_test_mod). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + case test_server:is_cover() of + true -> + {skip,"Test server is running cover already - skipping"}; + false -> + ct_test_support:init_per_suite(Config) + end. + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + try apply(?MODULE,TestCase,[cleanup,Config]) + catch error:undef -> ok + end, + ct_test_support:end_per_testcase(TestCase, Config). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [ + default, + cover_stop_true, + cover_stop_false, + slave, + slave_start_slave, + cover_node_option, + ct_cover_add_remove_nodes, + otp_9956, + cross + ]. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%% Check that cover is collected from test node +%% Also check that cover is by default stopped after test is completed +default(Config) -> + {ok,Events} = run_test(default,Config), + false = check_cover(Config), + check_calls(Events,1), + ok. + +%% Check that cover is stopped when cover_stop option is set to true +cover_stop_true(Config) -> + {ok,_Events} = run_test(cover_stop_true,[{cover_stop,true}],Config), + false = check_cover(Config). + +%% Check that cover is not stopped when cover_stop option is set to false +cover_stop_false(Config) -> + {ok,_Events} = run_test(cover_stop_false,[{cover_stop,false}],Config), + {true,[],[?mod]} = check_cover(Config), + CTNode = proplists:get_value(ct_node, Config), + ok = rpc:call(CTNode,cover,stop,[]), + false = check_cover(Config), + ok. + +%% Let test node start a slave node - check that cover is collected +%% from both nodes +slave(Config) -> + {ok,Events} = run_test(slave,slave,[],Config), + check_calls(Events,2), + ok. + +%% Let test node start a slave node which in turn starts another slave +%% node - check that cover is collected from all three nodes +slave_start_slave(Config) -> + {ok,Events} = run_test(slave_start_slave,slave_start_slave,[],Config), + check_calls(Events,3), + ok. + +%% Start a slave node before test starts - the node is listed in cover +%% spec file. +%% Check that cover is collected from test node and slave node. +cover_node_option(Config) -> + DataDir = ?config(data_dir,Config), + {ok,Node} = start_slave(existing_node_1, "-pa " ++ DataDir), + false = check_cover(Node), + CoverSpec = default_cover_file_content() ++ [{nodes,[Node]}], + CoverFile = create_cover_file(cover_node_option,CoverSpec,Config), + {ok,Events} = run_test(cover_node_option,cover_node_option, + [{cover,CoverFile}],Config), + check_calls(Events,2), + {ok,Node} = ct_slave:stop(existing_node_1), + ok. + +cover_node_option(cleanup,_Config) -> + _ = ct_slave:stop(existing_node_1), + ok. + +%% Test ct_cover:add_nodes/1 and ct_cover:remove_nodes/1 +%% Check that cover is collected from added node +ct_cover_add_remove_nodes(Config) -> + DataDir = ?config(data_dir,Config), + {ok,Node} = start_slave(existing_node_2, "-pa " ++ DataDir), + false = check_cover(Node), + {ok,Events} = run_test(ct_cover_add_remove_nodes,ct_cover_add_remove_nodes, + [],Config), + check_calls(Events,2), + {ok,Node} = ct_slave:stop(existing_node_2), + ok. + +ct_cover_add_remove_nodes(cleanup,_Config) -> + _ = ct_slave:stop(existing_node_2), + ok. + +%% Test that the test suite itself can be cover compiled and that +%% data_dir is set correctly (OTP-9956) +otp_9956(Config) -> + CoverFile = create_cover_file(otp_9956,[{incl_mods,[?suite]}],Config), + {ok,Events} = run_test(otp_9956,otp_9956,[{cover,CoverFile}],Config), + check_calls(Events,{?suite,otp_9956,1},1), + ok. + +%% Test cross cover mechanism +cross(Config) -> + {ok,Events1} = run_test(cross1,Config), + check_calls(Events1,1), + + CoverFile2 = create_cover_file(cross1,[{cross,[{cross1,[?mod]}]}],Config), + {ok,Events2} = run_test(cross2,[{cover,CoverFile2}],Config), + check_calls(Events2,1), + + %% Get the log dirs for each test and run cross cover analyse + [D11,D12] = lists:sort(get_run_dirs(Events1)), + [D21,D22] = lists:sort(get_run_dirs(Events2)), + + ct_cover:cross_cover_analyse(details,[{cross1,D11},{cross2,D21}]), + ct_cover:cross_cover_analyse(details,[{cross1,D12},{cross2,D22}]), + + %% Get the cross cover logs and read for each test + [C11,C12,C21,C22] = + [filename:join(D,"cross_cover.html") || D <- [D11,D12,D21,D22]], + + {ok,CrossData} = file:read_file(C11), + {ok,CrossData} = file:read_file(C12), + + {ok,Def} = file:read_file(C21), + {ok,Def} = file:read_file(C22), + + %% A simple test: just check that the test module exists in the + %% log from cross1 test, and that it does not exist in the log + %% from cross2 test. + TestMod = list_to_binary(atom_to_list(?mod)), + {_,_} = binary:match(CrossData,TestMod), + nomatch = binary:match(Def,TestMod), + {_,_} = binary:match(Def, + <<"No cross cover modules exist for this application">>), + + ok. + + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- +run_test(Label,Config) -> + run_test(Label,[],Config). +run_test(Label,ExtraOpts,Config) -> + run_test(Label,default,ExtraOpts,Config). +run_test(Label,Testcase,ExtraOpts,Config) -> + DataDir = ?config(data_dir, Config), + Suite = filename:join(DataDir, ?suite), + CoverFile = + case proplists:get_value(cover,ExtraOpts) of + undefined -> + create_default_cover_file(Label,Config); + CF -> + CF + end, + RestOpts = lists:keydelete(cover,1,ExtraOpts), + {Opts,ERPid} = setup([{suite,Suite},{testcase,Testcase}, + {cover,CoverFile},{label,Label}] ++ RestOpts, Config), + execute(Label, Testcase, Opts, ERPid, Config). + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +execute(Name, Testcase, Opts, ERPid, Config) -> + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(Name, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + TestEvents = events_to_check(Testcase), + R = ct_test_support:verify_events(TestEvents, Events, Config), + {R,Events}. + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). + +events_to_check(Testcase) -> + OneTest = + [{?eh,start_logging,{'DEF','RUNDIR'}}] ++ + [{?eh,tc_done,{?suite,Testcase,ok}}] ++ + [{?eh,stop_logging,[]}], + + %% 2 tests (ct:run_test + script_start) is default + OneTest ++ OneTest. + +check_cover(Config) when is_list(Config) -> + CTNode = proplists:get_value(ct_node, Config), + check_cover(CTNode); +check_cover(Node) when is_atom(Node) -> + case rpc:call(Node,test_server,is_cover,[]) of + true -> + {true, + rpc:call(Node,cover,which_nodes,[]), + rpc:call(Node,cover,modules,[])}; + false -> + false + end. + +%% Get the log dir "run.<timestamp>" for all (both!) tests +get_run_dirs(Events) -> + [filename:dirname(TCLog) || + {ct_test_support_eh, + {event,tc_logfile,_Node, + {{?suite,init_per_suite},TCLog}}} <- Events]. + +%% Check that each coverlog includes N calls to ?mod:foo/0 +check_calls(Events,N) -> + check_calls(Events,{?mod,foo,0},N). +check_calls(Events,MFA,N) -> + CoverLogs = [filename:join(D,"all.coverdata") || D <- get_run_dirs(Events)], + do_check_logs(CoverLogs,MFA,N). + +do_check_logs([CoverLog|CoverLogs],{Mod,_,_} = MFA,N) -> + {ok,_} = cover:start(), + ok = cover:import(CoverLog), + {ok,Calls} = cover:analyse(Mod,calls,function), + ok = cover:stop(), + {MFA,N} = lists:keyfind(MFA,1,Calls), + do_check_logs(CoverLogs,MFA,N); +do_check_logs([],_,_) -> + ok. + +fullname(Name) -> + {ok,Host} = inet:gethostname(), + list_to_atom(atom_to_list(Name) ++ "@" ++ Host). + +default_cover_file_content() -> + [{incl_mods,[?mod]}]. + +create_default_cover_file(Filename,Config) -> + create_cover_file(Filename,default_cover_file_content(),Config). + +create_cover_file(Filename,Terms,Config) -> + PrivDir = ?config(priv_dir,Config), + File = filename:join(PrivDir,Filename) ++ ".cover", + {ok,Fd} = file:open(File,[write]), + lists:foreach(fun(Term) -> + file:write(Fd,io_lib:format("~p.~n",[Term])) + end,Terms), + ok = file:close(Fd), + File. + +start_slave(Name,Args) -> + {ok, HostStr}=inet:gethostname(), + Host = list_to_atom(HostStr), + ct_slave:start(Host,Name, + [{erl_flags,Args}, + {boot_timeout,10}, % extending some timers for slow test hosts + {init_timeout,10}, + {startup_timeout,10}]). diff --git a/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl new file mode 100644 index 0000000000..d967590c72 --- /dev/null +++ b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl @@ -0,0 +1,164 @@ +%%-------------------------------------------------------------------- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012-2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%%---------------------------------------------------------------------- +%% File: cover_SUITE.erl +%% +%% Description: +%% This file contains the test cases for the code coverage support +%% +%% @author Support +%% @doc Test of code coverage support in common_test +%% @end +%%---------------------------------------------------------------------- +%%---------------------------------------------------------------------- +-module(cover_SUITE). +-include_lib("common_test/include/ct.hrl"). + +-compile(export_all). + +%% Default timetrap timeout (set in init_per_testcase). +-define(default_timeout, ?t:minutes(1)). + +suite() -> + []. + +all() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + Config. + +init_per_testcase(_Case, Config) -> + Dog = test_server:timetrap(?default_timeout), + [{watchdog, Dog}|Config]. + +end_per_testcase(Case, Config) -> + %% try apply(?MODULE,Case,[cleanup,Config]) + %% catch error:undef -> ok + %% end, + + kill_slaves(Case,nodes()), + Dog=?config(watchdog, Config), + test_server:timetrap_cancel(Dog), + ok. + +%%%----------------------------------------------------------------- +%%% Test cases +break(_Config) -> + test_server:break(""), + ok. + +default(Config) -> + cover_compiled = code:which(cover_test_mod), + cover_test_mod:foo(), + ok. + +slave(Config) -> + cover_compiled = code:which(cover_test_mod), + cover_test_mod:foo(), + N1 = nodename(slave,1), + {ok,Node} = start_slave(N1), + cover_compiled = rpc:call(Node,code,which,[cover_test_mod]), + rpc:call(Node,cover_test_mod,foo,[]), + {ok,Node} = ct_slave:stop(N1), + ok. + +slave_start_slave(Config) -> + cover_compiled = code:which(cover_test_mod), + cover_test_mod:foo(), + N1 = nodename(slave_start_slave,1), + N2 = nodename(slave_start_slave,2), + {ok,Node} = start_slave(N1), + cover_compiled = rpc:call(Node,code,which,[cover_test_mod]), + rpc:call(Node,cover_test_mod,foo,[]), + {ok,Node2} = rpc:call(Node,ct_slave,start,[N2]), + rpc:call(Node2,cover_test_mod,foo,[]), + {ok,Node2} = rpc:call(Node,ct_slave,stop,[N2]), + {ok,Node} = ct_slave:stop(N1), + ok. + +cover_node_option(Config) -> + cover_compiled = code:which(cover_test_mod), + cover_test_mod:foo(), + Node = fullname(existing_node_1), + cover_compiled = rpc:call(Node,code,which,[cover_test_mod]), + rpc:call(Node,cover_test_mod,foo,[]), + ok. + +ct_cover_add_remove_nodes(Config) -> + cover_compiled = code:which(cover_test_mod), + cover_test_mod:foo(), + Node = fullname(existing_node_2), + Beam = rpc:call(Node,code,which,[cover_test_mod]), + false = (Beam == cover_compiled), + + rpc:call(Node,cover_test_mod,foo,[]), % should not be collected + {ok,[Node]} = ct_cover:add_nodes([Node]), + cover_compiled = rpc:call(Node,code,which,[cover_test_mod]), + rpc:call(Node,cover_test_mod,foo,[]), % should be collected + ok = ct_cover:remove_nodes([Node]), + rpc:call(Node,cover_test_mod,foo,[]), % should not be collected + + Beam = rpc:call(Node,code,which,[cover_test_mod]), + + ok. + +otp_9956(Config) -> + cover_compiled = code:which(?MODULE), + DataDir = ?config(data_dir,Config), + absolute = filename:pathtype(DataDir), + true = filelib:is_dir(DataDir), + ok. + + +%%%----------------------------------------------------------------- +%%% Internal +nodename(Case,N) -> + list_to_atom(nodeprefix(Case) ++ integer_to_list(N)). + +nodeprefix(Case) -> + atom_to_list(?MODULE) ++ "_" ++ atom_to_list(Case) ++ "_node". + + +fullname(Name) -> + {ok,Host} = inet:gethostname(), + list_to_atom(atom_to_list(Name) ++ "@" ++ Host). + +kill_slaves(Case, [Node|Nodes]) -> + Prefix = nodeprefix(Case), + case lists:prefix(Prefix,atom_to_list(Node)) of + true -> + rpc:call(Node,erlang,halt,[]); + _ -> + ok + end, + kill_slaves(Case,Nodes); +kill_slaves(_,[]) -> + ok. + +start_slave(Name) -> + {ok, HostStr}=inet:gethostname(), + Host = list_to_atom(HostStr), + ct_slave:start(Host,Name, + [{boot_timeout,10}, % extending some timers for slow test hosts + {init_timeout,10}, + {startup_timeout,10}]). diff --git a/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE_data/.gitignore b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE_data/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE_data/.gitignore diff --git a/lib/common_test/test/ct_cover_SUITE_data/cover_test_mod.erl b/lib/common_test/test/ct_cover_SUITE_data/cover_test_mod.erl new file mode 100644 index 0000000000..d4f69452c3 --- /dev/null +++ b/lib/common_test/test/ct_cover_SUITE_data/cover_test_mod.erl @@ -0,0 +1,4 @@ +-module(cover_test_mod). +-compile(export_all). +foo() -> + ok. diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index 338e76264e..3881ced17d 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% Copyright Ericsson AB 2009-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -44,8 +44,12 @@ %% there will be clashes with logging processes etc). %%-------------------------------------------------------------------- init_per_suite(Config) -> - Config1 = ct_test_support:init_per_suite(Config), - Config1. + DataDir = ?config(data_dir, Config), + TestDir = filename:join(DataDir, "error/test/"), + CTH = filename:join(TestDir, "verify_config.erl"), + ct:pal("Compiling ~p: ~p", + [CTH,compile:file(CTH,[{outdir,TestDir},debug_info])]), + ct_test_support:init_per_suite([{path_dirs,[TestDir]} | Config]). end_per_suite(Config) -> ct_test_support:end_per_suite(Config). @@ -61,7 +65,8 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [cfg_error, lib_error, no_compile, timetrap_end_conf, timetrap_normal, timetrap_extended, timetrap_parallel, - timetrap_fun, misc_errors]. + timetrap_fun, timetrap_fun_group, misc_errors, + config_restored]. groups() -> []. @@ -251,6 +256,24 @@ timetrap_fun(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% +timetrap_fun_group(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + Join = fun(D, S) -> filename:join(D, "error/test/"++S) end, + Suites = [Join(DataDir, "timetrap_8_SUITE")], + {Opts,ERPid} = setup([{suite,Suites}], Config), + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(timetrap_fun_group, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(timetrap_fun_group), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + +%%%----------------------------------------------------------------- +%%% misc_errors(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), Join = fun(D, S) -> filename:join(D, "error/test/"++S) end, @@ -267,6 +290,24 @@ misc_errors(Config) when is_list(Config) -> TestEvents = events_to_check(misc_errors), ok = ct_test_support:verify_events(TestEvents, Events, Config). +%%%----------------------------------------------------------------- +%%% +config_restored(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + Suite = filename:join(DataDir, "error/test/config_restored_SUITE"), + {Opts,ERPid} = setup([{suite,Suite}, + {ct_hooks,[verify_config]}], + Config), + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(config_restored, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(config_restored), + ok = ct_test_support:verify_events(TestEvents, Events, Config). %%%----------------------------------------------------------------- %%% HELP FUNCTIONS @@ -429,8 +470,7 @@ test_events(cfg_error) -> {'EXIT',{init_per_group_fails,g1}}}}}}], [{?eh,tc_start,{cfg_error_8_SUITE,{init_per_group,g2,[]}}}, - {?eh,tc_done,{cfg_error_8_SUITE, - {init_per_group,unknown,[]}, + {?eh,tc_done,{cfg_error_8_SUITE,{init_per_group,g2,[]}, {failed,{timetrap_timeout,2000}}}}, {?eh,tc_auto_skip,{cfg_error_8_SUITE,tc1, {failed,{cfg_error_8_SUITE,init_per_group, @@ -500,7 +540,7 @@ test_events(cfg_error) -> {?eh,tc_done,{cfg_error_8_SUITE,tc1,ok}}, {?eh,test_stats,{9,0,{0,14}}}, {?eh,tc_start,{cfg_error_8_SUITE,{end_per_group,g12,[]}}}, - {?eh,tc_done,{cfg_error_8_SUITE,{end_per_group,unknown,[]}, + {?eh,tc_done,{cfg_error_8_SUITE,{end_per_group,g12,[]}, {failed,{timetrap_timeout,2000}}}}], {?eh,tc_start,{cfg_error_8_SUITE,end_per_suite}}, @@ -971,11 +1011,423 @@ test_events(timetrap_fun) -> {?eh,stop_logging,[]} ]; +test_events(timetrap_fun_group) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,58}}, + {?eh,tc_start,{timetrap_8_SUITE,init_per_suite}}, + {?eh,tc_done,{timetrap_8_SUITE,init_per_suite,ok}}, + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g0,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g0,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{0,1,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{0,2,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g0,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g0,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g1,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g1,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{0,3,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{0,4,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g1,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g1,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g2,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g2,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc1}}, + {?eh,tc_done,{timetrap_8_SUITE,tc1, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{0,5,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{0,6,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g2,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g2,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g3,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g3,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc4}}, + {?eh,tc_done,{timetrap_8_SUITE,tc4, + {failed,{timetrap_timeout,{'$approx',2000}}}}}, + {?eh,test_stats,{0,7,{0,0}}}, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g1,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g1,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{0,8,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{0,9,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g1,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g1,[]},ok}}], + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g2,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g2,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc1}}, + {?eh,tc_done,{timetrap_8_SUITE,tc1, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{0,10,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{0,11,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g2,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g2,[]},ok}}], + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g3,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g3,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g4,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g4,[]}, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{0,11,{0,1}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{0,11,{0,2}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g5,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g5,[]}, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{0,11,{0,3}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{0,11,{0,4}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g6,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g6,[]}, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}, + {?eh,test_stats,{0,11,{0,5}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}, + {?eh,test_stats,{0,11,{0,6}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g7,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g7,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{1,11,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g7,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g7,[]}, + {user_timetrap_error,{kaboom,'_'}}}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g8,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g8,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{2,11,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g8,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g8,[]}, + {failed,{timetrap_timeout,{'$approx',500}}}}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g9,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g9,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{3,11,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,test_stats,{3,12,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g9,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g9,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g10,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g10,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,test_stats,{3,13,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{4,13,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g10,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g10,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g11,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g11,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc3}}, + {?eh,tc_done,{timetrap_8_SUITE,tc3, + {failed,{timetrap_timeout,{'$approx',4000}}}}}, + {?eh,test_stats,{4,14,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,15,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g11,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g11,[]},ok}}], + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg0,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg0,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{4,16,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,17,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg0,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg0,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg1,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg1,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{4,18,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,19,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg1,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg1,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg2,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg2,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc1}}, + {?eh,tc_done,{timetrap_8_SUITE,tc1, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{4,20,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,21,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg2,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg2,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg3,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg3,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc4}}, + {?eh,tc_done,{timetrap_8_SUITE,tc4, + {failed,{timetrap_timeout,{'$approx',2000}}}}}, + {?eh,test_stats,{4,22,{0,6}}}, + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg1,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg1,[parallel]}, + ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{4,23,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,24,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg1,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg1,[parallel]}, + ok}}]}, + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg2,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg2,[parallel]}, + ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc1}}, + {?eh,tc_done,{timetrap_8_SUITE,tc1, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{4,25,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,26,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg2,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg2,[parallel]}, + ok}}]}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg3,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg3,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg4,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg4,[parallel]}, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{4,26,{0,7}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{4,26,{0,8}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg5,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg5,[parallel]}, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{4,26,{0,9}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{4,26,{0,10}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg6,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg6,[parallel]}, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}, + {?eh,test_stats,{4,26,{0,11}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}, + {?eh,test_stats,{4,26,{0,12}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg7,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg7,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{5,26,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg7,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg7,[parallel]}, + {user_timetrap_error,{kaboom,'_'}}}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg8,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg8,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{6,26,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg8,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg8,[parallel]}, + {failed,{timetrap_timeout,{'$approx',500}}}}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg9,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg9,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {user_timetrap_error,{kaboom,'_'}}}}, + %% Due to parallelism only checking final test stat in group + {?eh,test_stats,{7,27,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg9,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg9,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg10,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg10,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + %% Due to parallelism only checking final test stat in group + {?eh,test_stats,{8,28,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg10,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg10,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg11,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg11,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc3}}, + {?eh,tc_done,{timetrap_8_SUITE,tc3, + {failed,{timetrap_timeout,{'$approx',4000}}}}}, + {?eh,test_stats,{8,29,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{8,30,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg11,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg11,[parallel]},ok}}]}, + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,sg1,[sequence]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,sg1,[sequence]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{9,30,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,test_stats,{9,31,{0,12}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc1, + {failed,{timetrap_8_SUITE,tc0}}}}, + {?eh,test_stats,{9,31,{0,13}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,tc0}}}}, + {?eh,test_stats,{9,31,{0,14}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,sg1,[sequence]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,sg1,[sequence]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,sg2,[sequence]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,sg2,[sequence]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{10,31,{0,14}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{10,32,{0,14}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc1, + {failed,{timetrap_8_SUITE,tc0}}}}, + {?eh,test_stats,{10,32,{0,15}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,tc0}}}}, + {?eh,test_stats,{10,32,{0,16}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,sg2,[sequence]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,sg2,[sequence]},ok}}], + + {?eh,tc_start,{timetrap_8_SUITE,end_per_suite}}, + {?eh,tc_done,{timetrap_8_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + test_events(misc_errors) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, - {?eh,start_info,{1,1,7}}, + {?eh,start_info,{1,1,9}}, {?eh,tc_start,{misc_error_1_SUITE,ct_fail_1}}, {?eh,tc_done,{misc_error_1_SUITE,ct_fail_1, {failed,{error,{test_case_failed,{error,this_is_expected}}}}}}, @@ -1002,7 +1454,48 @@ test_events(misc_errors) -> {?eh,tc_start,{misc_error_1_SUITE,killed_by_signal_2}}, {?eh,tc_done,{misc_error_1_SUITE,killed_by_signal_2, {failed,testcase_aborted_or_killed}}}, - {?eh,test_stats,{0,7,{0,0}}}, + {parallel, + [{?eh,tc_start,{misc_error_1_SUITE,p1}}, + {?eh,tc_done,{misc_error_1_SUITE,p1,ok}}, + {?eh,tc_start,{misc_error_1_SUITE,p2}}, + {?eh,tc_done,{misc_error_1_SUITE,p2,ok}}]}, + {?eh,test_stats,{2,7,{0,0}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(config_restored) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{config_restored_SUITE,init_per_suite}}, + {?eh,tc_done,{config_restored_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{config_restored_SUITE,to_tc}}, + {?eh,cth,{verify_config,post_end_per_testcase,{to_tc,diff_ok}}}, + {?eh,tc_done, + {config_restored_SUITE,to_tc,{failed,{timetrap_timeout,1000}}}}, + {?eh,test_stats,{0,1,{0,0}}}, + {?eh,tc_start,{config_restored_SUITE,exit_tc}}, + {?eh,cth,{verify_config,post_end_per_testcase,{exit_tc,diff_ok}}}, + {?eh,tc_done,{config_restored_SUITE,exit_tc, + {failed,{error,{test_case_failed,"Goodbye!"}}}}}, + {?eh,test_stats,{0,2,{0,0}}}, + [{?eh,tc_start,{config_restored_SUITE,{init_per_group,g1,[]}}}, + {?eh,tc_start,{config_restored_SUITE,to_tc}}, + {?eh,cth,{verify_config,post_end_per_testcase,{to_tc,diff_ok}}}, + {?eh,tc_done, + {config_restored_SUITE,to_tc,{failed,{timetrap_timeout,1000}}}}, + {?eh,test_stats,{0,3,{0,0}}}, + {?eh,tc_start,{config_restored_SUITE,exit_tc}}, + {?eh,cth,{verify_config,post_end_per_testcase,{exit_tc,diff_ok}}}, + {?eh,tc_done,{config_restored_SUITE,exit_tc, + {failed,{error,{test_case_failed,"Goodbye!"}}}}}, + {?eh,test_stats,{0,4,{0,0}}}, + {?eh,tc_start,{config_restored_SUITE,{end_per_group,g1,[]}}}, + {?eh,tc_done,{config_restored_SUITE,{end_per_group,g1,[]},ok}}], + {?eh,tc_start,{config_restored_SUITE,end_per_suite}}, + {?eh,tc_done,{config_restored_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} ]. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/config_restored_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/config_restored_SUITE.erl new file mode 100644 index 0000000000..bcbf972a36 --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/config_restored_SUITE.erl @@ -0,0 +1,175 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(config_restored_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,1000}]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config1 = [{init_per_suite,?MODULE} | Config], + TabPid = spawn(fun() -> + ets:new(?MODULE, [named_table, set, public]), + receive _ -> ok end + end), + [{tab,TabPid} | Config1]. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_suite(Config) -> + ets:delete(?MODULE), + exit(?config(tab, Config), kill), + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_group(GroupName, Config) -> + [{init_per_group,GroupName} | Config]. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_testcase(TC, Config) -> + Config1 = [{init_per_testcase,TC} | Config], + ets:insert(?MODULE, {config,Config}), + %% ct:pal("Config after init_per_testcase(~w) = ~p", [TC,Config1]), + Config1. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_testcase(TC, Config) -> + ct:pal("Config in end_per_testcase(~w) = ~p", [TC,Config]), + [{_,MemConfig}] = ets:lookup(?MODULE, config), + diff_config(Config, MemConfig, [tc_status]), + ?MODULE = proplists:get_value(init_per_suite, Config), + TC = proplists:get_value(init_per_testcase, Config), + case ?config(tc_group_properties, Config) of + undefined -> + ok; + Props -> + GName = proplists:get_value(name, Props), + GName = proplists:get_value(init_per_group, Config) + end, + ok. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + [{g1,[],[to_tc, exit_tc]}]. + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [to_tc, exit_tc, + {group,g1}]. + + +to_tc(Config) -> + %% ct:pal("Config for to_tc = ~p", [Config]), + [{_,MemConfig}] = ets:lookup(?MODULE, config), + diff_config(Config, MemConfig, []), + ct:sleep(2000). + +exit_tc(Config) -> + %% ct:pal("Config for exit_tc = ~p", [Config]), + [{_,MemConfig}] = ets:lookup(?MODULE, config), + diff_config(Config, MemConfig, []), + ct:fail("Goodbye!"). + + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS + +diff_config(Cfg1, Cfg2, DiffKeys) -> + diff_config(Cfg1, Cfg2, DiffKeys, []). + +diff_config([{K,V} | Cfg1], Cfg2, DiffKeys, RemKeys) -> + case proplists:get_value(K, Cfg2) of + undefined -> + diff_config(Cfg1, Cfg2, proplists:delete(K, DiffKeys), RemKeys); + V -> + diff_config(Cfg1, proplists:delete(K, Cfg2), DiffKeys, RemKeys); + _Other -> + case proplists:is_defined(K, DiffKeys) of + true -> + diff_config(Cfg1, Cfg2, proplists:delete(K, DiffKeys), RemKeys); + false -> + diff_config(Cfg1, Cfg2, DiffKeys, [K | RemKeys]) + end + end; +diff_config([], [], [], []) -> + ct:pal("Diff ok!", []), + ok; +diff_config([], Cfg2, DiffKeys, RemKeys) -> + Result = {diff_failed, {cfg2,Cfg2}, {diffkeys,DiffKeys}, {remkeys,RemKeys}}, + ct:pal("Diff failed! Result = ~p", [Result]), + exit(Result). + diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/misc_error_1_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/misc_error_1_SUITE.erl index 99c3ed05ec..61f3fa7e59 100644 --- a/lib/common_test/test/ct_error_SUITE_data/error/test/misc_error_1_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/misc_error_1_SUITE.erl @@ -96,7 +96,7 @@ end_per_testcase(_TestCase, _Config) -> %% N = integer() | forever %%-------------------------------------------------------------------- groups() -> - []. + [{p,[parallel],[p1,p2]}]. %%-------------------------------------------------------------------- %% Function: all() -> GroupsAndTestCases | {skip,Reason} @@ -107,7 +107,8 @@ groups() -> %%-------------------------------------------------------------------- all() -> [ct_fail_1, ct_fail_2, ct_fail_3, ts_fail_1, ts_fail_2, - killed_by_signal_1, killed_by_signal_2]. + killed_by_signal_1, killed_by_signal_2, + {group,p}]. ct_fail_1(_) -> ct:fail({error,this_is_expected}), @@ -152,3 +153,10 @@ killed_by_signal_2(_) -> end), ct:sleep(1000), exit(this_should_not_be_seen). + +p1(_) -> + {error,parallel_group} = ct:abort_current_testcase(aborted), + ok. + +p2(_) -> + receive after 1000 -> ok end. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_8_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_8_SUITE.erl new file mode 100644 index 0000000000..ff138f38b5 --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_8_SUITE.erl @@ -0,0 +1,258 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(timetrap_8_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +-define(TO, 4). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{timetrap_utils,timetrap_val,[{seconds,?TO}]}}]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_group(G6, Config) when G6==g6; G6==pg6 -> + ct:sleep({seconds,1}), + Config; +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_group(G7or8, _Config) when G7or8==g7; G7or8==pg7; G7or8==g8; G7or8==pg8 -> + ct:sleep({seconds,5}), + ok; +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_testcase(_, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_testcase(_, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + [ + {g0,[],[tc0,tc2]}, % group override suite and tc overrides group + {g1,[],[tc0,tc2]}, % group override suite and tc overrides group + {g2,[],[tc1,tc2]}, % tc override group + {g3,[],[tc4,{group,g1},{group,g2}]}, % subgroup override group + {g4,[],[tc0,tc2]}, % exit during init_per_group + {g5,[],[tc0,tc2]}, % exit during init_per_group + {g6,[],[tc0,tc2]}, % timeout during init_per_group + {g7,[],[tc5]}, % exit during end_per_group + {g8,[],[tc5]}, % timeout during end_per_group + {g9,[],[tc5,tc0]}, % exit during testcase + {g10,[],[tc0,tc5]}, % exit during testcase + {g11,[],[tc3,tc2]}, % suite is valid if nothing else is specified + {pg0,[parallel],[tc0,tc2]}, % group override suite and tc overrides group + {pg1,[parallel],[tc0,tc2]}, % group override suite and tc overrides group + {pg2,[parallel],[tc1,tc2]}, % tc override group + {pg3,[parallel],[tc4,{group,pg1},{group,pg2}]}, % subgroup override group + {pg4,[parallel],[tc0,tc2]}, % exit during init_per_group + {pg5,[parallel],[tc0,tc2]}, % exit during init_per_group + {pg6,[parallel],[tc0,tc2]}, % timeout during init_per_group + {pg7,[parallel],[tc5]}, % exit during end_per_group + {pg8,[parallel],[tc5]}, % timeout during end_per_group + {pg9,[parallel],[tc5,tc0]}, % exit during testcase + {pg10,[parallel],[tc0,tc5]},% exit during testcase + {pg11,[parallel],[tc3,tc2]},% suite is valid if nothing else is specified + {sg1,[sequence],[tc5,tc0,tc1,tc2]}, % exit during sequencial testcase + {sg2,[sequence],[tc5,tc0,tc1,tc2]}].% timeout during sequencial testcase + +group(g0) -> + [{timetrap,{timetrap_utils,timetrap_val,[{seconds,1}]}}]; +group(g1) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(1000) end}]; +group(g2) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(3000) end}]; +group(g3) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(2000) end}]; +group(g4) -> + [{timetrap,{timetrap_utils,timetrap_exit,[kaboom]}}]; +group(g5) -> + [{timetrap,fun() -> exit(kaboom) end}]; +group(g6) -> + [{timetrap,{timetrap_utils,timetrap_val,[500]}}]; +group(g7) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(g8) -> + [{timetrap,{timetrap_utils,timetrap_val,[500]}}]; +group(g9) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(g10) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(g11) -> + []; +group(pg0) -> + [{timetrap,{timetrap_utils,timetrap_val,[{seconds,1}]}}]; +group(pg1) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(1000) end}]; +group(pg2) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(3000) end}]; +group(pg3) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(2000) end}]; +group(pg4) -> + [{timetrap,{timetrap_utils,timetrap_exit,[kaboom]}}]; +group(pg5) -> + [{timetrap,fun() -> exit(kaboom) end}]; +group(pg6) -> + [{timetrap,{timetrap_utils,timetrap_val,[500]}}]; +group(pg7) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(pg8) -> + [{timetrap,{timetrap_utils,timetrap_val,[500]}}]; +group(pg9) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(pg10) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(pg11) -> + []; +group(sg1) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(sg2) -> + [{timetrap,{timetrap_utils,timetrap_val,[{seconds,1}]}}]. + + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [ + {group,g0}, + {group,g1}, + {group,g2}, + {group,g3}, + {group,g4}, + {group,g5}, + {group,g6}, + {group,g7}, + {group,g8}, + {group,g9}, + {group,g10}, + {group,g11}, + {group,pg0}, + {group,pg1}, + {group,pg2}, + {group,pg3}, + {group,pg4}, + {group,pg5}, + {group,pg6}, + {group,pg7}, + {group,pg8}, + {group,pg9}, + {group,pg10}, + {group,pg11}, + {group,sg1}, + {group,sg2}]. + + + +tc0(_) -> + ct:comment("TO set by group"), + ct:sleep({seconds,5}), + ok. + +tc1() -> + [{timetrap,{timetrap_utils,timetrap_val,[1000]}}]. +tc1(_) -> + ct:comment("TO after 1 sec"), + ct:sleep({seconds,2}), + ok. + +tc2() -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(500) end}]. +tc2(_) -> + ct:comment("TO after 0.5 sec"), + ct:sleep({seconds,2}), + ok. + +tc3(_) -> + ct:comment(io_lib:format("TO after ~w sec", [?TO])), + ct:sleep({seconds,5}), + ok. + +tc4(_) -> + ct:comment("TO set by group"), + ct:sleep({seconds,5}), + ok. + +tc5(_) -> + ct:comment("No TO in this testcase, maybe later"), + ok. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/verify_config.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/verify_config.erl new file mode 100644 index 0000000000..446dd8bfdf --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/verify_config.erl @@ -0,0 +1,239 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%% @doc Common Test Example Suite Callback module. +%%% +%%% <p>This module gives an example of a common test CTH (Common Test Hook). +%%% There are many ways to add a CTH to a test run, you can do it either in +%%% the command line using -ct_hook, in a test spec using +%%% {ct_hook,M} or in the suite it self by returning ct_hook +%%% from either suite/0, init_per_suite/1, init_per_group/2 and +%%% init_per_testcase/2. The scope of the CTH is determined by where is it +%%% started. If it is started in the command line or test spec then it will +%%% be stopped at the end of all tests. If it is started in init_per_suite, +%%% it will be stopped after end_per_suite and so on. See terminate +%%% documentation for a table describing the scoping machanics. +%%% +%%% All of callbacks except init/1 in a CTH are optional.</p> + +-module(verify_config). + +%% CT Hooks +-export([id/1]). +-export([init/2]). + +-export([pre_init_per_suite/3]). +-export([post_init_per_suite/4]). +-export([pre_end_per_suite/3]). +-export([post_end_per_suite/4]). + +-export([pre_init_per_group/3]). +-export([post_init_per_group/4]). +-export([pre_end_per_group/3]). +-export([post_end_per_group/4]). + +-export([pre_init_per_testcase/3]). +-export([post_end_per_testcase/4]). + +-export([on_tc_fail/3]). +-export([on_tc_skip/3]). + +-export([terminate/1]). + +-include_lib("common_test/src/ct_util.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-type config() :: proplists:proplist(). +-type reason() :: term(). +-type skip_or_fail() :: {skip, reason()} | + {auto_skip, reason()} | + {fail, reason()} | + {'EXIT',reason()}. + +-record(state, { id = ?MODULE :: term()}). + +%% @doc Always called before any other callback function. Use this to initiate +%% any common state. It should return an state for this CTH. +-spec init(Id :: term(), Opts :: proplists:proplist()) -> + {ok, State :: #state{}}. +init(Id, Opts) -> + {ok,Opts}. + +%% @doc The ID is used to uniquly identify an CTH instance, if two CTH's +%% return the same ID the seconds CTH is ignored. This function should NOT +%% have any side effects as it might be called multiple times by common test. +-spec id(Opts :: proplists:proplist()) -> + Id :: term(). +id(Opts) -> + now(). + +%% @doc Called before init_per_suite is called. Note that this callback is +%% only called if the CTH is added before init_per_suite is run (eg. in a test +%% specification, suite/0 function etc). +%% You can change the config in the this function. +-spec pre_init_per_suite(Suite :: atom(), + Config :: config(), + State :: #state{}) -> + {config() | skip_or_fail(), NewState :: #state{}}. +pre_init_per_suite(Suite,Config,State) -> + {Config, State}. + +%% @doc Called after init_per_suite. +%% you can change the return value in this function. +-spec post_init_per_suite(Suite :: atom(), + Config :: config(), + Return :: config() | skip_or_fail(), + State :: #state{}) -> + {config() | skip_or_fail(), NewState :: #state{}}. +post_init_per_suite(Suite,Config,Return,State) -> + {Return, State}. + +%% @doc Called before end_per_suite. The config/state can be changed here, +%% though it will only affect the *end_per_suite function. +-spec pre_end_per_suite(Suite :: atom(), + Config :: config() | skip_or_fail(), + State :: #state{}) -> + {ok | skip_or_fail(), NewState :: #state{}}. +pre_end_per_suite(Suite,Config,State) -> + {Config, State}. + +%% @doc Called after end_per_suite. Note that the config cannot be +%% changed here, only the status of the suite. +-spec post_end_per_suite(Suite :: atom(), + Config :: config(), + Return :: term(), + State :: #state{}) -> + {ok | skip_or_fail(), NewState :: #state{}}. +post_end_per_suite(Suite,Config,Return,State) -> + {Return, State}. + +%% @doc Called before each init_per_group. +%% You can change the config in this function. +-spec pre_init_per_group(Group :: atom(), + Config :: config(), + State :: #state{}) -> + {config() | skip_or_fail(), NewState :: #state{}}. +pre_init_per_group(Group,Config,State) -> + {Config, State}. + +%% @doc Called after each init_per_group. +%% You can change the return value in this function. +-spec post_init_per_group(Group :: atom(), + Config :: config(), + Return :: config() | skip_or_fail(), + State :: #state{}) -> + {config() | skip_or_fail(), NewState :: #state{}}. +post_init_per_group(Group,Config,Return,State) -> + {Return, State}. + +%% @doc Called after each end_per_group. The config/state can be changed here, +%% though it will only affect the *end_per_group functions. +-spec pre_end_per_group(Group :: atom(), + Config :: config() | skip_or_fail(), + State :: #state{}) -> + {ok | skip_or_fail(), NewState :: #state{}}. +pre_end_per_group(Group,Config,State) -> + {Config, State}. + +%% @doc Called after each end_per_group. Note that the config cannot be +%% changed here, only the status of the group. +-spec post_end_per_group(Group :: atom(), + Config :: config(), + Return :: term(), + State :: #state{}) -> + {ok | skip_or_fail(), NewState :: #state{}}. +post_end_per_group(Group,Config,Return,State) -> + {Return, State}. + +%% @doc Called before each test case. +%% You can change the config in this function. +-spec pre_init_per_testcase(TC :: atom(), + Config :: config(), + State :: #state{}) -> + {config() | skip_or_fail(), NewState :: #state{}}. +pre_init_per_testcase(TC,Config,State) -> + {Config, State}. + +%% @doc Called after each test case. Note that the config cannot be +%% changed here, only the status of the test case. +-spec post_end_per_testcase(TC :: atom(), + Config :: config(), + Return :: term(), + State :: #state{}) -> + {ok | skip_or_fail(), NewState :: #state{}}. +post_end_per_testcase(TC,Config,Return,State) -> + %% check that config has been restored + ct:pal("Config in verify_config:post_end_per_testcase(~w) = ~p", + [TC,Config]), + [{_,MemConfig}] = ets:lookup(config_restored_SUITE, config), + try config_restored_SUITE:diff_config(Config, MemConfig, [tc_status]) of + ok -> + gen_event:notify( + ?CT_EVMGR_REF, #event{ name = cth, node = node(), + data = {?MODULE, post_end_per_testcase, + {TC,diff_ok}}}) + catch + _:_ -> + gen_event:notify( + ?CT_EVMGR_REF, #event{ name = cth, node = node(), + data = {?MODULE, post_end_per_testcase, + {TC,diff_failed}}}) + end, + {Return, State}. + +%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group, +%% post_end_per_group and post_end_per_tc if the suite, group or test case failed. +%% This function should be used for extra cleanup which might be needed. +%% It is not possible to modify the config or the status of the test run. +-spec on_tc_fail(TC :: init_per_suite | end_per_suite | + init_per_group | end_per_group | atom(), + Reason :: term(), State :: #state{}) -> + NewState :: #state{}. +on_tc_fail(TC, Reason, State) -> + State. + +%% @doc Called when a test case is skipped by either user action +%% or due to an init function failing. Test case can be +%% end_per_suite, init_per_group, end_per_group and the actual test cases. +-spec on_tc_skip(TC :: end_per_suite | + init_per_group | end_per_group | atom(), + {tc_auto_skip, {failed, {Mod :: atom(), Function :: atom(), Reason :: term()}}} | + {tc_user_skip, {skipped, Reason :: term()}}, + State :: #state{}) -> + NewState :: #state{}. +on_tc_skip(TC, Reason, State) -> + State. + +%% @doc Called when the scope of the CTH is done, this depends on +%% when the CTH was specified. This translation table describes when this +%% function is called. +%% +%% | Started in | terminate called | +%% |---------------------|-------------------------| +%% | command_line | after all tests are run | +%% | test spec | after all tests are run | +%% | suite/0 | after SUITE is done | +%% | init_per_suite/1 | after SUITE is done | +%% | init_per_group/2 | after group is done | +%% |-----------------------------------------------| +%% +-spec terminate(State :: #state{}) -> + term(). +terminate(State) -> + ok. diff --git a/lib/common_test/test/ct_group_leader_SUITE.erl b/lib/common_test/test/ct_group_leader_SUITE.erl new file mode 100644 index 0000000000..cde3061d6a --- /dev/null +++ b/lib/common_test/test/ct_group_leader_SUITE.erl @@ -0,0 +1,181 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_system_error_SUITE +%%% +%%% Description: +%%% +%%% Test the group leader functionality in the test_server application. +%%%------------------------------------------------------------------- +-module(ct_group_leader_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config1 = ct_test_support:init_per_suite(Config), + Config1. + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [ + basic + ]. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%%%----------------------------------------------------------------- +%%% +basic(Config) -> + TC = basic, + DataDir = ?config(data_dir, Config), + Suite = filename:join(DataDir, "group_leader_SUITE"), + {Opts,ERPid} = setup([{suite,Suite},{label,TC}], Config), + SuiteLog = execute(TC, Opts, ERPid, Config), + {ok,Data} = file:read_file(SuiteLog), + Lines = binary:split(Data, <<"\n">>, [global]), + {ok,RE} = re:compile("(\\S+):(\\S+)$"), + Cases0 = [begin + {match,[M,F]} = re:run(Case, RE, [{capture,all_but_first,list}]), + {list_to_atom(M),list_to_atom(F)} + end || <<"=case ",Case/binary>> <- Lines], + Cases = [MF || {_,F}=MF <- Cases0, + F =/= init_per_suite, + F =/= end_per_suite, + F =/= init_per_group, + F =/= end_per_group], + io:format("~p\n", [Cases]), + [] = verify_cases(events_to_check(TC), Cases, false), + ok. + +verify_cases([{parallel,P}|Ts], Cases0, Par) -> + Cases = verify_cases(P, Cases0, true), + verify_cases(Ts, Cases, Par); +verify_cases([{?eh,tc_done,{M,F,_}}|Ts], Cases0, false) -> + [{M,F}|Cases] = Cases0, + verify_cases(Ts, Cases, false); +verify_cases([{?eh,tc_done,{M,F,_}}|Ts], Cases0, true) -> + case lists:member({M,F}, Cases0) of + true -> + Cases = Cases0 -- [{M,F}], + verify_cases(Ts, Cases, true); + false -> + io:format("~p not found\n", [{M,F}]), + ?t:fail() + end; +verify_cases([{?eh,_,_}|Ts], Cases, Par) -> + verify_cases(Ts, Cases, Par); +verify_cases([], Cases, _) -> + Cases; +verify_cases([List|Ts], Cases0, Par) when is_list(List) -> + Cases = verify_cases(List, Cases0, false), + verify_cases(Ts, Cases, Par). + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +execute(Name, Opts, ERPid, Config) -> + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(Name, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(Name), + ok = ct_test_support:verify_events(TestEvents, Events, Config), + {event,tc_logfile,_,{_,File}} = + lists:keyfind(tc_logfile, 2, [Ev || {?eh,Ev} <- Events]), + LogDir = filename:dirname(File), + filename:join(LogDir, "suite.log"). + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +%%%----------------------------------------------------------------- + +events_to_check(_Test) -> + [{?eh,tc_done,{group_leader_SUITE,tc1,ok}}, + {parallel,[{?eh,tc_start,{group_leader_SUITE,p1}}, + {?eh,tc_done,{group_leader_SUITE,p1,ok}}, + {?eh,tc_start,{group_leader_SUITE,p2}}, + {?eh,tc_done,{group_leader_SUITE,p2,ok}}]}, + {?eh,tc_done,{group_leader_SUITE,p_restart_my_io_server,ok}}, + {?eh,tc_done,{group_leader_SUITE,p3,ok}}, + {parallel,[ + {?eh,tc_start,{group_leader_SUITE,p10}}, + {?eh,tc_start,{group_leader_SUITE,p11}}, + {?eh,tc_done,{group_leader_SUITE,p10,ok}}, + {?eh,tc_done,{group_leader_SUITE,p11,ok}}, + [{?eh,tc_done,{group_leader_SUITE,s1,ok}}, + {?eh,tc_done,{group_leader_SUITE,s2,ok}}, + {?eh,tc_done,{group_leader_SUITE,s3,ok}}], + {?eh,tc_start,{group_leader_SUITE,p12}}, + {?eh,tc_done,{group_leader_SUITE,p12,ok}}, + [{?eh,tc_done,{group_leader_SUITE,s4,ok}}, + {?eh,tc_done,{group_leader_SUITE,s5,ok}}], + {?eh,tc_start,{group_leader_SUITE,p13}}, + {?eh,tc_done,{group_leader_SUITE,p13,ok}} ]}, + {?eh,tc_done,{group_leader_SUITE,cap1,ok}}, + {?eh,tc_done,{group_leader_SUITE,cap2,ok}}, + {parallel,[{?eh,tc_start,{group_leader_SUITE,cap1}}, + {?eh,tc_done,{group_leader_SUITE,cap1,ok}}, + {?eh,tc_start,{group_leader_SUITE,cap2}}, + {?eh,tc_done,{group_leader_SUITE,cap2,ok}}]}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]. diff --git a/lib/common_test/test/ct_group_leader_SUITE_data/group_leader_SUITE.erl b/lib/common_test/test/ct_group_leader_SUITE_data/group_leader_SUITE.erl new file mode 100644 index 0000000000..3f1844b4ae --- /dev/null +++ b/lib/common_test/test/ct_group_leader_SUITE_data/group_leader_SUITE.erl @@ -0,0 +1,252 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(group_leader_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% @spec suite() -> Info +%% Info = [tuple()] +%% @end +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{seconds,10}}]. + +%%-------------------------------------------------------------------- +%% @spec init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + start_my_io_server(), + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%% @end +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + my_io_server ! die, + ok. + +%%-------------------------------------------------------------------- +%% @spec init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% @end +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} | {fail,Reason} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +end_per_testcase(_TestCase, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%% @end +%%-------------------------------------------------------------------- +groups() -> + [{p,[parallel],[p1,p2]}, + {p_restart,[parallel],[p_restart_my_io_server]}, + {seq,[],[s1,s2,s3]}, + {seq2,[],[s4,s5]}, + {seq_in_par,[parallel],[p10,p11,{group,seq},p12,{group,seq2},p13]}, + {capture_io,[parallel],[cap1,cap2]}]. + +%%-------------------------------------------------------------------- +%% @spec all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +all() -> + [tc1,{group,p},{group,p_restart},p3, + {group,seq_in_par}, + cap1,cap2, + {group,capture_io}]. + +tc1(_C) -> + ok. + +p1(_) -> + %% OTP-10101: + %% + %% External apps/processes started by init_per_suite (common operation), + %% will inherit the group leader of the init_per_suite process, i.e. the + %% test_server test case control process (executing run_test_case_msgloop/7). + %% If, later, a parallel test case triggers the external app to print with + %% e.g. io:format() (also common operation), the calling process will hang! + %% The reason for this is that a parallel test case has a dedicated IO + %% server process, other than the central test case control process. The + %% latter process is not executing run_test_case_msgloop/7 and will not + %% respond to IO messages. The process is still group leader for the + %% external app, however, which is wrong. It's the IO process for the + %% parallel test case that should be group leader - but only for the + %% particular invokation, since other parallel test cases could be + %% invoking the external app too. + print("hej\n"). + +p2(_) -> + print("hopp\n"). + +p_restart_my_io_server(_) -> + %% Restart the IO server and change its group leader. This used + %% to set to the group leader to a process that would soon die. + Ref = erlang:monitor(process, my_io_server), + my_io_server ! die, + receive + {'DOWN',Ref,_,_,_} -> + start_my_io_server() + end. + +p3(_) -> + %% OTP-10125. This would crash since the group leader process + %% for the my_io_server had died. + print("hoppsan\n"). + +print(String) -> + my_io_server ! {print,self(),String}, + receive + {printed,String} -> + ok + end. + +start_my_io_server() -> + Parent = self(), + Pid = spawn(fun() -> my_io_server(Parent) end), + receive + {Pid,started} -> + io:format("~p\n", [process_info(Pid)]), + ok + end. + +my_io_server(Parent) -> + register(my_io_server, self()), + Parent ! {self(),started}, + my_io_server_loop(). + +my_io_server_loop() -> + receive + {print,From,String} -> + io:put_chars(String), + From ! {printed,String}, + my_io_server_loop(); + die -> + ok + end. + +p10(_) -> + receive after 1 -> ok end. + +p11(_) -> + ok. + +p12(_) -> + ok. + +p13(_) -> + ok. + +s1(_) -> + ok. + +s2(_) -> + ok. + +s3(_) -> + ok. + +s4(_) -> + ok. + +s5(_) -> + ok. + +cap1(_) -> + ct:capture_start(), + IO = gen_io(cap1, 10, []), + ct:capture_stop(), + IO = ct:capture_get(), + ok. + +cap2(_) -> + ct:capture_start(), + {Pid,Ref} = spawn_monitor(fun() -> + exit(gen_io(cap2, 42, [])) + end), + receive + {'DOWN',Ref,process,Pid,IO} -> + ct:capture_stop(), + IO = ct:capture_get(), + ok + end. + +gen_io(_, 0, Acc) -> + lists:reverse(Acc); +gen_io(Label, N, Acc) -> + S = lists:flatten(io_lib:format("~s: ~p\n", [Label,N])), + io:put_chars(S), + gen_io(Label, N-1, [S|Acc]). diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl index 405df1e978..796a0832d7 100644 --- a/lib/common_test/test/ct_hooks_SUITE.erl +++ b/lib/common_test/test/ct_hooks_SUITE.erl @@ -64,7 +64,7 @@ end_per_testcase(TestCase, Config) -> suite() -> - [{timetrap,{seconds,20}}]. + [{timetrap,{minutes,1}}]. all() -> all(suite). diff --git a/lib/common_test/test/ct_master_SUITE.erl b/lib/common_test/test/ct_master_SUITE.erl index 27243a0067..7408cbe376 100644 --- a/lib/common_test/test/ct_master_SUITE.erl +++ b/lib/common_test/test/ct_master_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2012. All Rights Reserved. +%% Copyright Ericsson AB 2010-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -109,7 +109,7 @@ ct_master_test(Config) when is_list(Config) -> ERPid = ct_test_support:start_event_receiver(Config), - [{TSFile,ok}] = run_test(ct_master_test, FileName, Config), + [{[TSFile],ok}] = run_test(ct_master_test, FileName, Config), Events = ct_test_support:get_events(ERPid, Config), @@ -117,14 +117,8 @@ ct_master_test(Config) when is_list(Config) -> reformat(Events, ?eh), PrivDir, []), - find_events(NodeNames, [{tc_start,{master_SUITE,init_per_suite}}, - {tc_start,{master_SUITE,first_testcase}}, - {tc_start,{master_SUITE,second_testcase}}, - {tc_start,{master_SUITE,third_testcase}}, - {tc_start,{master_SUITE,end_per_suite}}], - Events), - - ok. + TestEvents = events_to_check(ct_master_test), + ok = find_events(NodeNames, TestEvents, Events, Config). %%%----------------------------------------------------------------- %%% HELP FUNCTIONS @@ -153,13 +147,18 @@ make_spec(DataDir, FileName, NodeNames, Suites, Config) -> CM = [{config,master,filename:join(DataDir,"master/config.txt")}], + Env = [{"THIS_MUST_BE_SET","yes"}, + {"SO_MUST_THIS","value"}], NS = lists:map( fun(NodeName) -> {init,NodeName,[ {node_start,[{startup_functions,[]}, - {monitor_master,true}]}, - {eval,{erlang,nodes,[]}} - ] + {monitor_master,true}, + {boot_timeout,10}, + {init_timeout,10}, + {startup_timeout,10}, + {env,Env}]}, + {eval,{erlang,nodes,[]}}] } end, NodeNames), @@ -196,13 +195,12 @@ get_log_dir(_,PrivDir,NodeName) -> run_test(_Name, FileName, Config) -> %% run the test twice, using different html versions - [{FileName,ok}] = ct_test_support:run({ct_master,run,[FileName]}, - [{ct_master,basic_html,[true]}], - Config), - timer:sleep(5000), - [{FileName,ok}] = ct_test_support:run({ct_master,run,[FileName]}, - [{ct_master,basic_html,[false]}], - Config). + [{[FileName],ok}] = ct_test_support:run({ct_master,run,[FileName]}, + [{ct_master,basic_html,[true]}], + Config), + [{[FileName],ok}] = ct_test_support:run({ct_master,run,[FileName]}, + [{ct_master,basic_html,[false]}], + Config). reformat(Events, EH) -> ct_test_support:reformat(Events, EH). @@ -210,28 +208,26 @@ reformat(Events, EH) -> %%%----------------------------------------------------------------- %%% TEST EVENTS %%%----------------------------------------------------------------- -find_events([], _CheckEvents, _) -> - ok; -find_events([NodeName|NodeNames],CheckEvents,AllEvents) -> - find_events(NodeNames, CheckEvents, - remove_events(add_host(NodeName),CheckEvents, AllEvents, [])). - -remove_events(Node,[{Name,Data} | RestChecks], - [{?eh,#event{ name = Name, node = Node, data = Data }}|RestEvs], - Acc) -> - remove_events(Node, RestChecks, RestEvs, Acc); -remove_events(Node, Checks, [Event|RestEvs], Acc) -> - remove_events(Node, Checks, RestEvs, [Event | Acc]); -remove_events(_Node, [], [], Acc) -> - lists:reverse(Acc); -remove_events(Node, Events, [], Acc) -> - test_server:format("Could not find events: ~p in ~p for node ~p", - [Events, lists:reverse(Acc), Node]), - exit(event_not_found). + +find_events(NodeNames, TestEvents, Events, Config) -> + [begin + Node = add_host(Node0), + io:format("Searching for events for node: ~s", [Node]), + ok = ct_test_support:verify_events(TestEvents, Events, Node, Config), + io:nl() + end || Node0 <- NodeNames], + ok. add_host(NodeName) -> {ok, HostName} = inet:gethostname(), list_to_atom(atom_to_list(NodeName)++"@"++HostName). -expected_events(_) -> - []. +events_to_check(_) -> + [{?eh,tc_start,{master_SUITE,first_testcase}}, + {?eh,tc_done,{master_SUITE,first_testcase,ok}}, + {?eh,tc_start,{master_SUITE,second_testcase}}, + {?eh,tc_done,{master_SUITE,second_testcase,ok}}, + {?eh,tc_start,{master_SUITE,third_testcase}}, + {?eh,tc_done,{master_SUITE,third_testcase,ok}}, + {?eh,tc_start,{master_SUITE,env_vars}}, + {?eh,tc_done,{master_SUITE,env_vars,ok}}]. diff --git a/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl b/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl index 032d69ad9f..df54c4419c 100644 --- a/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl +++ b/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010. All Rights Reserved. +%% Copyright Ericsson AB 2010-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -39,7 +39,8 @@ init_per_suite(Config) -> end_per_suite(_) -> ok. -all() -> [first_testcase, second_testcase, third_testcase]. +all() -> [first_testcase, second_testcase, third_testcase, + env_vars]. init_per_testcase(_, Config) -> Config. @@ -56,3 +57,9 @@ second_testcase(_)-> third_testcase(_)-> A = 4, A = 2*2. + +env_vars(_) -> + io:format("~p\n", [os:getenv()]), + "yes" = os:getenv("THIS_MUST_BE_SET"), + "value" = os:getenv("SO_MUST_THIS"), + ok. diff --git a/lib/common_test/test/ct_netconfc_SUITE.erl b/lib/common_test/test/ct_netconfc_SUITE.erl index 3042a924fe..c89a4cdabe 100644 --- a/lib/common_test/test/ct_netconfc_SUITE.erl +++ b/lib/common_test/test/ct_netconfc_SUITE.erl @@ -43,12 +43,11 @@ %% there will be clashes with logging processes etc). %%-------------------------------------------------------------------- init_per_suite(Config) -> - Config1 = ct_test_support:init_per_suite(Config), case application:load(crypto) of - {error,Reason} -> + {error,Reason} when Reason=/={already_loaded,crypto} -> {skip, Reason}; _ -> - Config1 + ct_test_support:init_per_suite(Config) end. end_per_suite(Config) -> diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl index d337158bce..54526e8e83 100644 --- a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl +++ b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl @@ -1044,12 +1044,9 @@ gen_dsa(LSize,NSize) when is_integer(LSize), is_integer(NSize) -> Key = gen_dsa2(LSize, NSize), {Key, encode_key(Key)}. -encode_key(Key = #'RSAPrivateKey'{}) -> - {ok, Der} = 'OTP-PUB-KEY':encode('RSAPrivateKey', Key), - {'RSAPrivateKey', list_to_binary(Der), not_encrypted}; encode_key(Key = #'DSAPrivateKey'{}) -> - {ok, Der} = 'OTP-PUB-KEY':encode('DSAPrivateKey', Key), - {'DSAPrivateKey', list_to_binary(Der), not_encrypted}. + Der = public_key:der_encode('DSAPrivateKey', Key), + {'DSAPrivateKey', Der, not_encrypted}. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl index 2427f37f52..09217f60a3 100644 --- a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl +++ b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2012. All Rights Reserved. +%% Copyright Ericsson AB 2012-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -426,7 +426,7 @@ reply(ConnRef,Reply) -> send(ConnRef, make_msg(Reply)). from_simple(Simple) -> - list_to_binary(xmerl:export_simple_element(Simple,xmerl_xml)). + unicode_c2b(xmerl:export_simple_element(Simple,xmerl_xml)). xml(Content) -> <<"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n", @@ -435,30 +435,30 @@ xml(Content) -> rpc_reply(Content) when is_binary(Content) -> MsgId = case erase(msg_id) of undefined -> <<>>; - Id -> list_to_binary([" message-id=\"",Id,"\""]) + Id -> unicode_c2b([" message-id=\"",Id,"\""]) end, <<"<rpc-reply xmlns=\"",?NETCONF_NAMESPACE,"\"",MsgId/binary,">\n", Content/binary,"\n</rpc-reply>">>; rpc_reply(Content) -> - rpc_reply(list_to_binary(Content)). + rpc_reply(unicode_c2b(Content)). session_id(no_session_id) -> <<>>; session_id(SessionId0) -> - SessionId = list_to_binary(integer_to_list(SessionId0)), + SessionId = unicode_c2b(integer_to_list(SessionId0)), <<"<session-id>",SessionId/binary,"</session-id>\n">>. capabilities(undefined) -> - CapsXml = list_to_binary([["<capability>",C,"</capability>\n"] + CapsXml = unicode_c2b([["<capability>",C,"</capability>\n"] || C <- ?CAPABILITIES]), <<"<capabilities>\n",CapsXml/binary,"</capabilities>\n">>; capabilities({base,Vsn}) -> - CapsXml = list_to_binary([["<capability>",C,"</capability>\n"] + CapsXml = unicode_c2b([["<capability>",C,"</capability>\n"] || C <- ?CAPABILITIES_VSN(Vsn)]), <<"<capabilities>\n",CapsXml/binary,"</capabilities>\n">>; capabilities(no_base) -> [_|Caps] = ?CAPABILITIES, - CapsXml = list_to_binary([["<capability>",C,"</capability>\n"] || C <- Caps]), + CapsXml = unicode_c2b([["<capability>",C,"</capability>\n"] || C <- Caps]), <<"<capabilities>\n",CapsXml/binary,"</capabilities>\n">>; capabilities(no_caps) -> <<>>. @@ -553,3 +553,8 @@ make_msg(Xml) when is_binary(Xml) -> xml(Xml); make_msg(Simple) when is_tuple(Simple) -> xml(from_simple(Simple)). + +%%%----------------------------------------------------------------- +%%% Convert to unicode binary, since we use UTF-8 encoding in XML +unicode_c2b(Characters) -> + unicode:characters_to_binary(Characters). diff --git a/lib/common_test/test/ct_snmp_SUITE.erl b/lib/common_test/test/ct_snmp_SUITE.erl new file mode 100644 index 0000000000..f8b4543770 --- /dev/null +++ b/lib/common_test/test/ct_snmp_SUITE.erl @@ -0,0 +1,141 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_snmp_SUITE +%%% +%%% Description: +%%% Test ct_snmp module +%%% +%%%------------------------------------------------------------------- +-module(ct_snmp_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config1 = ct_test_support:init_per_suite(Config), + Config1. + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [ + default + ]. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%%%----------------------------------------------------------------- +%%% +default(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + Suite = filename:join(DataDir, "snmp_SUITE"), + CfgFile = filename:join(DataDir, "snmp.cfg"), + {Opts,ERPid} = setup([{suite,Suite},{config,CfgFile}, + {label,default}], Config), + + ok = execute(default, Opts, ERPid, Config). + + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +execute(Name, Opts, ERPid, Config) -> + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(Name, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(Name,Config), + ct_test_support:verify_events(TestEvents, Events, Config). + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +%%%----------------------------------------------------------------- +events_to_check(_TestName,Config) -> + {module,_} = code:load_abs(filename:join(?config(data_dir,Config), + snmp_SUITE)), + TCs = get_tcs(), + code:purge(snmp_SUITE), + code:delete(snmp_SUITE), + + OneTest = + [{?eh,start_logging,{'DEF','RUNDIR'}}] ++ + [{?eh,tc_done,{snmp_SUITE,TC,ok}} || TC <- TCs] ++ + [{?eh,stop_logging,[]}], + + %% 2 tests (ct:run_test + script_start) is default + OneTest ++ OneTest. + + +get_tcs() -> + All = snmp_SUITE:all(), + Groups = + try snmp_SUITE:groups() + catch error:undef -> [] + end, + flatten_tcs(All,Groups). + +flatten_tcs([H|T],Groups) when is_atom(H) -> + [H|flatten_tcs(T,Groups)]; +flatten_tcs([{group,Group}|T],Groups) -> + TCs = proplists:get_value(Group,Groups), + flatten_tcs(TCs ++ T,Groups); +flatten_tcs([],_) -> + []. diff --git a/lib/common_test/test/ct_snmp_SUITE_data/snmp.cfg b/lib/common_test/test/ct_snmp_SUITE_data/snmp.cfg new file mode 100644 index 0000000000..895e097de6 --- /dev/null +++ b/lib/common_test/test/ct_snmp_SUITE_data/snmp.cfg @@ -0,0 +1,44 @@ +%% -*- erlang -*- +{snmp1, [{start_agent,true}, + {users,[{user_name,[snmpm_user_default,[]]}]}, + {managed_agents,[{agent_name, [user_name, {127,0,0,1}, 4000, + [{engine_id,"ct_snmp-test-engine"}, + {version,v2}]]}]}, + {engine_id,"ct_snmp-test-engine"}, + {agent_vsns,[v2]} + ]}. +{snmp2, [{start_agent,true}, + {engine_id,"ct_snmp-test-engine"} + ]}. +{snmp3, [{start_agent,true}, + {engine_id,"ct_snmp-test-engine"}, + {agent_vsns,[v1,v2,v3]}, + {agent_contexts,{data_dir_file,"context.conf"}}, + {agent_usm,{data_dir_file,"usm.conf"}}, + {agent_community,{data_dir_file,"community.conf"}}, + {agent_notify_def,{data_dir_file,"notify.conf"}}, + {agent_sysinfo,{data_dir_file,"standard.conf"}}, + {agent_target_address_def,{data_dir_file,"target_addr.conf"}}, + {agent_target_param_def,{data_dir_file,"target_params.conf"}}, + {agent_vacm,{data_dir_file,"vacm.conf"}}]}. +{snmp_app1,[{manager, [{config, [{verbosity, silence}]}, + {server,[{verbosity,silence}]}, + {net_if,[{verbosity,silence}]}, + {versions,[v2]} + ]}, + {agent, [{config, [{verbosity, silence}]}, + {net_if,[{verbosity,silence}]}, + {mib_server,[{verbosity,silence}]}, + {local_db,[{verbosity,silence}]}, + {agent_verbosity,silence} + ]}]}. +{snmp_app2,[{manager, [{config, [{verbosity, silence}]}, + {server,[{verbosity,silence}]}, + {net_if,[{verbosity,silence}]} + ]}, + {agent, [{config, [{verbosity, silence}]}, + {net_if,[{verbosity,silence}]}, + {mib_server,[{verbosity,silence}]}, + {local_db,[{verbosity,silence}]}, + {agent_verbosity,silence} + ]}]}. diff --git a/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE.erl b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE.erl new file mode 100644 index 0000000000..16b2b5690c --- /dev/null +++ b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE.erl @@ -0,0 +1,395 @@ +%%-------------------------------------------------------------------- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%%---------------------------------------------------------------------- +%% File: ct_snmp_SUITE.erl +%% +%% Description: +%% This file contains the test cases for the ct_snmp API. +%% +%% @author Support +%% @doc Test of SNMP support in common_test +%% @end +%%---------------------------------------------------------------------- +%%---------------------------------------------------------------------- +-module(snmp_SUITE). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snmp/include/STANDARD-MIB.hrl"). +-include_lib("snmp/include/SNMP-USER-BASED-SM-MIB.hrl"). +-include_lib("snmp/include/snmp_types.hrl"). + +-compile(export_all). + +%% Default timetrap timeout (set in init_per_testcase). +-define(default_timeout, ?t:minutes(1)). + +-define(AGENT_UDP, 4000). + +suite() -> + [ + {require, snmp1, snmp1}, + {require, snmp_app1, snmp_app1}, + {require, snmp2, snmp2}, + {require, snmp_app2, snmp_app2}, + {require, snmp3, snmp3} + ]. + +all() -> + [start_stop, + {group,get_set}, + {group,register}, + {group,override}, + set_info]. + + +groups() -> + [{get_set,[get_values, + get_next_values, + set_values, + load_mibs]}, + {register,[register_users, + register_users_fail, + register_agents, + register_agents_fail, + register_usm_users, + register_usm_users_fail]}, + {override,[override_usm, + override_standard, + override_context, + override_community, + override_notify, + override_target_addr, + override_target_params, + override_vacm]}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + Config. + +init_per_group(get_set, Config) -> + ok = ct_snmp:start(Config,snmp1,snmp_app1), + Config; +init_per_group(register, Config) -> + ok = ct_snmp:start(Config,snmp2,snmp_app2), + Config; +init_per_group(_, Config) -> + ok = ct_snmp:start(Config,snmp3,snmp_app2), + Config. + +end_per_group(_Group, Config) -> + catch ct_snmp:stop(Config), + Config. + +init_per_testcase(_Case, Config) -> + Dog = test_server:timetrap(?default_timeout), + [{watchdog, Dog}|Config]. + +end_per_testcase(Case, Config) -> + try apply(?MODULE,Case,[cleanup,Config]) + catch error:undef -> ok + end, + Dog=?config(watchdog, Config), + test_server:timetrap_cancel(Dog), + ok. + +%%%----------------------------------------------------------------- +%%% Test cases +break(_Config) -> + test_server:break(""), + ok. + +start_stop(Config) -> + ok = ct_snmp:start(Config,snmp1,snmp_app1), + timer:sleep(1000), + {snmp,_,_} = lists:keyfind(snmp,1,application:which_applications()), + [_|_] = filelib:wildcard("*/*.conf",?config(priv_dir,Config)), + + ok = ct_snmp:stop(Config), + timer:sleep(1000), + false = lists:keyfind(snmp,1,application:which_applications()), + [] = filelib:wildcard("*/*.conf",?config(priv_dir,Config)), + ok. + +get_values(_Config) -> + Oids1 = [?sysDescr_instance, ?sysName_instance], + {noError,_,V1} = ct_snmp:get_values(agent_name,Oids1,snmp1), + [#varbind{oid=?sysDescr_instance,value="Erlang SNMP agent"}, + #varbind{oid=?sysName_instance,value="ct_test"}] = V1, + ok. + +get_next_values(_Config) -> + Oids2 = [?system], + {noError,_,V2} = ct_snmp:get_next_values(agent_name,Oids2,snmp1), + [#varbind{oid=?sysDescr_instance,value="Erlang SNMP agent"}] = V2, + ok. + +set_values(Config) -> + Oid3 = ?sysName_instance, + NewName = "ct_test changed by " ++ atom_to_list(?MODULE), + VarsAndVals = [{Oid3,s,NewName}], + {noError,_,_} = + ct_snmp:set_values(agent_name,VarsAndVals,snmp1,Config), + + Oids4 = [?sysName_instance], + {noError,_,V4} = ct_snmp:get_values(agent_name,Oids4,snmp1), + [#varbind{oid=?sysName_instance,value=NewName}] = V4, + + ok. + +load_mibs(_Config) -> + [{'SNMPv2-MIB',_}=SnmpV2Mib] = snmpa:which_mibs(), + Mib = filename:join([code:priv_dir(snmp),"mibs","SNMP-USER-BASED-SM-MIB"]), + ok = ct_snmp:load_mibs([Mib]), + TwoMibs = [_,_] = snmpa:which_mibs(), + [{'SNMP-USER-BASED-SM-MIB',_}] = lists:delete(SnmpV2Mib,TwoMibs), + ok = ct_snmp:unload_mibs([Mib]), + [{'SNMPv2-MIB',_}] = snmpa:which_mibs(), + ok. + + +register_users(_Config) -> + [] = snmpm:which_users(), + ok = ct_snmp:register_users(snmp2,[{reg_user1,[snmpm_user_default,[]]}]), + [_] = snmpm:which_users(), + [_] = ct:get_config({snmp2,users}), + ok = ct_snmp:register_users(snmp2,[{reg_user2,[snmpm_user_default,[]]}]), + [_,_] = snmpm:which_users(), + [_,_] = ct:get_config({snmp2,users}), + ok = ct_snmp:register_users(snmp2,[{reg_user3,[snmpm_user_default,[]]}]), + [_,_,_] = snmpm:which_users(), + [_,_,_] = ct:get_config({snmp2,users}), + ok = ct_snmp:unregister_users(snmp2,[reg_user3]), + [_,_] = snmpm:which_users(), + [_,_] = ct:get_config({snmp2,users}), + ok = ct_snmp:unregister_users(snmp2), + [] = snmpm:which_users(), + [] = ct:get_config({snmp2,users}), + ok. +register_users(cleanup,_Config) -> + ct_snmp:unregister_users(snmp2). + +register_users_fail(_Config) -> + [] = snmpm:which_users(), + {error,_} = ct_snmp:register_users(snmp2,[{reg_user3,[unknown_module,[]]}]), + [] = snmpm:which_users(), + ok. +register_users_fail(cleanup,_Config) -> + ct_snmp:unregister_users(snmp2). + +register_agents(_Config) -> + {ok, HostName} = inet:gethostname(), + {ok, Addr} = inet:getaddr(HostName, inet), + + [] = snmpm:which_agents(), + ok = ct_snmp:register_users(snmp2,[{reg_user1,[snmpm_user_default,[]]}]), + ok = ct_snmp:register_agents(snmp2,[{reg_agent1, + [reg_user1,Addr,?AGENT_UDP,[]]}]), + [_] = snmpm:which_agents(), + [_] = ct:get_config({snmp2,managed_agents}), + ok = ct_snmp:register_agents(snmp2,[{reg_agent2, + [reg_user1,Addr,?AGENT_UDP,[]]}]), + [_,_] = snmpm:which_agents(), + [_,_] = ct:get_config({snmp2,managed_agents}), + + ok = ct_snmp:register_agents(snmp2,[{reg_agent3, + [reg_user1,Addr,?AGENT_UDP,[]]}]), + [_,_,_] = snmpm:which_agents(), + [_,_,_] = ct:get_config({snmp2,managed_agents}), + + ok = ct_snmp:unregister_agents(snmp2,[reg_agent3]), + [_,_] = snmpm:which_agents(), + [_,_] = ct:get_config({snmp2,managed_agents}), + + ok = ct_snmp:unregister_agents(snmp2), + ok = ct_snmp:unregister_users(snmp2), + [] = snmpm:which_agents(), + [] = ct:get_config({snmp2,managed_agents}), + ok. +register_agents(cleanup,_Config) -> + ct_snmp:unregister_agents(snmp2), + ct_snmp:unregister_users(snmp2). + +register_agents_fail(_Config) -> + {ok, HostName} = inet:gethostname(), + {ok, Addr} = inet:getaddr(HostName, inet), + + [] = snmpm:which_agents(), + {error,_} + = ct_snmp:register_agents(snmp2,[{reg_agent3, + [unknown_user,Addr,?AGENT_UDP,[]]}]), + [] = snmpm:which_agents(), + ok. +register_agents_fail(cleanup,_Config) -> + ct_snmp:unregister_agents(snmp2). + +register_usm_users(_Config) -> + [] = snmpm:which_usm_users(), + ok = ct_snmp:register_usm_users(snmp2,[{"reg_usm_user1",[]}]), + [_] = snmpm:which_usm_users(), + [_] = ct:get_config({snmp2,usm_users}), + ok = ct_snmp:register_usm_users(snmp2,[{"reg_usm_user2",[]}]), + [_,_] = snmpm:which_usm_users(), + [_,_] = ct:get_config({snmp2,usm_users}), + ok = ct_snmp:register_usm_users(snmp2,[{"reg_usm_user3",[]}]), + [_,_,_] = snmpm:which_usm_users(), + [_,_,_] = ct:get_config({snmp2,usm_users}), + ok = ct_snmp:unregister_usm_users(snmp2,["reg_usm_user3"]), + [_,_] = snmpm:which_usm_users(), + [_,_] = ct:get_config({snmp2,usm_users}), + ok = ct_snmp:unregister_usm_users(snmp2), + [] = snmpm:which_usm_users(), + [] = ct:get_config({snmp2,usm_users}), + ok. +register_usm_users(cleanup,_Config) -> + ct_snmp:unregister_usm_users(snmp2). + +register_usm_users_fail(_Config) -> + [] = snmpm:which_usm_users(), + {error,_} + = ct_snmp:register_usm_users(snmp2,[{"reg_usm_user3", + [{sec_name,invalid_data_type}]}]), + [] = snmpm:which_usm_users(), + ok. +register_usm_users_fail(cleanup,_Config) -> + ct_snmp:unregister_usm_users(snmp2). + +%% Test that functionality for overriding default configuration file +%% works - i.e. that the files are written and that the configuration +%% is actually used. +%% +%% Note that the config files used in this test case do not +%% necessarily make up a reasonable configuration for the snmp +%% application... +override_usm(Config) -> + DataDir = ?config(data_dir,Config), + PrivDir = ?config(priv_dir,Config), + ConfDir = filename:join(PrivDir,"conf"), + + Mib = filename:join([code:priv_dir(snmp),"mibs","SNMP-USER-BASED-SM-MIB"]), + ok = ct_snmp:load_mibs([Mib]), + + %% Check that usm.conf is overwritten + {ok,MyUsm} = snmpa_conf:read_usm_config(DataDir), + {ok,UsedUsm} = snmpa_conf:read_usm_config(ConfDir), + true = (MyUsm == UsedUsm), + + %% Check that the usm user is actually configured... + [{Index,"secname"}] = + snmp_user_based_sm_mib:usmUserTable(get_next,?usmUserEntry,[3]), + true = lists:suffix("usm_user_name",Index), + ok. + +override_standard(Config) -> + DataDir = ?config(data_dir,Config), + PrivDir = ?config(priv_dir,Config), + ConfDir = filename:join(PrivDir,"conf"), + + %% Check that standard.conf is overwritten + {ok,MyStandard} = snmpa_conf:read_standard_config(DataDir), + {ok,UsedStandard} = snmpa_conf:read_standard_config(ConfDir), + true = (MyStandard == UsedStandard), + + %% Check that the values from standard.conf is actually configured... + {value,"name for override test"} = snmp_standard_mib:sysName(get), + {value,"agent for ct_snmp override test"} = snmp_standard_mib:sysDescr(get), + ok. + +override_context(Config) -> + DataDir = ?config(data_dir,Config), + PrivDir = ?config(priv_dir,Config), + ConfDir = filename:join(PrivDir,"conf"), + + %% Check that context.conf is overwritten + {ok,MyContext} = snmpa_conf:read_context_config(DataDir), + {ok,UsedContext} = snmpa_conf:read_context_config(ConfDir), + true = (MyContext == UsedContext), + ok. + +override_community(Config) -> + DataDir = ?config(data_dir,Config), + PrivDir = ?config(priv_dir,Config), + ConfDir = filename:join(PrivDir,"conf"), + + %% Check that community.conf is overwritten + {ok,MyCommunity} = snmpa_conf:read_community_config(DataDir), + {ok,UsedCommunity} = snmpa_conf:read_community_config(ConfDir), + true = (MyCommunity == UsedCommunity), + ok. + +override_notify(Config) -> + DataDir = ?config(data_dir,Config), + PrivDir = ?config(priv_dir,Config), + ConfDir = filename:join(PrivDir,"conf"), + + %% Check that notify.conf is overwritten + {ok,MyNotify} = snmpa_conf:read_notify_config(DataDir), + {ok,UsedNotify} = snmpa_conf:read_notify_config(ConfDir), + true = (MyNotify == UsedNotify), + ok. + +override_target_addr(Config) -> + DataDir = ?config(data_dir,Config), + PrivDir = ?config(priv_dir,Config), + ConfDir = filename:join(PrivDir,"conf"), + + %% Check that target_addr.conf is overwritten + {ok,MyTargetAddr} = snmpa_conf:read_target_addr_config(DataDir), + {ok,UsedTargetAddr} = snmpa_conf:read_target_addr_config(ConfDir), + true = (MyTargetAddr == UsedTargetAddr), + ok. + +override_target_params(Config) -> + DataDir = ?config(data_dir,Config), + PrivDir = ?config(priv_dir,Config), + ConfDir = filename:join(PrivDir,"conf"), + + %% Check that target_params.conf is overwritten + {ok,MyTargetParams} = snmpa_conf:read_target_params_config(DataDir), + {ok,UsedTargetParams} = snmpa_conf:read_target_params_config(ConfDir), + true = (MyTargetParams == UsedTargetParams), + ok. + +override_vacm(Config) -> + DataDir = ?config(data_dir,Config), + PrivDir = ?config(priv_dir,Config), + ConfDir = filename:join(PrivDir,"conf"), + + %% Check that vacm.conf is overwritten + {ok,MyVacm} = snmpa_conf:read_vacm_config(DataDir), + {ok,UsedVacm} = snmpa_conf:read_vacm_config(ConfDir), + true = (MyVacm == UsedVacm), + ok. + + + + +%% NOTE!! This test must always be executed last in the suite, and +%% should match all set requests performed in the suite. I.e. if you +%% add a set request, you must add an entry in the return value of +%% ct_snmp:set_info/1 below. +set_info(Config) -> + %% From test case set_values/1: + Oid1 = ?sysName_instance, + NewValue1 = "ct_test changed by " ++ atom_to_list(?MODULE), + + %% The test... + [{_AgentName,_,[{Oid1,_,NewValue1}]}] + = ct_snmp:set_info(Config), + ok. diff --git a/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/community.conf b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/community.conf new file mode 100644 index 0000000000..5a64df6605 --- /dev/null +++ b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/community.conf @@ -0,0 +1 @@ +{"public", "public", "initial", "", ""}. diff --git a/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/context.conf b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/context.conf new file mode 100644 index 0000000000..feed5e1d11 --- /dev/null +++ b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/context.conf @@ -0,0 +1 @@ +"testcontext". diff --git a/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/notify.conf b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/notify.conf new file mode 100644 index 0000000000..367ba3aa4b --- /dev/null +++ b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/notify.conf @@ -0,0 +1 @@ +{"standard inform", "std_inform", inform}. diff --git a/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/standard.conf b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/standard.conf new file mode 100644 index 0000000000..79908fb355 --- /dev/null +++ b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/standard.conf @@ -0,0 +1,7 @@ +{sysDescr, "agent for ct_snmp override test"}. +{sysObjectID, [1,2,3]}. +{sysContact, "[email protected]"}. +{sysLocation, "erlang"}. +{sysServices, 72}. +{snmpEnableAuthenTraps, enabled}. +{sysName, "name for override test"}. diff --git a/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/target_addr.conf b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/target_addr.conf new file mode 100644 index 0000000000..d02672a074 --- /dev/null +++ b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/target_addr.conf @@ -0,0 +1,2 @@ +{"target1", snmpUDPDomain, [147,214,122,73], 5000, 1500, 3, "std_trap", "target_v3", "", [], 2048}. +{"target2", snmpUDPDomain, [147,214,122,73], 5000, 1500, 3, "std_inform", "target_v3", "", [], 2048}. diff --git a/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/target_params.conf b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/target_params.conf new file mode 100644 index 0000000000..5a9a619422 --- /dev/null +++ b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/target_params.conf @@ -0,0 +1 @@ +{"target_v3", v3, usm, "initial", noAuthNoPriv}. diff --git a/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/usm.conf b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/usm.conf new file mode 100644 index 0000000000..d6e245914e --- /dev/null +++ b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/usm.conf @@ -0,0 +1 @@ +{"ct_snmp-test-engine","usm_user_name","secname",zeroDotZero,usmNoAuthProtocol,"","",usmNoPrivProtocol,"","","","",""}. diff --git a/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/vacm.conf b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/vacm.conf new file mode 100644 index 0000000000..158fe02e3b --- /dev/null +++ b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/vacm.conf @@ -0,0 +1,6 @@ +{vacmSecurityToGroup, usm, "initial", "initial"}. +{vacmAccess, "initial", "", any, noAuthNoPriv, exact, "restricted", "", "restricted"}. +{vacmAccess, "initial", "", usm, authNoPriv, exact, "internet", "internet", "internet"}. +{vacmAccess, "initial", "", usm, authPriv, exact, "internet", "internet", "internet"}. +{vacmViewTreeFamily, "restricted", [1,3,6,1], included, null}. +{vacmViewTreeFamily, "internet", [1,3,6,1], included, null}. diff --git a/lib/common_test/test/ct_surefire_SUITE.erl b/lib/common_test/test/ct_surefire_SUITE.erl new file mode 100644 index 0000000000..b86b47f0a2 --- /dev/null +++ b/lib/common_test/test/ct_surefire_SUITE.erl @@ -0,0 +1,374 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012-2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_surefire_SUITE +%%% +%%% Description: +%%% Test cth_surefire hook +%%% +%%%------------------------------------------------------------------- +-module(ct_surefire_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-include_lib("xmerl/include/xmerl.hrl"). +-include_lib("kernel/include/file.hrl"). + +-define(eh, ct_test_support_eh). + +-define(url_base,"http://my.host.com/"). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config1 = ct_test_support:init_per_suite(Config), + Config1. + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [ + default, + absolute_path, + relative_path, + url, + logdir + ]. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%%%----------------------------------------------------------------- +%%% +default(Config) when is_list(Config) -> + run(default,[cth_surefire],"junit_report.xml",Config). + +absolute_path(Config) when is_list(Config) -> + PrivDir = ?config(priv_dir,Config), + Path = filename:join(PrivDir,"abspath.xml"), + run(absolute_path,[{cth_surefire,[{path,Path}]}],Path,Config). + +relative_path(Config) when is_list(Config) -> + Path = "relpath.xml", + run(relative_path,[{cth_surefire,[{path,Path}]}],Path,Config). + +url(Config) when is_list(Config) -> + Path = "url.xml", + run(url,[{cth_surefire,[{url_base,?url_base},{path,Path}]}], + Path,Config). + +logdir(Config) when is_list(Config) -> + Opts = ct_test_support:get_opts(Config), + LogDir = + case lists:keyfind(logdir,1,Opts) of + {logdir,LD} -> LD; + false -> ?config(priv_dir,Config) + end, + MyLogDir = filename:join(LogDir,"specific_logdir"), + ensure_exists_empty(MyLogDir), + Path = "logdir.xml", + run(logdir,[{cth_surefire,[{path,Path}]}],Path,Config,[{logdir,MyLogDir}]). + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- +run(Case,CTHs,Report,Config) -> + run(Case,CTHs,Report,Config,[]). +run(Case,CTHs,Report,Config,ExtraOpts) -> + DataDir = ?config(data_dir, Config), + Suite = filename:join(DataDir, "surefire_SUITE"), + {Opts,ERPid} = setup([{suite,Suite},{ct_hooks,CTHs},{label,Case}|ExtraOpts], + Config), + ok = execute(Case, Opts, ERPid, Config), + LogDir = + case lists:keyfind(logdir,1,Opts) of + {logdir,LD} -> LD; + false -> ?config(priv_dir,Config) + end, + Re = filename:join([LogDir,"*",Report]), + check_xml(Case,Re). + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Opts1 = + case lists:keymember(logdir,1,Test) of + true -> lists:keydelete(logdir,1,Opts0); + false -> Opts0 + end, + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts1 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +execute(Name, Opts, ERPid, Config) -> + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(Name, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(Name), + ct_test_support:verify_events(TestEvents, Events, Config). + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +%%%----------------------------------------------------------------- +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). + +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + test_events(Test) ++ events_to_check(Test, N-1). + +test_events(_) -> + [{?eh,start_logging,'_'}, + {?eh,start_info,{1,1,9}}, + {?eh,tc_start,{surefire_SUITE,init_per_suite}}, + {?eh,tc_done,{surefire_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{surefire_SUITE,tc_ok}}, + {?eh,tc_done,{surefire_SUITE,tc_ok,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{surefire_SUITE,tc_fail}}, + {?eh,tc_done,{surefire_SUITE,tc_fail, + {failed,{error,{test_case_failed,"this test should fail"}}}}}, + {?eh,test_stats,{1,1,{0,0}}}, + {?eh,tc_start,{surefire_SUITE,tc_skip}}, + {?eh,tc_done,{surefire_SUITE,tc_skip,{skipped,"this test is skipped"}}}, + {?eh,test_stats,{1,1,{1,0}}}, + {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, + {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, + {skipped,{require_failed,'_'}}}}, + {?eh,test_stats,{1,1,{1,1}}}, + [{?eh,tc_start,{surefire_SUITE,{init_per_group,g,[]}}}, + {?eh,tc_done,{surefire_SUITE,{init_per_group,g,[]},ok}}, + {?eh,tc_start,{surefire_SUITE,tc_ok}}, + {?eh,tc_done,{surefire_SUITE,tc_ok,ok}}, + {?eh,test_stats,{2,1,{1,1}}}, + {?eh,tc_start,{surefire_SUITE,tc_fail}}, + {?eh,tc_done,{surefire_SUITE,tc_fail, + {failed,{error,{test_case_failed,"this test should fail"}}}}}, + {?eh,test_stats,{2,2,{1,1}}}, + {?eh,tc_start,{surefire_SUITE,tc_skip}}, + {?eh,tc_done,{surefire_SUITE,tc_skip,{skipped,"this test is skipped"}}}, + {?eh,test_stats,{2,2,{2,1}}}, + {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, + {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, + {skipped,{require_failed,'_'}}}}, + {?eh,test_stats,{2,2,{2,2}}}, + {?eh,tc_start,{surefire_SUITE,{end_per_group,g,[]}}}, + {?eh,tc_done,{surefire_SUITE,{end_per_group,g,[]},ok}}], + [{?eh,tc_start,{surefire_SUITE,{init_per_group,g_fail,[]}}}, + {?eh,tc_done,{surefire_SUITE,{init_per_group,g_fail,[]}, + {failed,{error,all_cases_should_be_skipped}}}}, + {?eh,tc_auto_skip,{surefire_SUITE,tc_ok, + {failed, + {surefire_SUITE,init_per_group, + {'EXIT',all_cases_should_be_skipped}}}}}, + {?eh,test_stats,{2,2,{2,3}}}, + {?eh,tc_auto_skip,{surefire_SUITE,end_per_group, + {failed, + {surefire_SUITE,init_per_group, + {'EXIT',all_cases_should_be_skipped}}}}}], + {?eh,tc_start,{surefire_SUITE,end_per_suite}}, + {?eh,tc_done,{surefire_SUITE,end_per_suite,ok}}, + {?eh,stop_logging,[]}]. + + +%%%----------------------------------------------------------------- +%%% Check generated xml log files +check_xml(Case,XmlRe) -> + case filelib:wildcard(XmlRe) of + [] -> + ct:fail("No xml files found with regexp ~p~n", [XmlRe]); + [_] = Xmls when Case==absolute_path -> + do_check_xml(Case,Xmls); + [_,_] = Xmls -> + do_check_xml(Case,Xmls) + end. + +%% Allowed structure: +%% <testsuites> +%% <testsuite> +%% <properties> +%% <property/> +%% ... +%% </properties> +%% <testcase> +%% [<failure/> | <error/> | <skipped/> ] +%% </testcase> +%% ... +%% </testsuite> +%% ... +%% </testsuites> +do_check_xml(Case,[Xml|Xmls]) -> + ct:log("Checking <a href=~p>~s</a>~n",[Xml,Xml]), + {E,_} = xmerl_scan:file(Xml), + Expected = events_to_result(lists:flatten(test_events(Case))), + ParseResult = testsuites(Case,E), + ct:log("Expecting: ~p~n",[[Expected]]), + ct:log("Actual : ~p~n",[ParseResult]), + [Expected] = ParseResult, + do_check_xml(Case,Xmls); +do_check_xml(_,[]) -> + ok. + +%% Scanning the XML to get the same type of result as events_to_result/1 +testsuites(Case,#xmlElement{name=testsuites,content=TS}) -> + %% OTP-10589 - move properties element to <testsuite> + false = lists:keytake(properties,#xmlElement.name,TS), + testsuite(Case,TS). + +testsuite(Case,[#xmlElement{name=testsuite,content=TC,attributes=A}|TS]) -> + {ET,EF,ES} = events_to_numbers(lists:flatten(test_events(Case))), + {T,E,F,S} = get_numbers_from_attrs(A,false,false,false,false), + ct:log("Expecting total:~p, error:~p, failure:~p, skipped:~p~n",[ET,0,EF,ES]), + ct:log("Actual total:~p, error:~p, failure:~p, skipped:~p~n",[T,E,F,S]), + {ET,0,EF,ES} = {T,E,F,S}, + + %% properties should only be there if given a options to hook + false = lists:keytake(properties,#xmlElement.name,TC), + %% system-out and system-err is not used by common_test + false = lists:keytake('system-out',#xmlElement.name,TC), + false = lists:keytake('system-err',#xmlElement.name,TC), + R=testcase(Case,TC), + [R|testsuite(Case,TS)]; +testsuite(_Case,[]) -> + []. + +testcase(url=Case,[#xmlElement{name=testcase,attributes=A,content=C}|TC]) -> + R = failed_or_skipped(C), + case R of + [s] -> + case lists:keyfind(url,#xmlAttribute.name,A) of + false -> ok; + #xmlAttribute{value=UrlAttr} -> + lists:keyfind(url,#xmlAttribute.name,A), + true = lists:prefix(?url_base,UrlAttr) + end; + _ -> + #xmlAttribute{value=UrlAttr} = + lists:keyfind(url,#xmlAttribute.name,A), + true = lists:prefix(?url_base,UrlAttr) + end, + [R|testcase(Case,TC)]; +testcase(Case,[#xmlElement{name=testcase,attributes=A,content=C}|TC]) -> + false = lists:keyfind(url,#xmlAttribute.name,A), + R = failed_or_skipped(C), + [R|testcase(Case,TC)]; +testcase(_Case,[]) -> + []. + +failed_or_skipped([#xmlElement{name=failure}|E]) -> + [f|failed_or_skipped(E)]; +failed_or_skipped([#xmlElement{name=error}|E]) -> + [e|failed_or_skipped(E)]; +failed_or_skipped([#xmlElement{name=skipped}|E]) -> + [s|failed_or_skipped(E)]; +failed_or_skipped([]) -> + []. + +%% Using the expected events to produce the expected result of the XML scanning. +%% The result is a list of test suites: +%% Testsuites = [Testsuite] +%% Testsuite = [Testcase] +%% Testcase = [] | [f] | [s], indicating ok, failed and skipped respectively +events_to_result([{?eh,tc_done,{_Suite,_Case,R}}|E]) -> + [result(R)|events_to_result(E)]; +events_to_result([{?eh,tc_auto_skip,_}|E]) -> + [[s]|events_to_result(E)]; +events_to_result([_|E]) -> + events_to_result(E); +events_to_result([]) -> + []. + +result(ok) ->[]; +result({skipped,_}) -> [s]; +result({failed,_}) -> [f]. + +%% Using the expected events' last test_stats element to produce the +%% expected number of totla, errors, failed and skipped testcases. +events_to_numbers(E) -> + RevE = lists:reverse(E), + {?eh,test_stats,{Ok,F,{US,AS}}} = lists:keyfind(test_stats,2,RevE), + {Ok+F+US+AS,F,US+AS}. + +get_numbers_from_attrs([#xmlAttribute{name=tests,value=X}|A],false,E,F,S) -> + get_numbers_from_attrs(A,list_to_integer(X),E,F,S); +get_numbers_from_attrs([#xmlAttribute{name=errors,value=X}|A],T,false,F,S) -> + get_numbers_from_attrs(A,T,list_to_integer(X),F,S); +get_numbers_from_attrs([#xmlAttribute{name=failures,value=X}|A],T,E,false,S) -> + get_numbers_from_attrs(A,T,E,list_to_integer(X),S); +get_numbers_from_attrs([#xmlAttribute{name=skipped,value=X}|A],T,E,F,false) -> + get_numbers_from_attrs(A,T,E,F,list_to_integer(X)); +get_numbers_from_attrs([_|A],T,E,F,S) -> + get_numbers_from_attrs(A,T,E,F,S); +get_numbers_from_attrs([],T,E,F,S) -> + {T,E,F,S}. + +ensure_exists_empty(Dir) -> + case file:list_dir(Dir) of + {error,enoent} -> + file:make_dir(Dir); + {ok,Files} -> + del_files(Dir,Files) + end. + +del_files(Dir,[F0|Fs] ) -> + F = filename:join(Dir,F0), + case file:read_file_info(F) of + {ok,#file_info{type=directory}} -> + {ok,Files} = file:list_dir(F), + del_files(F,Files), + file:del_dir(F), + del_files(Dir,Fs); + _ -> + file:delete(F), + del_files(Dir,Fs) + end; +del_files(_,[]) -> + ok. diff --git a/lib/common_test/test/ct_surefire_SUITE_data/surefire_SUITE.erl b/lib/common_test/test/ct_surefire_SUITE_data/surefire_SUITE.erl new file mode 100644 index 0000000000..677aee46c5 --- /dev/null +++ b/lib/common_test/test/ct_surefire_SUITE_data/surefire_SUITE.erl @@ -0,0 +1,92 @@ +%%-------------------------------------------------------------------- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%%---------------------------------------------------------------------- +%% File: surefire_SUITE.erl +%% +%% Description: +%% This file contains the test cases for cth_surefire. +%% +%% @author Support +%% @doc Test of surefire support in common_test +%% @end +%%---------------------------------------------------------------------- +%%---------------------------------------------------------------------- +-module(surefire_SUITE). +-include_lib("common_test/include/ct.hrl"). + +-compile(export_all). + +%% Default timetrap timeout (set in init_per_testcase). +-define(default_timeout, ?t:minutes(1)). + +all() -> + testcases() ++ [{group,g},{group,g_fail}]. + +groups() -> + [{g,testcases()}, + {g_fail,[tc_ok]}]. + +testcases() -> + [tc_ok, + tc_fail, + tc_skip, + tc_autoskip_require]. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + Config. + +init_per_group(g_fail, _Config) -> + exit(all_cases_should_be_skipped); +init_per_group(_, Config) -> + Config. + +end_per_group(_Group, Config) -> + Config. + +init_per_testcase(_Case, Config) -> + Dog = test_server:timetrap(?default_timeout), + [{watchdog, Dog}|Config]. + +end_per_testcase(_Case, Config) -> + Dog=?config(watchdog, Config), + test_server:timetrap_cancel(Dog), + ok. + +%%%----------------------------------------------------------------- +%%% Test cases +break(_Config) -> + test_server:break(""), + ok. + +tc_ok(_Config) -> + ok. + +tc_fail(_Config) -> + ct:fail("this test should fail"). + +tc_skip(_Config) -> + {skip,"this test is skipped"}. + +tc_autoskip_require() -> + [{require,whatever}]. +tc_autoskip_require(Config) -> + ct:fail("this test should never be executed - it should be autoskipped"). diff --git a/lib/common_test/test/ct_system_error_SUITE.erl b/lib/common_test/test/ct_system_error_SUITE.erl new file mode 100644 index 0000000000..f2d6ef4b1b --- /dev/null +++ b/lib/common_test/test/ct_system_error_SUITE.erl @@ -0,0 +1,132 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_system_error_SUITE +%%% +%%% Description: +%%% +%%% Test that severe system errors (such as failure to write logs) are +%%% noticed and handled. +%%%------------------------------------------------------------------- +-module(ct_system_error_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config1 = ct_test_support:init_per_suite(Config), + Config1. + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [ + test_server_failing_logs + ]. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%%%----------------------------------------------------------------- +%%% +test_server_failing_logs(Config) -> + TC = test_server_failing_logs, + DataDir = ?config(data_dir, Config), + Suite = filename:join(DataDir, "a_SUITE"), + {Opts,ERPid} = setup([{suite,Suite},{label,TC}], Config), + crash_test_server(Config), + {error,{cannot_create_log_dir,__}} = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + ct_test_support:log_events(TC, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(TC), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + +crash_test_server(Config) -> + DataDir = ?config(data_dir, Config), + Root = proplists:get_value(logdir, ct_test_support:get_opts(Config)), + [$@|Host] = lists:dropwhile(fun(C) -> + C =/= $@ + end, atom_to_list(node())), + Format = filename:join(Root, + "ct_run.ct@" ++ Host ++ + ".~4..0w-~2..0w-~2..0w_" + "~2..0w.~2..0w.~2..0w"), + [C2,C1|_] = lists:reverse(filename:split(DataDir)), + LogDir = C1 ++ "." ++ C2 ++ ".a_SUITE.logs", + T = calendar:datetime_to_gregorian_seconds(calendar:local_time()), + [begin + {{Y,Mon,D},{H,Min,S}} = + calendar:gregorian_seconds_to_datetime(T+Offset), + Dir0 = io_lib:format(Format, [Y,Mon,D,H,Min,S]), + Dir = lists:flatten(Dir0), + file:make_dir(Dir), + File = filename:join(Dir, LogDir), + file:write_file(File, "anything goes\n") + end || Offset <- lists:seq(0, 20)], + ok. + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +%%%----------------------------------------------------------------- + +events_to_check(_Test) -> + [{?eh,severe_error,{cannot_create_log_dir,{'_','_'}}}]. diff --git a/lib/common_test/test/ct_system_error_SUITE_data/a_SUITE.erl b/lib/common_test/test/ct_system_error_SUITE_data/a_SUITE.erl new file mode 100644 index 0000000000..c6e3ddfd5d --- /dev/null +++ b/lib/common_test/test/ct_system_error_SUITE_data/a_SUITE.erl @@ -0,0 +1,122 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(a_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% @spec suite() -> Info +%% Info = [tuple()] +%% @end +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{seconds,10}}]. + +%%-------------------------------------------------------------------- +%% @spec init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%% @end +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% @end +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} | {fail,Reason} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +end_per_testcase(_TestCase, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%% @end +%%-------------------------------------------------------------------- +groups() -> + []. + +%%-------------------------------------------------------------------- +%% @spec all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +all() -> + [tc1]. + +tc1(_C) -> + ok. diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl index 80cca4a1cc..5e109e98e9 100644 --- a/lib/common_test/test/ct_test_support.erl +++ b/lib/common_test/test/ct_test_support.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2012. All Rights Reserved. +%% Copyright Ericsson AB 2008-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -32,7 +32,7 @@ run/2, run/3, run/4, get_opts/1, wait_for_ct_stop/1]). -export([handle_event/2, start_event_receiver/1, get_events/2, - verify_events/3, reformat/2, log_events/4, + verify_events/3, verify_events/4, reformat/2, log_events/4, join_abs_dirs/2]). -export([ct_test_halt/1]). @@ -117,8 +117,7 @@ end_per_suite(Config) -> CTNode = proplists:get_value(ct_node, Config), PrivDir = proplists:get_value(priv_dir, Config), true = rpc:call(CTNode, code, del_path, [filename:join(PrivDir,"")]), - cover:stop(CTNode), - slave:stop(CTNode), + slave_stop(CTNode), ok. %%%----------------------------------------------------------------- @@ -149,8 +148,7 @@ end_per_testcase(_TestCase, Config) -> case wait_for_ct_stop(CTNode) of %% Common test was not stopped to we restart node. false -> - cover:stop(CTNode), - slave:stop(CTNode), + slave_stop(CTNode), start_slave(Config,proplists:get_value(trace_level,Config)), {fail, "Could not stop common_test"}; true -> @@ -314,8 +312,10 @@ wait_for_ct_stop(Retries, CTNode) -> undefined -> true; Pid -> + Info = (catch process_info(Pid)), test_server:format(0, "Waiting for CT (~p) to finish (~p)...", [Pid,Retries]), + test_server:format(0, "Process info for ~p:~n~p", [Info]), timer:sleep(5000), wait_for_ct_stop(Retries-1, CTNode) end. @@ -330,12 +330,17 @@ handle_event(EH, Event) -> start_event_receiver(Config) -> CTNode = proplists:get_value(ct_node, Config), - spawn_link(CTNode, fun() -> er() end). + Level = proplists:get_value(trace_level, Config), + ER = spawn_link(CTNode, fun() -> er() end), + test_server:format(Level, "~nEvent receiver ~w started!~n", [ER]), + ER. get_events(_, Config) -> CTNode = proplists:get_value(ct_node, Config), + Level = proplists:get_value(trace_level, Config), {event_receiver,CTNode} ! {self(),get_events}, Events = receive {event_receiver,Evs} -> Evs end, + test_server:format(Level, "Stopping event receiver!~n", []), {event_receiver,CTNode} ! stop, Events. @@ -364,6 +369,14 @@ verify_events(TEvs, Evs, Config) -> ok end. +verify_events(TEvs, Evs, Node, Config) -> + case catch verify_events1(TEvs, Evs, Node, Config) of + {'EXIT',Reason} -> + Reason; + _ -> + ok + end. + verify_events1([TestEv|_], [{TEH,#event{name=stop_logging,node=Node,data=_}}|_], Node, _) when element(1,TestEv) == TEH, element(2,TestEv) =/= stop_logging -> test_server:format("Failed to find ~p in the list of events!~n", [TestEv]), @@ -612,8 +625,11 @@ locate({parallel,TEvs}, Node, Evs, Config) -> fun({EH,#event{name=tc_auto_skip, node=EvNode, data={Mod,end_per_group,Reason}}}) when - EH == TEH, EvNode == Node, Mod == M, Reason == R -> - false; + EH == TEH, EvNode == Node, Mod == M -> + case match_data(R, Reason) of + match -> false; + _ -> true + end; ({EH,#event{name=stop_logging, node=EvNode,data=_}}) when EH == TEH, EvNode == Node -> @@ -627,23 +643,12 @@ locate({parallel,TEvs}, Node, Evs, Config) -> [_AutoSkip | RemEvs2] -> {Done,RemEvs2,length(RemEvs2)} end; - %% match other event than test case - (TEv={TEH,N,D}, Acc) when D == '_' -> - case [E || E={EH,#event{name=Name, - node=EvNode, - data=_}} <- Evs1, - EH == TEH, EvNode == Node, Name == N] of - [] -> - exit({unmatched,TEv}); - _ -> - test_server:format("Found ~p!", [TEv]), - Acc - end; (TEv={TEH,N,D}, Acc) -> case [E || E={EH,#event{name=Name, node=EvNode, data=Data}} <- Evs1, - EH == TEH, EvNode == Node, Name == N, Data == D] of + EH == TEH, EvNode == Node, Name == N, + match == match_data(D,Data)] of [] -> exit({unmatched,TEv}); _ -> @@ -1002,33 +1007,39 @@ locate({TEH,Name,Data}, Node, [{TEH,#event{name=Name, data = EvData, node = Node}}|Evs], Config) -> - try match_data(Data, EvData) of + case match_data(Data, EvData) of match -> - {Config,Evs} - catch _:_ -> + {Config,Evs}; + _ -> nomatch end; locate({_TEH,_Name,_Data}, _Node, [_|_Evs], _Config) -> nomatch. -match_data(D,D) -> +match_data(Data, EvData) -> + try do_match_data(Data, EvData) + catch _:_ -> + nomatch + end. + +do_match_data(D,D) -> match; -match_data('_',_) -> +do_match_data('_',_) -> match; -match_data(Fun,Data) when is_function(Fun) -> +do_match_data(Fun,Data) when is_function(Fun) -> Fun(Data); -match_data('$proplist',Proplist) -> - match_data( +do_match_data('$proplist',Proplist) -> + do_match_data( fun(List) -> lists:foreach(fun({_,_}) -> ok end,List) end,Proplist); -match_data([H1|MatchT],[H2|ValT]) -> - match_data(H1,H2), - match_data(MatchT,ValT); -match_data(Tuple1,Tuple2) when is_tuple(Tuple1),is_tuple(Tuple2) -> - match_data(tuple_to_list(Tuple1),tuple_to_list(Tuple2)); -match_data([],[]) -> +do_match_data([H1|MatchT],[H2|ValT]) -> + do_match_data(H1,H2), + do_match_data(MatchT,ValT); +do_match_data(Tuple1,Tuple2) when is_tuple(Tuple1),is_tuple(Tuple2) -> + do_match_data(tuple_to_list(Tuple1),tuple_to_list(Tuple2)); +do_match_data([],[]) -> match. result_match({SkipOrFail,{ErrorInd,{Why,'_'}}}, @@ -1039,10 +1050,13 @@ result_match({SkipOrFail,{ErrorInd,{EMod,EFunc,{Why,'_'}}}}, true; result_match({failed,{timetrap_timeout,{'$approx',Num}}}, {failed,{timetrap_timeout,Value}}) -> - if Value >= trunc(Num-0.02*Num), - Value =< trunc(Num+0.02*Num) -> true; + if Value >= trunc(Num-0.05*Num), + Value =< trunc(Num+0.05*Num) -> true; true -> false end; +result_match({user_timetrap_error,{Why,'_'}}, + {user_timetrap_error,{Why,_Stack}}) -> + true; result_match(Result, Result) -> true; result_match(_, _) -> @@ -1125,6 +1139,8 @@ reformat([{_EH,#event{name=test_start,data=_}} | Events], EH) -> [{EH,test_start,{'DEF',{'START_TIME','LOGDIR'}}} | reformat(Events, EH)]; reformat([{_EH,#event{name=test_done,data=_}} | Events], EH) -> [{EH,test_done,{'DEF','STOP_TIME'}} | reformat(Events, EH)]; +reformat([{_EH,#event{name=tc_logfile,data=_}} | Events], EH) -> + reformat(Events, EH); reformat([{_EH,#event{name=test_stats,data=Data}} | Events], EH) -> [{EH,test_stats,Data} | reformat(Events, EH)]; %% use this to only print the last test_stats event: @@ -1259,3 +1275,22 @@ rm_files([F | Fs]) -> rm_files([]) -> ok. +%%%----------------------------------------------------------------- +%%% +slave_stop(Node) -> + Cover = test_server:is_cover(), + if Cover-> cover:flush(Node); + true -> ok + end, + erlang:monitor_node(Node, true), + slave:stop(Node), + receive + {nodedown, Node} -> + if Cover -> cover:stop(Node); + true -> ok + end + after 5000 -> + erlang:monitor_node(Node, false), + receive {nodedown, Node} -> ok after 0 -> ok end %flush + end, + ok. diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl index b7e19f25dd..6a4a4acd80 100644 --- a/lib/common_test/test/ct_testspec_1_SUITE.erl +++ b/lib/common_test/test/ct_testspec_1_SUITE.erl @@ -58,7 +58,7 @@ end_per_testcase(TestCase, Config) -> suite() -> [{ct_hooks,[ts_install_cth]}]. -all() -> +all() -> [all_suites, skip_all_suites, suite, skip_suite, all_testcases, skip_all_testcases, testcase, skip_testcase, all_groups, skip_all_groups, group, @@ -67,23 +67,23 @@ all() -> skip_group_testcase, topgroup, subgroup, skip_subgroup, subgroup_all_testcases, skip_subgroup_all_testcases, subgroup_testcase, skip_subgroup_testcase, - sub_skipped_by_top, testcase_in_multiple_groups, - order_of_tests_in_multiple_dirs_no_merge_tests, - order_of_tests_in_multiple_suites_no_merge_tests, - order_of_suites_in_multiple_dirs_no_merge_tests, - order_of_groups_in_multiple_dirs_no_merge_tests, - order_of_groups_in_multiple_suites_no_merge_tests, - order_of_tests_in_multiple_dirs, - order_of_tests_in_multiple_suites, - order_of_suites_in_multiple_dirs, - order_of_groups_in_multiple_dirs, - order_of_groups_in_multiple_suites, - order_of_tests_in_multiple_suites_with_skip_no_merge_tests, - order_of_tests_in_multiple_suites_with_skip, + sub_skipped_by_top, testcase_many_groups, + order_of_tests_many_dirs_no_merge_tests, + order_of_tests_many_suites_no_merge_tests, + order_of_suites_many_dirs_no_merge_tests, + order_of_groups_many_dirs_no_merge_tests, + order_of_groups_many_suites_no_merge_tests, + order_of_tests_many_dirs, + order_of_tests_many_suites, + order_of_suites_many_dirs, + order_of_groups_many_dirs, + order_of_groups_many_suites, + order_of_tests_many_suites_with_skip_no_merge_tests, + order_of_tests_many_suites_with_skip, all_plus_one_tc_no_merge_tests, all_plus_one_tc]. -groups() -> +groups() -> []. init_per_group(_GroupName, Config) -> @@ -373,19 +373,19 @@ sub_skipped_by_top(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% -testcase_in_multiple_groups(Config) when is_list(Config) -> +testcase_many_groups(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), TestDir = filename:join(DataDir, "groups_1"), TestSpec = [{cases,TestDir,groups_12_SUITE,[testcase_1a,testcase_1b]}, {skip_cases,TestDir,groups_12_SUITE,[testcase_1b],"SKIPPED!"}], - setup_and_execute(testcase_in_multiple_groups, TestSpec, Config). + setup_and_execute(testcase_many_groups, TestSpec, Config). %%%----------------------------------------------------------------- %%% -order_of_tests_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) -> +order_of_tests_many_dirs_no_merge_tests(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), TestDir1 = filename:join(DataDir, "groups_1"), @@ -395,13 +395,13 @@ order_of_tests_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) -> {cases,TestDir2,groups_22_SUITE,[testcase_1]}, {cases,TestDir1,groups_12_SUITE,[testcase_1b]}], - setup_and_execute(order_of_tests_in_multiple_dirs_no_merge_tests, + setup_and_execute(order_of_tests_many_dirs_no_merge_tests, TestSpec, Config). %%%----------------------------------------------------------------- %%% -order_of_tests_in_multiple_suites_no_merge_tests(Config) when is_list(Config) -> +order_of_tests_many_suites_no_merge_tests(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), TestDir1 = filename:join(DataDir, "groups_1"), @@ -410,13 +410,13 @@ order_of_tests_in_multiple_suites_no_merge_tests(Config) when is_list(Config) -> {cases,TestDir1,groups_11_SUITE,[testcase_1]}, {cases,TestDir1,groups_12_SUITE,[testcase_1b]}], - setup_and_execute(order_of_tests_in_multiple_suites_no_merge_tests, + setup_and_execute(order_of_tests_many_suites_no_merge_tests, TestSpec, Config). %%%----------------------------------------------------------------- %%% -order_of_suites_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) -> +order_of_suites_many_dirs_no_merge_tests(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), TestDir1 = filename:join(DataDir, "groups_1"), @@ -426,13 +426,13 @@ order_of_suites_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) -> {suites,TestDir2,groups_22_SUITE}, {suites,TestDir1,groups_11_SUITE}], - setup_and_execute(order_of_suites_in_multiple_dirs_no_merge_tests, + setup_and_execute(order_of_suites_many_dirs_no_merge_tests, TestSpec, Config). %%%----------------------------------------------------------------- %%% -order_of_groups_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) -> +order_of_groups_many_dirs_no_merge_tests(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), TestDir1 = filename:join(DataDir, "groups_1"), @@ -442,13 +442,13 @@ order_of_groups_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) -> {groups,TestDir2,groups_22_SUITE,test_group_1a}, {groups,TestDir1,groups_12_SUITE,test_group_1b}], - setup_and_execute(order_of_groups_in_multiple_dirs_no_merge_tests, + setup_and_execute(order_of_groups_many_dirs_no_merge_tests, TestSpec, Config). %%%----------------------------------------------------------------- %%% -order_of_groups_in_multiple_suites_no_merge_tests(Config) +order_of_groups_many_suites_no_merge_tests(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -458,13 +458,13 @@ order_of_groups_in_multiple_suites_no_merge_tests(Config) {groups,TestDir1,groups_11_SUITE,test_group_1a}, {groups,TestDir1,groups_12_SUITE,test_group_1b}], - setup_and_execute(order_of_groups_in_multiple_suites_no_merge_tests, + setup_and_execute(order_of_groups_many_suites_no_merge_tests, TestSpec, Config). %%%----------------------------------------------------------------- %%% -order_of_tests_in_multiple_suites_with_skip_no_merge_tests(Config) +order_of_tests_many_suites_with_skip_no_merge_tests(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -477,14 +477,14 @@ order_of_tests_in_multiple_suites_with_skip_no_merge_tests(Config) {skip_cases,TestDir1,groups_12_SUITE,[testcase_1b],"Skip it"}], setup_and_execute( - order_of_tests_in_multiple_suites_with_skip_no_merge_tests, + order_of_tests_many_suites_with_skip_no_merge_tests, TestSpec, Config). %%%----------------------------------------------------------------- %%% -order_of_tests_in_multiple_dirs(Config) when is_list(Config) -> +order_of_tests_many_dirs(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), TestDir1 = filename:join(DataDir, "groups_1"), @@ -493,13 +493,13 @@ order_of_tests_in_multiple_dirs(Config) when is_list(Config) -> {cases,TestDir2,groups_22_SUITE,[testcase_1]}, {cases,TestDir1,groups_12_SUITE,[testcase_1b]}], - setup_and_execute(order_of_tests_in_multiple_dirs, + setup_and_execute(order_of_tests_many_dirs, TestSpec, Config). %%%----------------------------------------------------------------- %%% -order_of_tests_in_multiple_suites(Config) when is_list(Config) -> +order_of_tests_many_suites(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), TestDir1 = filename:join(DataDir, "groups_1"), @@ -507,13 +507,13 @@ order_of_tests_in_multiple_suites(Config) when is_list(Config) -> {cases,TestDir1,groups_11_SUITE,[testcase_1]}, {cases,TestDir1,groups_12_SUITE,[testcase_1b]}], - setup_and_execute(order_of_tests_in_multiple_suites, + setup_and_execute(order_of_tests_many_suites, TestSpec, Config). %%%----------------------------------------------------------------- %%% -order_of_suites_in_multiple_dirs(Config) when is_list(Config) -> +order_of_suites_many_dirs(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), TestDir1 = filename:join(DataDir, "groups_1"), @@ -522,13 +522,13 @@ order_of_suites_in_multiple_dirs(Config) when is_list(Config) -> {suites,TestDir2,groups_22_SUITE}, {suites,TestDir1,groups_11_SUITE}], - setup_and_execute(order_of_suites_in_multiple_dirs, + setup_and_execute(order_of_suites_many_dirs, TestSpec, Config). %%%----------------------------------------------------------------- %%% -order_of_groups_in_multiple_dirs(Config) when is_list(Config) -> +order_of_groups_many_dirs(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), TestDir1 = filename:join(DataDir, "groups_1"), @@ -537,13 +537,13 @@ order_of_groups_in_multiple_dirs(Config) when is_list(Config) -> {groups,TestDir2,groups_22_SUITE,test_group_1a}, {groups,TestDir1,groups_12_SUITE,test_group_1b}], - setup_and_execute(order_of_groups_in_multiple_dirs, + setup_and_execute(order_of_groups_many_dirs, TestSpec, Config). %%%----------------------------------------------------------------- %%% -order_of_groups_in_multiple_suites(Config) when is_list(Config) -> +order_of_groups_many_suites(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), TestDir1 = filename:join(DataDir, "groups_1"), @@ -551,13 +551,13 @@ order_of_groups_in_multiple_suites(Config) when is_list(Config) -> {groups,TestDir1,groups_11_SUITE,test_group_1a}, {groups,TestDir1,groups_12_SUITE,test_group_1b}], - setup_and_execute(order_of_groups_in_multiple_suites, + setup_and_execute(order_of_groups_many_suites, TestSpec, Config). %%%----------------------------------------------------------------- %%% -order_of_tests_in_multiple_suites_with_skip(Config) when is_list(Config) -> +order_of_tests_many_suites_with_skip(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), TestDir1 = filename:join(DataDir, "groups_1"), @@ -567,7 +567,7 @@ order_of_tests_in_multiple_suites_with_skip(Config) when is_list(Config) -> {cases,TestDir1,groups_11_SUITE,[testcase_2]}, {skip_cases,TestDir1,groups_12_SUITE,[testcase_1b],"Skip it!"}], - setup_and_execute(order_of_tests_in_multiple_suites_with_skip, + setup_and_execute(order_of_tests_many_suites_with_skip, TestSpec, Config). %%%----------------------------------------------------------------- @@ -1204,10 +1204,10 @@ test_events(sub_skipped_by_top) -> {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} ]; -test_events(testcase_in_multiple_groups) -> +test_events(testcase_many_groups) -> []; -test_events(order_of_tests_in_multiple_dirs_no_merge_tests) -> +test_events(order_of_tests_many_dirs_no_merge_tests) -> [{?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, {?eh,tc_done, {groups_12_SUITE,testcase_1a, @@ -1219,7 +1219,7 @@ test_events(order_of_tests_in_multiple_dirs_no_merge_tests) -> {failed,{error,{test_case_failed,no_group_data}}}}}, {?eh,stop_logging,[]} ]; -test_events(order_of_tests_in_multiple_suites_no_merge_tests) -> +test_events(order_of_tests_many_suites_no_merge_tests) -> [{?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, {?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, @@ -1229,7 +1229,7 @@ test_events(order_of_tests_in_multiple_suites_no_merge_tests) -> {?eh,tc_done,{groups_12_SUITE,testcase_1b,'_'}}, {?eh,stop_logging,[]} ]; -test_events(order_of_suites_in_multiple_dirs_no_merge_tests) -> +test_events(order_of_suites_many_dirs_no_merge_tests) -> [{?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, {?eh,tc_done,{groups_12_SUITE,init_per_suite,'_'}}, @@ -1244,7 +1244,7 @@ test_events(order_of_suites_in_multiple_dirs_no_merge_tests) -> {?eh,tc_start,{groups_11_SUITE,end_per_suite}}, {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, {?eh,stop_logging,[]}]; -test_events(order_of_groups_in_multiple_dirs_no_merge_tests) -> +test_events(order_of_groups_many_dirs_no_merge_tests) -> [{?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, @@ -1257,7 +1257,7 @@ test_events(order_of_groups_in_multiple_dirs_no_merge_tests) -> {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, {?eh,stop_logging,[]}]; -test_events(order_of_groups_in_multiple_suites_no_merge_tests) -> +test_events(order_of_groups_many_suites_no_merge_tests) -> [{?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, @@ -1270,7 +1270,7 @@ test_events(order_of_groups_in_multiple_suites_no_merge_tests) -> {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, {?eh,stop_logging,[]}]; -test_events(order_of_tests_in_multiple_suites_with_skip_no_merge_tests) -> +test_events(order_of_tests_many_suites_with_skip_no_merge_tests) -> [{?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, {?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, @@ -1282,7 +1282,7 @@ test_events(order_of_tests_in_multiple_suites_with_skip_no_merge_tests) -> {?eh,stop_logging,[]} ]; -test_events(order_of_tests_in_multiple_dirs) -> +test_events(order_of_tests_many_dirs) -> [{?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, {?eh,tc_done, @@ -1296,7 +1296,7 @@ test_events(order_of_tests_in_multiple_dirs) -> {?eh,tc_done,{groups_22_SUITE,testcase_1,ok}}, {?eh,stop_logging,[]} ]; -test_events(order_of_tests_in_multiple_suites) -> +test_events(order_of_tests_many_suites) -> [{?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, {?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, @@ -1308,7 +1308,7 @@ test_events(order_of_tests_in_multiple_suites) -> {?eh,tc_done,{groups_11_SUITE,testcase_1,ok}}, {?eh,stop_logging,[]} ]; -test_events(order_of_suites_in_multiple_dirs) -> +test_events(order_of_suites_many_dirs) -> [{?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, {?eh,tc_done,{groups_12_SUITE,init_per_suite,'_'}}, @@ -1325,7 +1325,7 @@ test_events(order_of_suites_in_multiple_dirs) -> {?eh,tc_start,{groups_22_SUITE,end_per_suite}}, {?eh,tc_done,{groups_22_SUITE,end_per_suite,'_'}}, {?eh,stop_logging,[]}]; -test_events(order_of_groups_in_multiple_dirs) -> +test_events(order_of_groups_many_dirs) -> [{?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, @@ -1338,7 +1338,7 @@ test_events(order_of_groups_in_multiple_dirs) -> {?eh,tc_done, {groups_22_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, {?eh,stop_logging,[]}]; -test_events(order_of_groups_in_multiple_suites) -> +test_events(order_of_groups_many_suites) -> [{?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, @@ -1352,7 +1352,7 @@ test_events(order_of_groups_in_multiple_suites) -> {?eh,stop_logging,[]}]; -test_events(order_of_tests_in_multiple_suites_with_skip) -> +test_events(order_of_tests_many_suites_with_skip) -> [{?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, {?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, diff --git a/lib/common_test/test/ct_testspec_2_SUITE.erl b/lib/common_test/test/ct_testspec_2_SUITE.erl index 9d2dc84ad3..518352e87c 100644 --- a/lib/common_test/test/ct_testspec_2_SUITE.erl +++ b/lib/common_test/test/ct_testspec_2_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% Copyright Ericsson AB 2009-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -479,7 +479,7 @@ multiple_specs(_Config) -> "multiple_specs.1.spec"), SpecFile2 = ct_test_support:write_testspec(Spec2,SpecDir, "multiple_specs.2.spec"), - FileResult = ct_testspec:collect_tests_from_file([SpecFile1,SpecFile2], + FileResult = ct_testspec:collect_tests_from_file([[SpecFile1,SpecFile2]], false), ct:pal("TESTSPEC RECORD FROM FILE:~n~p~n", [rec2proplist(FileResult)]), @@ -490,7 +490,7 @@ multiple_specs(_Config) -> [{Node2,get_absdir(filename:join(SpecDir,CfgDir))} || CfgDir <- CfgDir2]], LogDirV = get_absdir(filename:join(SpecDir,"../logs")), - Verify = #testspec{merge_tests = false, + Verify = #testspec{merge_tests = true, spec_dir = SpecDir, nodes = [{undefined,Node},{n1,Node1},{n2,Node2}], alias = [{to1,TO1V},{to2,TO2V}], @@ -500,22 +500,13 @@ multiple_specs(_Config) -> logdir = [{Node,LogDirV},{Node1,LogDirV},{Node2,LogDirV},"."], config = CFGs, tests = [{{Node1,TO1V}, - [{x_SUITE,[all]}]}, - {{Node1,TO1V}, - [{y_SUITE,[{g1,all},{g2,all}]}]}, - {{Node1,TO1V}, - [{y_SUITE,[tc1,tc2]}]}, - {{Node1,TO1V}, - [{z_SUITE,[{all,{skip,"skipped"}}]}]}, - {{Node2,TO2V}, - [{x_SUITE,[all]}]}, - {{Node2,TO2V}, - [{y_SUITE,[all]}]}, - {{Node2,TO2V}, - [{x_SUITE,[{{g1,all},{skip,"skipped"}}, - {{g2,all},{skip,"skipped"}}]}]}, + [{x_SUITE,[all]}, + {y_SUITE,[{g1,all},{g2,all},tc1,tc2]}, + {z_SUITE,[{all,{skip,"skipped"}}]}]}, {{Node2,TO2V}, - [{y_SUITE,[{tc1,{skip,"skipped"}}, + [{x_SUITE,[all,{{g1,all},{skip,"skipped"}}, + {{g2,all},{skip,"skipped"}}]}, + {y_SUITE,[all,{tc1,{skip,"skipped"}}, {tc2,{skip,"skipped"}}]}]}]}, verify_result(Verify,FileResult,FileResult). @@ -524,7 +515,7 @@ multiple_specs(_Config) -> %%% misc_config_terms(_Config) -> CfgDir = "../cfgs/to1", - + TODir = "../tests/to1", Spec = [{node,x,n1@h1},{node,y,n2@h2}, @@ -554,7 +545,9 @@ misc_config_terms(_Config) -> {create_priv_dir,[auto_per_tc]}, {create_priv_dir,n1@h1,[manual_per_tc]}, - {create_priv_dir,n2@h2,[auto_per_run]} + {create_priv_dir,n2@h2,[auto_per_run]}, + + {suites,n1@h1,TODir,[x_SUITE]} ], {ok,SpecDir} = file:get_cwd(), @@ -599,7 +592,9 @@ misc_config_terms(_Config) -> {n2@h2,CSS2}], create_priv_dir = [{Node,[auto_per_tc]}, {n1@h1,[manual_per_tc]}, - {n2@h2,[auto_per_run]}] + {n2@h2,[auto_per_run]}], + tests = [{{n1@h1,get_absdir(filename:join(SpecDir,TODir))}, + [{x_SUITE,[all]}]}] }, verify_result(Verify,ListResult,FileResult). @@ -688,10 +683,10 @@ define_names_1(_Config) -> %%% HELP FUNCTIONS %%%----------------------------------------------------------------- -verify_result(Verify,ListResult,FileResult) -> +verify_result(VerificationRec,ListResult,FileResult) -> {_,TSLTuples} = rec2proplist(ListResult), {_,TSFTuples} = rec2proplist(FileResult), - {_,VTuples} = rec2proplist(Verify), + {_,VTuples} = rec2proplist(VerificationRec), VResult = (catch lists:foldl(fun({Tag,Val},{[{Tag,Val}|TSL],[{Tag,Val}|TSF]}) -> {TSL,TSF}; @@ -720,6 +715,8 @@ read_config(S) -> rec2proplist(E={error,_What}) -> exit({invalid_testspec_record,E}); +rec2proplist([{Specs,Rec}]) when is_list(Specs) -> + rec2proplist(Rec); rec2proplist(Rec) -> [RecName|RecList] = tuple_to_list(Rec), FieldNames = diff --git a/lib/common_test/test/ct_testspec_3_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE.erl new file mode 100644 index 0000000000..6b4b729552 --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE.erl @@ -0,0 +1,745 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_testspec_1_SUITE +%%% +%%% Description: +%%% Test test specifications +%%% +%%% The suites used for the test are located in the data directory. +%%%------------------------------------------------------------------- +-module(ct_testspec_3_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + DataDir = ?config(data_dir, Config), + Config1 = ct_test_support:init_per_suite(Config), + SpecsDir1 = filename:join(DataDir, "specs1"), + SpecsDir2 = filename:join(DataDir, "specs2"), + [{specs_dir1,SpecsDir1},{specs_dir2,SpecsDir2} | Config1]. + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [start_separate, + start_join, + incl_separate1, + incl_separate2, + incl_join1, + incl_join2, + incl_both1, + incl_both2, + incl_both_and_join1, + incl_both_and_join2, + rec_incl_separate1, + rec_incl_separate2, + rec_incl_join1, + rec_incl_join2, + rec_incl_separate_join1, + rec_incl_separate_join2, + rec_incl_join_separate1, + rec_incl_join_separate2 + ]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%%%----------------------------------------------------------------- +%%% + +start_separate(Config) -> + Specs = [fname(specs_dir1, "flat_spec1", Config), + fname(specs_dir2, "flat_spec2", Config)], + setup_and_execute(start_separate, Specs, [], Config). + +%%%----------------------------------------------------------------- +%%% + +start_join(Config) -> + Specs = [fname(specs_dir1, "flat_spec1", Config), + fname(specs_dir2, "flat_spec2", Config)], + setup_and_execute(start_join, Specs, [{join_specs,true}], Config). + +%%%----------------------------------------------------------------- +%%% + +incl_separate1(Config) -> + Specs = [fname(specs_dir1, "spec_sep1", Config), + fname(specs_dir2, "spec_sep2", Config)], + setup_and_execute(incl_separate1, Specs, [], Config). + +incl_separate2(Config) -> + Specs = [fname(specs_dir1, "spec_sep1", Config), + fname(specs_dir2, "spec_sep2", Config)], + setup_and_execute(incl_separate2, Specs, [{join_specs,true}], Config). + +%%%----------------------------------------------------------------- +%%% + +incl_join1(Config) -> + Specs = [fname(specs_dir1, "spec_join1", Config), + fname(specs_dir2, "spec_join2", Config)], + setup_and_execute(incl_join1, Specs, [], Config). + +incl_join2(Config) -> + Specs = [fname(specs_dir1, "spec_join1", Config), + fname(specs_dir2, "spec_join2", Config)], + setup_and_execute(incl_join2, Specs, [{join_specs,true}], Config). + +%%%----------------------------------------------------------------- +%%% + +incl_both1(Config) -> + Specs = [fname(specs_dir1, "spec_both1", Config), + fname(specs_dir2, "spec_both2", Config)], + setup_and_execute(incl_both1, Specs, [], Config). + +incl_both2(Config) -> + Specs = [fname(specs_dir1, "spec_both1", Config), + fname(specs_dir2, "spec_both2", Config)], + setup_and_execute(incl_both2, Specs, [{join_specs,true}], Config). + +%%%----------------------------------------------------------------- +%%% + +incl_both_and_join1(Config) -> + Specs = [fname(specs_dir1, "spec_both_join1", Config), + fname(specs_dir2, "spec_both_join2", Config)], + setup_and_execute(incl_both_and_join1, Specs, [], Config). + +incl_both_and_join2(Config) -> + Specs = [fname(specs_dir1, "spec_both_join1", Config), + fname(specs_dir2, "spec_both_join2", Config)], + setup_and_execute(incl_both_and_join2, Specs, [{join_specs,true}], Config). + +%%%----------------------------------------------------------------- +%%% + +rec_incl_separate1(Config) -> + Specs = [fname(specs_dir1, "rec_spec_sep1", Config), + fname(specs_dir2, "rec_spec_sep2", Config)], + setup_and_execute(rec_incl_separate1, Specs, [], Config). + +rec_incl_separate2(Config) -> + Specs = [fname(specs_dir1, "rec_spec_sep1", Config), + fname(specs_dir2, "rec_spec_sep2", Config)], + setup_and_execute(rec_incl_separate2, Specs, [{join_specs,true}], Config). + +%%%----------------------------------------------------------------- +%%% + +rec_incl_join1(Config) -> + Specs = [fname(specs_dir1, "rec_spec_join1", Config), + fname(specs_dir2, "rec_spec_join2", Config)], + setup_and_execute(rec_incl_join1, Specs, [], Config). + +rec_incl_join2(Config) -> + Specs = [fname(specs_dir1, "rec_spec_join1", Config), + fname(specs_dir2, "rec_spec_join2", Config)], + setup_and_execute(rec_incl_join2, Specs, [{join_specs,true}], Config). + + +%%%----------------------------------------------------------------- +%%% + +rec_incl_separate_join1(Config) -> + Specs = [fname(specs_dir1, "rec_spec_sep_join1", Config), + fname(specs_dir2, "rec_spec_sep_join2", Config)], + setup_and_execute(rec_incl_separate_join1, Specs, [], Config). + +rec_incl_separate_join2(Config) -> + Specs = [fname(specs_dir1, "rec_spec_sep_join1", Config), + fname(specs_dir2, "rec_spec_sep_join2", Config)], + setup_and_execute(rec_incl_separate_join2, Specs, + [{join_specs,true}], Config). + +%%%----------------------------------------------------------------- +%%% + +rec_incl_join_separate1(Config) -> + Specs = [fname(specs_dir1, "rec_spec_join_sep1", Config), + fname(specs_dir2, "rec_spec_join_sep2", Config)], + setup_and_execute(rec_incl_join_separate1, Specs, [], Config). + +rec_incl_join_separate2(Config) -> + Specs = [fname(specs_dir1, "rec_spec_join_sep1", Config), + fname(specs_dir2, "rec_spec_join_sep2", Config)], + setup_and_execute(rec_incl_join_separate2, Specs, + [{join_specs,true}], Config). + + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- + +fname(Tag, File, Config) -> + filename:join(?config(Tag, Config), File). + +check_parameter(TCID) -> + {ok,{config,TCID}}. + +read_config(TCID) -> + {ok,[{tcname,list_to_atom(TCID)}]}. + +setup_and_execute(TCName, Specs, TestOpts, Config) -> + + TestID = {userconfig,{?MODULE,atom_to_list(TCName)}}, + TestTerms = [TestID,{spec,Specs},{label,TCName}] ++ TestOpts, + + {Opts,ERPid} = setup(TestTerms, Config), + + case ct_test_support:run(Opts, Config) of + ok -> + ok; + Error -> + ct:pal("Error executing with opts: ~p", [Opts]), + exit(Error) + end, + + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(TCName, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(TCName), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + +setup(Test, Config) when is_tuple(Test) -> + setup([Test], Config); +setup(Tests, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ Tests ++ [{event_handler,{?eh,EvHArgs}}], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). +%reformat(Events, _EH) -> +% Events. + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +%%%----------------------------------------------------------------- +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). + +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + test_events(Test) ++ events_to_check(Test, N-1). + +test_events(start_separate) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{2,2,10}}, + {?eh,tc_start,{t11_SUITE,init_per_suite}}, + {?eh,tc_done,{t11_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t11_SUITE,end_per_suite}}, + {?eh,tc_done,{t11_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t21_SUITE,init_per_suite}}, + {?eh,tc_done,{t21_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{2,4,{2,2}}}, + {?eh,tc_start,{t21_SUITE,end_per_suite}}, + {?eh,tc_done,{t21_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}, + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{3,2,15}}, + {?eh,tc_start,{t12_SUITE,init_per_suite}}, + {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t12_SUITE,end_per_suite}}, + {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t12_SUITE,init_per_suite}}, + {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{2,4,{2,2}}}, + {?eh,tc_start,{t12_SUITE,end_per_suite}}, + {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t22_SUITE,init_per_suite}}, + {?eh,tc_done,{t22_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{3,6,{3,3}}}, + {?eh,tc_start,{t22_SUITE,end_per_suite}}, + {?eh,tc_done,{t22_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}]; + +test_events(start_join) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{4,4,20}}, + {?eh,tc_start,{t11_SUITE,init_per_suite}}, + {?eh,tc_done,{t11_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t11_SUITE,end_per_suite}}, + {?eh,tc_done,{t11_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t12_SUITE,init_per_suite}}, + {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{2,4,{2,2}}}, + {?eh,tc_start,{t12_SUITE,end_per_suite}}, + {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t21_SUITE,init_per_suite}}, + {?eh,tc_done,{t21_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{3,6,{3,3}}}, + {?eh,tc_start,{t21_SUITE,end_per_suite}}, + {?eh,tc_done,{t21_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t22_SUITE,init_per_suite}}, + {?eh,tc_done,{t22_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{4,8,{4,4}}}, + {?eh,tc_start,{t22_SUITE,end_per_suite}}, + {?eh,tc_done,{t22_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}]; + +test_events(incl_separate1) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{3,2,15}}, + {?eh,tc_start,{t12_SUITE,init_per_suite}}, + {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t12_SUITE,end_per_suite}}, + {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t12_SUITE,init_per_suite}}, + {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{2,4,{2,2}}}, + {?eh,tc_start,{t12_SUITE,end_per_suite}}, + {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t22_SUITE,init_per_suite}}, + {?eh,tc_done,{t22_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{3,6,{3,3}}}, + {?eh,tc_start,{t22_SUITE,end_per_suite}}, + {?eh,tc_done,{t22_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}, + + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{2,2,10}}, + {?eh,tc_start,{t11_SUITE,init_per_suite}}, + {?eh,tc_done,{t11_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t11_SUITE,end_per_suite}}, + {?eh,tc_done,{t11_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t21_SUITE,init_per_suite}}, + {?eh,tc_done,{t21_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{2,4,{2,2}}}, + {?eh,tc_start,{t21_SUITE,end_per_suite}}, + {?eh,tc_done,{t21_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}, + + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,5}}, + {?eh,tc_start,{t22_SUITE,init_per_suite}}, + {?eh,tc_done,{t22_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t22_SUITE,end_per_suite}}, + {?eh,tc_done,{t22_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}, + + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{2,2,10}}, + {?eh,tc_start,{t11_SUITE,init_per_suite}}, + {?eh,tc_done,{t11_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t11_SUITE,end_per_suite}}, + {?eh,tc_done,{t11_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t21_SUITE,init_per_suite}}, + {?eh,tc_done,{t21_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{2,4,{2,2}}}, + {?eh,tc_start,{t21_SUITE,end_per_suite}}, + {?eh,tc_done,{t21_SUITE,end_per_suite,ok}}, + + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}, + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{3,2,15}}, + {?eh,tc_start,{t12_SUITE,init_per_suite}}, + {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t12_SUITE,end_per_suite}}, + {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t12_SUITE,init_per_suite}}, + {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{2,4,{2,2}}}, + {?eh,tc_start,{t12_SUITE,end_per_suite}}, + {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t22_SUITE,init_per_suite}}, + {?eh,tc_done,{t22_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{3,6,{3,3}}}, + {?eh,tc_start,{t22_SUITE,end_per_suite}}, + {?eh,tc_done,{t22_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}]; + +test_events(incl_separate2) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,5}}, + {?eh,tc_start,{t22_SUITE,init_per_suite}}, + {?eh,tc_done,{t22_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t22_SUITE,end_per_suite}}, + {?eh,tc_done,{t22_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}, + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{3,2,15}}, + {?eh,tc_start,{t12_SUITE,init_per_suite}}, + {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t12_SUITE,end_per_suite}}, + {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t12_SUITE,init_per_suite}}, + {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{2,4,{2,2}}}, + {?eh,tc_start,{t12_SUITE,end_per_suite}}, + {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t22_SUITE,init_per_suite}}, + {?eh,tc_done,{t22_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{3,6,{3,3}}}, + {?eh,tc_start,{t22_SUITE,end_per_suite}}, + {?eh,tc_done,{t22_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}, + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{2,2,10}}, + {?eh,tc_start,{t11_SUITE,init_per_suite}}, + {?eh,tc_done,{t11_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t11_SUITE,end_per_suite}}, + {?eh,tc_done,{t11_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t21_SUITE,init_per_suite}}, + {?eh,tc_done,{t21_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{2,4,{2,2}}}, + {?eh,tc_start,{t21_SUITE,end_per_suite}}, + {?eh,tc_done,{t21_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}, + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{2,2,10}}, + {?eh,tc_start,{t11_SUITE,init_per_suite}}, + {?eh,tc_done,{t11_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t11_SUITE,end_per_suite}}, + {?eh,tc_done,{t11_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t21_SUITE,init_per_suite}}, + {?eh,tc_done,{t21_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{2,4,{2,2}}}, + {?eh,tc_start,{t21_SUITE,end_per_suite}}, + {?eh,tc_done,{t21_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}, + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{3,2,15}}, + {?eh,tc_start,{t12_SUITE,init_per_suite}}, + {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t12_SUITE,end_per_suite}}, + {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t12_SUITE,init_per_suite}}, + {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{2,4,{2,2}}}, + {?eh,tc_start,{t12_SUITE,end_per_suite}}, + {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t22_SUITE,init_per_suite}}, + {?eh,tc_done,{t22_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{3,6,{3,3}}}, + {?eh,tc_start,{t22_SUITE,end_per_suite}}, + {?eh,tc_done,{t22_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}]; + +test_events(incl_join1) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{4,4,20}}, + {?eh,tc_start,{t12_SUITE,init_per_suite}}, + {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t12_SUITE,end_per_suite}}, + {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t11_SUITE,init_per_suite}}, + {?eh,tc_done,{t11_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{2,4,{2,2}}}, + {?eh,tc_start,{t11_SUITE,end_per_suite}}, + {?eh,tc_done,{t11_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t22_SUITE,init_per_suite}}, + {?eh,tc_done,{t22_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{3,6,{3,3}}}, + {?eh,tc_start,{t22_SUITE,end_per_suite}}, + {?eh,tc_done,{t22_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t21_SUITE,init_per_suite}}, + {?eh,tc_done,{t21_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{4,8,{4,4}}}, + {?eh,tc_start,{t21_SUITE,end_per_suite}}, + {?eh,tc_done,{t21_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}, + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{4,4,20}}, + {?eh,tc_start,{t22_SUITE,init_per_suite}}, + {?eh,tc_done,{t22_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t22_SUITE,end_per_suite}}, + {?eh,tc_done,{t22_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t21_SUITE,init_per_suite}}, + {?eh,tc_done,{t21_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{2,4,{2,2}}}, + {?eh,tc_start,{t21_SUITE,end_per_suite}}, + {?eh,tc_done,{t21_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t11_SUITE,init_per_suite}}, + {?eh,tc_done,{t11_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{3,6,{3,3}}}, + {?eh,tc_start,{t11_SUITE,end_per_suite}}, + {?eh,tc_done,{t11_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t12_SUITE,init_per_suite}}, + {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{4,8,{4,4}}}, + {?eh,tc_start,{t12_SUITE,end_per_suite}}, + {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}]; + +test_events(incl_join2) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{4,4,20}}, + {?eh,tc_start,{t12_SUITE,init_per_suite}}, + {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t12_SUITE,end_per_suite}}, + {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t11_SUITE,init_per_suite}}, + {?eh,tc_done,{t11_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{2,4,{2,2}}}, + {?eh,tc_start,{t11_SUITE,end_per_suite}}, + {?eh,tc_done,{t11_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t22_SUITE,init_per_suite}}, + {?eh,tc_done,{t22_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{3,6,{3,3}}}, + {?eh,tc_start,{t22_SUITE,end_per_suite}}, + {?eh,tc_done,{t22_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t21_SUITE,init_per_suite}}, + {?eh,tc_done,{t21_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{4,8,{4,4}}}, + {?eh,tc_start,{t21_SUITE,end_per_suite}}, + {?eh,tc_done,{t21_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}]; + +test_events(incl_both1) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{2,2,10}}, + {?eh,tc_start,{t11_SUITE,init_per_suite}}, + {?eh,tc_done,{t11_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t11_SUITE,end_per_suite}}, + {?eh,tc_done,{t11_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t21_SUITE,init_per_suite}}, + {?eh,tc_done,{t21_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{2,4,{2,2}}}, + {?eh,tc_start,{t21_SUITE,end_per_suite}}, + {?eh,tc_done,{t21_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}, + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{3,2,15}}, + {?eh,tc_start,{t12_SUITE,init_per_suite}}, + {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t12_SUITE,end_per_suite}}, + {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t12_SUITE,init_per_suite}}, + {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{2,4,{2,2}}}, + {?eh,tc_start,{t12_SUITE,end_per_suite}}, + {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t22_SUITE,init_per_suite}}, + {?eh,tc_done,{t22_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{3,6,{3,3}}}, + {?eh,tc_start,{t22_SUITE,end_per_suite}}, + {?eh,tc_done,{t22_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}, + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{2,2,10}}, + {?eh,tc_start,{t12_SUITE,init_per_suite}}, + {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t12_SUITE,end_per_suite}}, + {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t22_SUITE,init_per_suite}}, + {?eh,tc_done,{t22_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{2,4,{2,2}}}, + {?eh,tc_start,{t22_SUITE,end_per_suite}}, + {?eh,tc_done,{t22_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}, + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{2,2,10}}, + {?eh,tc_start,{t11_SUITE,init_per_suite}}, + {?eh,tc_done,{t11_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t11_SUITE,end_per_suite}}, + {?eh,tc_done,{t11_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t21_SUITE,init_per_suite}}, + {?eh,tc_done,{t21_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{2,4,{2,2}}}, + {?eh,tc_start,{t21_SUITE,end_per_suite}}, + {?eh,tc_done,{t21_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}]; + +test_events(incl_both2) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{4,4,20}}, + {?eh,tc_start,{t11_SUITE,init_per_suite}}, + {?eh,tc_done,{t11_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t11_SUITE,end_per_suite}}, + {?eh,tc_done,{t11_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t12_SUITE,init_per_suite}}, + {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{2,4,{2,2}}}, + {?eh,tc_start,{t12_SUITE,end_per_suite}}, + {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t21_SUITE,init_per_suite}}, + {?eh,tc_done,{t21_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{t21_SUITE,ok_tc}}, + {?eh,test_stats,{3,6,{3,3}}}, + {?eh,tc_start,{t21_SUITE,end_per_suite}}, + {?eh,tc_done,{t21_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t22_SUITE,init_per_suite}}, + {?eh,tc_done,{t22_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{4,8,{4,4}}}, + {?eh,tc_start,{t22_SUITE,end_per_suite}}, + {?eh,tc_done,{t22_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}, + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{3,2,15}}, + {?eh,tc_start,{t12_SUITE,init_per_suite}}, + {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t12_SUITE,end_per_suite}}, + {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t12_SUITE,init_per_suite}}, + {?eh,tc_done,{t12_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{2,4,{2,2}}}, + {?eh,tc_start,{t12_SUITE,end_per_suite}}, + {?eh,tc_done,{t12_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t22_SUITE,init_per_suite}}, + {?eh,tc_done,{t22_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{3,6,{3,3}}}, + {?eh,tc_start,{t22_SUITE,end_per_suite}}, + {?eh,tc_done,{t22_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}, + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{2,2,10}}, + {?eh,tc_start,{t11_SUITE,init_per_suite}}, + {?eh,tc_done,{t11_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_start,{t11_SUITE,end_per_suite}}, + {?eh,tc_done,{t11_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{t21_SUITE,init_per_suite}}, + {?eh,tc_done,{t21_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{2,4,{2,2}}}, + {?eh,tc_start,{t21_SUITE,end_per_suite}}, + {?eh,tc_done,{t21_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}]; + +test_events(incl_both_and_join1) -> []; +test_events(incl_both_and_join2) -> []; +test_events(rec_incl_separate1) -> []; +test_events(rec_incl_separate2) -> []; +test_events(rec_incl_join1) -> []; +test_events(rec_incl_join2) -> []; +test_events(rec_incl_separate_join1) -> []; +test_events(rec_incl_separate_join2) -> []; +test_events(rec_incl_join_separate1) -> []; +test_events(rec_incl_join_separate2) -> []; + +test_events(_) -> + []. + + diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg11 b/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg11 new file mode 100644 index 0000000000..bc672568da --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg11 @@ -0,0 +1 @@ +{file, cfg11}.
\ No newline at end of file diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg12 b/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg12 new file mode 100644 index 0000000000..30f2cf6857 --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg12 @@ -0,0 +1 @@ +{file, cfg12}.
\ No newline at end of file diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg13 b/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg13 new file mode 100644 index 0000000000..1860ec78e5 --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg13 @@ -0,0 +1 @@ +{file, cfg13}.
\ No newline at end of file diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/config2/cfg21 b/lib/common_test/test/ct_testspec_3_SUITE_data/config2/cfg21 new file mode 100644 index 0000000000..b18d35443e --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/config2/cfg21 @@ -0,0 +1 @@ +{file, cfg21}.
\ No newline at end of file diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/flat_spec1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/flat_spec1 new file mode 100644 index 0000000000..eff87222ea --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/flat_spec1 @@ -0,0 +1,4 @@ +{config, "../config1/cfg11"}. +{suites, "../tests1", t11_SUITE}. +{suites, "../tests1", t11_SUITE}. +{suites, "../tests2", t21_SUITE}. diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_join1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_join1 new file mode 100644 index 0000000000..a3387f48a3 --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_join1 @@ -0,0 +1,2 @@ +{specs,join,"spec_join1"}. +{specs,join,"../specs2/spec_join2"}. diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_join_sep1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_join_sep1 new file mode 100644 index 0000000000..fe127eb4b9 --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_join_sep1 @@ -0,0 +1,5 @@ +{specs,join,"spec_sep1"}. +{specs,join,"../specs2/spec_sep2"}. + +{config, "../config1/cfg13"}. +{suites, "../tests2", t23_SUITE}. diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_sep1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_sep1 new file mode 100644 index 0000000000..c778aa68a6 --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_sep1 @@ -0,0 +1,2 @@ +{specs,separate,"spec_sep1"}. +{specs,separate,"../specs2/spec_sep2"}. diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_sep_join1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_sep_join1 new file mode 100644 index 0000000000..7cb5a05fff --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_sep_join1 @@ -0,0 +1,2 @@ +{specs,separate,"spec_join1"}. +{specs,separate,"../specs2/spec_join2"}. diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_both1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_both1 new file mode 100644 index 0000000000..46111614dc --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_both1 @@ -0,0 +1,2 @@ +{specs, join, "../specs1/flat_spec1"}. +{specs, separate, "../specs2/flat_spec2"}. diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_both_join1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_both_join1 new file mode 100644 index 0000000000..f52b3ed030 --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_both_join1 @@ -0,0 +1,6 @@ +{specs, join, "../specs1/flat_spec1"}. +{specs, separate, "../specs2/flat_spec2"}. +{merge_tests,false}. +{config, "../config1/cfg12"}. +{suites, "../tests1", t11_SUITE}. +{suites, "../tests2", t21_SUITE}. diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_join1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_join1 new file mode 100644 index 0000000000..baaaf35be4 --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_join1 @@ -0,0 +1 @@ +{specs, join, ["../specs2/flat_spec2", "flat_spec1"]}.
\ No newline at end of file diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_sep1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_sep1 new file mode 100644 index 0000000000..89456c35e0 --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_sep1 @@ -0,0 +1,3 @@ +{specs, separate, "../specs2/flat_spec2"}. +{specs, separate, "flat_spec1"}. + diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/flat_spec2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/flat_spec2 new file mode 100644 index 0000000000..758d1e2514 --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/flat_spec2 @@ -0,0 +1,5 @@ +{merge_tests, false}. +{config, "../config2/cfg21"}. +{suites, "../tests1", t12_SUITE}. +{suites, "../tests1", t12_SUITE}. +{suites, "../tests2", t22_SUITE}. diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_join2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_join2 new file mode 100644 index 0000000000..19d3a3d8e2 --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_join2 @@ -0,0 +1,5 @@ +{specs,join,"spec_join2"}. +{specs,join,"../specs1/spec_join1"}. + +{config, "../config1/cfg13"}. +{suites, "../tests2", t23_SUITE}. diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_join_sep2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_join_sep2 new file mode 100644 index 0000000000..930e68c847 --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_join_sep2 @@ -0,0 +1,5 @@ +{specs,join,"spec_sep2"}. +{specs,join,"../specs1/spec_sep1"}. + +{config, "../config1/cfg13"}. +{suites, "../tests2", t23_SUITE}. diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_sep2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_sep2 new file mode 100644 index 0000000000..5026f329a7 --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_sep2 @@ -0,0 +1,5 @@ +{specs,separate,"spec_sep2"}. +{specs,separate,"../specs1/spec_sep1"}. + +{config, "../config1/cfg13"}. +{suites, "../tests2", t23_SUITE}. diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_sep_join2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_sep_join2 new file mode 100644 index 0000000000..17057088b4 --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_sep_join2 @@ -0,0 +1,5 @@ +{specs,separate,"spec_join2"}. +{specs,separate,"../specs1/spec_join1"}. + +{config, "../config1/cfg13"}. +{suites, "../tests2", t23_SUITE}. diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_both2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_both2 new file mode 100644 index 0000000000..4c83115d23 --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_both2 @@ -0,0 +1,4 @@ +{specs, separate, "../specs1/flat_spec1"}. +{specs, join, "../specs2/flat_spec2"}. +{config, "../config1/cfg12"}. + diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_both_join2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_both_join2 new file mode 100644 index 0000000000..ad81bfb4cc --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_both_join2 @@ -0,0 +1,9 @@ +{merge_tests,true}. +{config, "../config1/cfg12"}. +{suites, "../tests1", t11_SUITE}. +{suites, "../tests2", t21_SUITE}. +{suites, "../tests1", t12_SUITE}. +{suites, "../tests2", t22_SUITE}. + +{specs, separate, "../specs1/flat_spec1"}. +{specs, join, "../specs2/flat_spec2"}. diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_join2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_join2 new file mode 100644 index 0000000000..d652dbd78f --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_join2 @@ -0,0 +1,5 @@ +{specs, join, ["../specs1/flat_spec1"]}. +{specs, join, ["flat_spec2"]}. +{config, "../config1/cfg12"}. +{suites, "../tests2", t22_SUITE}. + diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_sep2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_sep2 new file mode 100644 index 0000000000..8d37f508b8 --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_sep2 @@ -0,0 +1,5 @@ +{specs, separate, "../specs1/flat_spec1"}. +{specs, separate, "flat_spec2"}. +{config, "../config1/cfg12"}. +{suites, "../tests2", t22_SUITE}. + diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t11_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t11_SUITE.erl new file mode 100644 index 0000000000..b8216c3596 --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t11_SUITE.erl @@ -0,0 +1,175 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2008-2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(t11_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% @spec suite() -> Info +%% Info = [tuple()] +%% @end +%%-------------------------------------------------------------------- +suite() -> + [{require,file}, + {require,tcname}, + {timetrap,{seconds,1}}]. + +%%-------------------------------------------------------------------- +%% @spec init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + %% verify that expected config file can be read + case {ct:get_config(tcname),ct:get_config(file,undefined,[all])} of + {start_separate,[cfg11]} -> ok; + {start_join,[cfg11,cfg21]} -> ok; + {incl_separate1,[cfg11]} -> ok; + {incl_separate2,[cfg11]} -> ok; + {incl_join1,[cfg21,cfg11]} -> ok; + {incl_join1,[cfg12,cfg11,cfg21]} -> ok; + {incl_join2,[cfg21,cfg11,cfg12]} -> ok; + {incl_both1,[cfg11]} -> ok; + {incl_both2,[cfg11,cfg12,cfg21]} -> ok; + {incl_both2,[cfg11]} -> ok; + _ -> ok + + end, + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%% @end +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% @end +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_testcase(autoskip_tc, Config) -> + exit(kaboom), + Config; + +init_per_testcase(userskip_tc, Config) -> + {skip,"user skipped"}; + +init_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} | {fail,Reason} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +end_per_testcase(_TestCase, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%% @end +%%-------------------------------------------------------------------- +groups() -> + []. + +%%-------------------------------------------------------------------- +%% @spec all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +all() -> + [ok_tc, exit_tc, to_tc, autoskip_tc, userskip_tc]. + +%%-------------------------------------------------------------------- +%% @spec TestCase(Config0) -> +%% ok | exit() | {skip,Reason} | {comment,Comment} | +%% {save_config,Config1} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% Comment = term() +%% @end +%%-------------------------------------------------------------------- +ok_tc(_) -> + ok. + +exit_tc(_) -> + exit(kaboom), + ok. + +to_tc(_) -> + ct:timetrap(1), + ct:sleep(100), + ok. + +autoskip_tc(_) -> + ok. + +userskip_tc(_) -> + ok. + + + + diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t12_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t12_SUITE.erl new file mode 100644 index 0000000000..7c51aca246 --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t12_SUITE.erl @@ -0,0 +1,175 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2008-2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(t12_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% @spec suite() -> Info +%% Info = [tuple()] +%% @end +%%-------------------------------------------------------------------- +suite() -> + [{require,file}, + {require,tcname}, + {timetrap,{seconds,30}}]. + +%%-------------------------------------------------------------------- +%% @spec init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + %% verify that expected config file can be read + case {ct:get_config(tcname),ct:get_config(file,undefined,[all])} of + {start_separate,[cfg21]} -> ok; + {start_join,[cfg11,cfg21]} -> ok; + {incl_separate1,[cfg21]} -> ok; + {incl_separate2,[cfg21]} -> ok; + {incl_join1,[cfg21,cfg11]} -> ok; + {incl_join1,[cfg12,cfg11,cfg21]} -> ok; + {incl_join2,[cfg21,cfg11,cfg12]} -> ok; + {incl_both1,[cfg21]} -> ok; + {incl_both1,[cfg12,cfg21]} -> ok; + {incl_both2,[cfg11,cfg12,cfg21]} -> ok; + {incl_both2,[cfg21]} -> ok; + _ -> ok + end, + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%% @end +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% @end +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_testcase(autoskip_tc, Config) -> + exit(kaboom), + Config; + +init_per_testcase(userskip_tc, Config) -> + {skip,"user skipped"}; + +init_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} | {fail,Reason} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +end_per_testcase(_TestCase, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%% @end +%%-------------------------------------------------------------------- +groups() -> + []. + +%%-------------------------------------------------------------------- +%% @spec all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +all() -> + [ok_tc, exit_tc, to_tc, autoskip_tc, userskip_tc]. + +%%-------------------------------------------------------------------- +%% @spec TestCase(Config0) -> +%% ok | exit() | {skip,Reason} | {comment,Comment} | +%% {save_config,Config1} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% Comment = term() +%% @end +%%-------------------------------------------------------------------- +ok_tc(_) -> + ok. + +exit_tc(_) -> + exit(kaboom), + ok. + +to_tc(_) -> + ct:timetrap(1), + ct:sleep(100), + ok. + +autoskip_tc(_) -> + ok. + +userskip_tc(_) -> + ok. + + + + diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t21_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t21_SUITE.erl new file mode 100644 index 0000000000..36c1b4279b --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t21_SUITE.erl @@ -0,0 +1,174 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2008-2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(t21_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% @spec suite() -> Info +%% Info = [tuple()] +%% @end +%%-------------------------------------------------------------------- +suite() -> + [{require,file}, + {require,tcname}, + {timetrap,{seconds,1}}]. + +%%-------------------------------------------------------------------- +%% @spec init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + %% verify that expected config file can be read + case {ct:get_config(tcname),ct:get_config(file,undefined,[all])} of + {start_separate,[cfg11]} -> ok; + {start_join,[cfg11,cfg21]} -> ok; + {incl_separate1,[cfg11]} -> ok; + {incl_separate2,[cfg11]} -> ok; + {incl_join1,[cfg21,cfg11]} -> ok; + {incl_join1,[cfg12,cfg11,cfg21]} -> ok; + {incl_join2,[cfg21,cfg11,cfg12]} -> ok; + {incl_both1,[cfg11]} -> ok; + {incl_both2,[cfg11,cfg12,cfg21]} -> ok; + {incl_both2,[cfg11]} -> ok; + _ -> ok + end, + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%% @end +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% @end +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_testcase(autoskip_tc, Config) -> + exit(kaboom), + Config; + +init_per_testcase(userskip_tc, Config) -> + {skip,"user skipped"}; + +init_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} | {fail,Reason} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +end_per_testcase(_TestCase, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%% @end +%%-------------------------------------------------------------------- +groups() -> + []. + +%%-------------------------------------------------------------------- +%% @spec all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +all() -> + [ok_tc, exit_tc, to_tc, autoskip_tc, userskip_tc]. + +%%-------------------------------------------------------------------- +%% @spec TestCase(Config0) -> +%% ok | exit() | {skip,Reason} | {comment,Comment} | +%% {save_config,Config1} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% Comment = term() +%% @end +%%-------------------------------------------------------------------- +ok_tc(_) -> + ok. + +exit_tc(_) -> + exit(kaboom), + ok. + +to_tc(_) -> + ct:timetrap(1), + ct:sleep(100), + ok. + +autoskip_tc(_) -> + ok. + +userskip_tc(_) -> + ok. + + + + diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t22_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t22_SUITE.erl new file mode 100644 index 0000000000..3f6336c7e2 --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t22_SUITE.erl @@ -0,0 +1,177 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2008-2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(t22_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% @spec suite() -> Info +%% Info = [tuple()] +%% @end +%%-------------------------------------------------------------------- +suite() -> + [{require,file}, + {require,tcname}, + {timetrap,{seconds,30}}]. + +%%-------------------------------------------------------------------- +%% @spec init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + %% verify that expected config file can be read + case {ct:get_config(tcname),ct:get_config(file,undefined,[all])} of + {start_separate,[cfg21]} -> ok; + {start_join,[cfg11,cfg21]} -> ok; + {incl_separate1,[cfg12]} -> ok; + {incl_separate1,[cfg21]} -> ok; + {incl_separate2,[cfg12]} -> ok; + {incl_separate2,[cfg21]} -> ok; + {incl_join1,[cfg21,cfg11]} -> ok; + {incl_join1,[cfg12,cfg11,cfg21]} -> ok; + {incl_join2,[cfg21,cfg11,cfg12]} -> ok; + {incl_both1,[cfg21]} -> ok; + {incl_both1,[cfg12,cfg21]} -> ok; + {incl_both2,[cfg11,cfg12,cfg21]} -> ok; + {incl_both2,[cfg21]} -> ok; + _ -> ok + end, + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%% @end +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% @end +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_testcase(autoskip_tc, Config) -> + exit(kaboom), + Config; + +init_per_testcase(userskip_tc, Config) -> + {skip,"user skipped"}; + +init_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} | {fail,Reason} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +end_per_testcase(_TestCase, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%% @end +%%-------------------------------------------------------------------- +groups() -> + []. + +%%-------------------------------------------------------------------- +%% @spec all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +all() -> + [ok_tc, exit_tc, to_tc, autoskip_tc, userskip_tc]. + +%%-------------------------------------------------------------------- +%% @spec TestCase(Config0) -> +%% ok | exit() | {skip,Reason} | {comment,Comment} | +%% {save_config,Config1} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% Comment = term() +%% @end +%%-------------------------------------------------------------------- +ok_tc(_) -> + ok. + +exit_tc(_) -> + exit(kaboom), + ok. + +to_tc(_) -> + ct:timetrap(1), + ct:sleep(100), + ok. + +autoskip_tc(_) -> + ok. + +userskip_tc(_) -> + ok. + + + + diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t23_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t23_SUITE.erl new file mode 100644 index 0000000000..d836ab57c1 --- /dev/null +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t23_SUITE.erl @@ -0,0 +1,158 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2008-2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(t23_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% @spec suite() -> Info +%% Info = [tuple()] +%% @end +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{seconds,30}}]. + +%%-------------------------------------------------------------------- +%% @spec init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%% @end +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% @end +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_testcase(autoskip_tc, Config) -> + exit(kaboom), + Config; + +init_per_testcase(userskip_tc, Config) -> + {skip,"user skipped"}; + +init_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} | {fail,Reason} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +end_per_testcase(_TestCase, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%% @end +%%-------------------------------------------------------------------- +groups() -> + []. + +%%-------------------------------------------------------------------- +%% @spec all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +all() -> + [ok_tc, exit_tc, to_tc, autoskip_tc, userskip_tc]. + +%%-------------------------------------------------------------------- +%% @spec TestCase(Config0) -> +%% ok | exit() | {skip,Reason} | {comment,Comment} | +%% {save_config,Config1} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% Comment = term() +%% @end +%%-------------------------------------------------------------------- +ok_tc(_) -> + ok. + +exit_tc(_) -> + exit(kaboom), + ok. + +to_tc(_) -> + ct:timetrap(1), + ct:sleep(100), + ok. + +autoskip_tc(_) -> + ok. + +userskip_tc(_) -> + ok. + + + + diff --git a/lib/common_test/test/ct_verbosity_SUITE.erl b/lib/common_test/test/ct_verbosity_SUITE.erl index 349319de94..32488b1db9 100644 --- a/lib/common_test/test/ct_verbosity_SUITE.erl +++ b/lib/common_test/test/ct_verbosity_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% Copyright Ericsson AB 2009-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -44,8 +44,11 @@ %% there will be clashes with logging processes etc). %%-------------------------------------------------------------------- init_per_suite(Config) -> - Config1 = ct_test_support:init_per_suite(Config), - Config1. + DataDir = ?config(data_dir, Config), + EvH = filename:join(DataDir,"simple_evh.erl"), + ct:pal("Compiling ~s: ~p", [EvH,compile:file(EvH,[{outdir,DataDir}, + debug_info])]), + ct_test_support:init_per_suite([{path_dirs,[DataDir]} | Config]). end_per_suite(Config) -> ct_test_support:end_per_suite(Config). @@ -56,7 +59,8 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -suite() -> [{ct_hooks,[ts_install_cth]}]. +suite() -> [{timetrap,{minutes,2}}, + {ct_hooks,[ts_install_cth]}]. all() -> [ @@ -67,7 +71,8 @@ all() -> change_default, combine_categories, testspec_only, - merge_with_testspec + merge_with_testspec, + possible_deadlock ]. %%-------------------------------------------------------------------- @@ -173,6 +178,17 @@ merge_with_testspec(Config) -> ok = execute(TC, Opts, ERPid, Config). %%%----------------------------------------------------------------- +%%% +possible_deadlock(Config) -> + TC = possible_deadlock, + DataDir = ?config(data_dir, Config), + Suite = filename:join(DataDir, "io_test_SUITE"), + {Opts,ERPid} = setup([{suite,Suite},{label,TC}, + {event_handler,[simple_evh]}], Config), + ok = execute(TC, Opts, ERPid, Config). + + +%%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- @@ -180,7 +196,14 @@ setup(Test, Config) -> Opts0 = ct_test_support:get_opts(Config), Level = ?config(trace_level, Config), EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], - Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], + Opts = + case proplists:get_value(event_handler, Test) of + undefined -> + Opts0 ++ [{event_handler,{?eh,EvHArgs}} | Test]; + EvHs -> + Opts0 ++ [{event_handler,{[?eh|EvHs],EvHArgs}} | + proplists:delete(event_handler, Test)] + end, ERPid = ct_test_support:start_event_receiver(Config), {Opts,ERPid}. diff --git a/lib/common_test/test/ct_verbosity_SUITE_data/simple_evh.erl b/lib/common_test/test/ct_verbosity_SUITE_data/simple_evh.erl new file mode 100644 index 0000000000..3e744f2596 --- /dev/null +++ b/lib/common_test/test/ct_verbosity_SUITE_data/simple_evh.erl @@ -0,0 +1,171 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2006-2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%% @doc Common Test Framework Event Handler +%%% +%%% <p>This module implements an event handler that CT uses to +%%% handle status and progress notifications during test runs. +%%% The notifications are handled locally (per node) and passed +%%% on to ct_master when CT runs in distributed mode. This +%%% module may be used as a template for other event handlers +%%% that can be plugged in to handle local logging and reporting.</p> +-module(simple_evh). + +-behaviour(gen_event). + +%% gen_event callbacks +-export([init/1, handle_event/2, handle_call/2, + handle_info/2, terminate/2, code_change/3]). + +-include_lib("common_test/include/ct_event.hrl"). +-include_lib("common_test/src/ct_util.hrl"). + +%%==================================================================== +%% gen_event callbacks +%%==================================================================== +%%-------------------------------------------------------------------- +%% Function: init(Args) -> {ok, State} +%% Description: Whenever a new event handler is added to an event manager, +%% this function is called to initialize the event handler. +%%-------------------------------------------------------------------- +init(_) -> + io:format("Event handler ~w started!~n", [?MODULE]), + {ok,[]}. + +%%-------------------------------------------------------------------- +%% Function: +%% handle_event(Event, State) -> {ok, State} | +%% {swap_handler, Args1, State1, Mod2, Args2} | +%% remove_handler +%% Description:Whenever an event manager receives an event sent using +%% gen_event:notify/2 or gen_event:sync_notify/2, this function is called for +%% each installed event handler to handle the event. +%%-------------------------------------------------------------------- +handle_event(Event = #event{name = test_stats},State) -> + %% this could cause a deadlock + ct:pal("~p: ~p~n", [Event#event.name,Event#event.data]), + {ok,State}; +handle_event(_Event,State) -> + {ok,State}. + +%%============================== EVENTS ============================== +%% +%% Name = test_start +%% Data = {StartTime,LogDir} +%% +%% Name = start_info +%% Data = {Tests,Suites,Cases} +%% Tests = Suites = Cases = integer() +%% +%% Name = test_done +%% Data = EndTime +%% +%% Name = start_make +%% Data = Dir +%% +%% Name = finished_make +%% Data = Dir +%% +%% Name = tc_start +%% Data = {Suite,CaseOrGroup} +%% CaseOrGroup = atom() | {Conf,GroupName,GroupProperties} +%% Conf = init_per_group | end_per_group +%% GroupName = atom() +%% GroupProperties = list() +%% +%% Name = tc_done +%% Data = {Suite,CaseOrGroup,Result} +%% CaseOrGroup = atom() | {Conf,GroupName,GroupProperties} +%% Conf = init_per_group | end_per_group +%% GroupName = atom() +%% GroupProperties = list() +%% Result = ok | {skipped,Reason} | {failed,Reason} +%% +%% Name = tc_user_skip +%% Data = {Suite,Case,Comment} +%% Comment = string() +%% +%% Name = tc_auto_skip +%% Data = {Suite,Case,Comment} +%% Comment = string() +%% +%% Name = test_stats +%% Data = {Ok,Failed,Skipped} +%% Ok = Failed = integer() +%% Skipped = {UserSkipped,AutoSkipped} +%% UserSkipped = AutoSkipped = integer() +%% +%% Name = start_logging +%% Data = CtRunDir +%% +%% Name = stop_logging +%% Data = [] +%% +%% Name = start_write_file +%% Data = FullNameFile +%% +%% Name = finished_write_file +%% Data = FullNameFile +%% +%% Name = +%% Data = +%% + +%%-------------------------------------------------------------------- +%% Function: +%% handle_call(Request, State) -> {ok, Reply, State} | +%% {swap_handler, Reply, Args1, State1, +%% Mod2, Args2} | +%% {remove_handler, Reply} +%% Description: Whenever an event manager receives a request sent using +%% gen_event:call/3,4, this function is called for the specified event +%% handler to handle the request. +%%-------------------------------------------------------------------- +handle_call(_Req, State) -> + Reply = ok, + {ok, Reply, State}. + +%%-------------------------------------------------------------------- +%% Function: +%% handle_info(Info, State) -> {ok, State} | +%% {swap_handler, Args1, State1, Mod2, Args2} | +%% remove_handler +%% Description: This function is called for each installed event handler when +%% an event manager receives any other message than an event or a synchronous +%% request (or a system message). +%%-------------------------------------------------------------------- +handle_info(_Info, State) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Function: terminate(Reason, State) -> void() +%% Description:Whenever an event handler is deleted from an event manager, +%% this function is called. It should be the opposite of Module:init/1 and +%% do any necessary cleaning up. +%%-------------------------------------------------------------------- +terminate(_Reason, _State) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: code_change(OldVsn, State, Extra) -> {ok, NewState} +%% Description: Convert process state when code is changed +%%-------------------------------------------------------------------- +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + |