aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPeter Andersson <[email protected]>2010-06-22 17:15:00 +0200
committerPeter Andersson <[email protected]>2010-07-06 02:44:26 +0200
commit4b399cdf4472975b0f3d4d21054f30064b32ed32 (patch)
tree5c83d28bd304413a8c64f08db864b93094815dd5
parent5737182da70d4f07a15e8d50197dc40251f615f6 (diff)
downloadotp-4b399cdf4472975b0f3d4d21054f30064b32ed32.tar.gz
otp-4b399cdf4472975b0f3d4d21054f30064b32ed32.tar.bz2
otp-4b399cdf4472975b0f3d4d21054f30064b32ed32.zip
Implement handling of group specifications
It's possible now to specify execution of test case groups (and test cases within groups) by means of run_test options and test specification terms. From test specifications, groups (and test cases) may also be skipped. ct_run has been improved to report errors better. New test suites have been introduced.
-rw-r--r--lib/common_test/src/ct_framework.erl2
-rw-r--r--lib/common_test/src/ct_run.erl163
-rw-r--r--lib/common_test/src/ct_testspec.erl20
-rw-r--r--lib/common_test/test/Makefile1
-rw-r--r--lib/common_test/test/ct_groups_test_2_SUITE.erl24
-rw-r--r--lib/common_test/test/ct_groups_test_2_SUITE_data/specs/groups_2.1.spec26
-rw-r--r--lib/common_test/test/ct_test_support.erl2
-rw-r--r--lib/common_test/test/ct_testspec_1_SUITE.erl209
-rw-r--r--lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl281
-rw-r--r--lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl314
-rw-r--r--lib/common_test/test/ct_testspec_1_SUITE_data/groups_2/groups_21_SUITE.erl281
-rw-r--r--lib/common_test/test/ct_testspec_1_SUITE_data/groups_2/groups_22_SUITE.erl314
-rw-r--r--lib/common_test/test/ct_testspec_1_SUITE_data/suites_1/simple_1_SUITE.erl146
-rw-r--r--lib/common_test/test/ct_testspec_1_SUITE_data/suites_1/simple_2_SUITE.erl146
-rw-r--r--lib/test_server/src/test_server_ctrl.erl110
15 files changed, 1904 insertions, 135 deletions
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index 2a21985fdb..25915b9db2 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -682,6 +682,8 @@ get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) ->
%% this makes test_server call error_in_suite as first
%% (and only) test case so we can report Error properly
[{?MODULE,error_in_suite,[[Error]]}];
+ [] ->
+ {error,{invalid_group_spec,Name}};
ConfTests ->
ConfTests
end;
diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index b6ead90a8c..adb49aa44d 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -153,6 +153,7 @@ script_start(Args) ->
Result
end,
stop_trace(Tracing),
+ timer:sleep(1000),
Res.
script_start1(Parent, Args) ->
@@ -777,7 +778,7 @@ run_spec_file(Relaxed,
multiply_timetraps = MultTT,
scale_timetraps = ScaleTT},
{Run,Skip} = ct_testspec:prepare_tests(TS, node()),
- do_run(Run, Skip, Opts1, StartOpts);
+ reformat_result(catch do_run(Run, Skip, Opts1, StartOpts));
{error,GCFReason} ->
exit(GCFReason)
end
@@ -790,7 +791,8 @@ run_prepared(Run, Skip, Opts = #opts{logdir = LogDir,
LogDir1 = which(logdir, LogDir),
case check_and_install_configfiles(CfgFiles, LogDir1, EvHandlers) of
ok ->
- do_run(Run, Skip, Opts#opts{logdir = LogDir1}, StartOpts);
+ reformat_result(catch do_run(Run, Skip, Opts#opts{logdir = LogDir1},
+ StartOpts));
{error,Reason} ->
exit(Reason)
end.
@@ -847,7 +849,7 @@ run_dir(Opts = #opts{logdir = LogDir,
{value,{_,Dirs=[Dir|_]}} when not is_integer(Dir),
length(Dirs)>1 ->
%% multiple dirs (no suite)
- do_run(tests(Dirs), [], Opts1, StartOpts);
+ reformat_result(catch do_run(tests(Dirs), [], Opts1, StartOpts));
false -> % no dir
%% fun for converting suite name to {Dir,Mod} tuple
S2M = fun(S) when is_list(S) ->
@@ -862,12 +864,15 @@ run_dir(Opts = #opts{logdir = LogDir,
case listify(proplists:get_value(group, StartOpts, [])) ++
listify(proplists:get_value(testcase, StartOpts, [])) of
[] ->
- do_run(tests(Dir, listify(Mod)), [], Opts1, StartOpts);
+ reformat_result(catch do_run(tests(Dir, listify(Mod)),
+ [], Opts1, StartOpts));
GsAndCs ->
- do_run(tests(Dir, Mod, GsAndCs), [], Opts1, StartOpts)
+ reformat_result(catch do_run(tests(Dir, Mod, GsAndCs),
+ [], Opts1, StartOpts))
end;
{value,{_,Suites}} ->
- do_run(tests(lists:map(S2M, Suites)), [], Opts1, StartOpts);
+ reformat_result(catch do_run(tests(lists:map(S2M, Suites)),
+ [], Opts1, StartOpts));
_ ->
exit(no_tests_specified)
end;
@@ -880,17 +885,22 @@ run_dir(Opts = #opts{logdir = LogDir,
case listify(proplists:get_value(group, StartOpts, [])) ++
listify(proplists:get_value(testcase, StartOpts, [])) of
[] ->
- do_run(tests(Dir, listify(Mod)), [], Opts1, StartOpts);
+ reformat_result(catch do_run(tests(Dir, listify(Mod)),
+ [], Opts1, StartOpts));
GsAndCs ->
- do_run(tests(Dir, Mod, GsAndCs), [], Opts1, StartOpts)
+ reformat_result(catch do_run(tests(Dir, Mod, GsAndCs),
+ [], Opts1, StartOpts))
end;
{value,{_,Suites=[Suite|_]}} when is_list(Suite) ->
Mods = lists:map(fun(Str) -> list_to_atom(Str) end, Suites),
- do_run(tests(delistify(Dir), Mods), [], Opts1, StartOpts);
+ reformat_result(catch do_run(tests(delistify(Dir), Mods),
+ [], Opts1, StartOpts));
{value,{_,Suites}} ->
- do_run(tests(delistify(Dir), Suites), [], Opts1, StartOpts);
+ reformat_result(catch do_run(tests(delistify(Dir), Suites),
+ [], Opts1, StartOpts));
false -> % no suite, only dir
- do_run(tests(listify(Dir)), [], Opts1, StartOpts)
+ reformat_result(catch do_run(tests(listify(Dir)),
+ [], Opts1, StartOpts))
end
end.
@@ -934,11 +944,11 @@ run_testspec1(TestSpec) ->
case check_and_install_configfiles(Opts#opts.config, LogDir1,
Opts#opts.event_handlers) of
ok ->
- Opts1 = Opts#opts{testspecs = [TestSpec],
+ Opts1 = Opts#opts{testspecs = [],
logdir = LogDir1,
include = AllInclude},
{Run,Skip} = ct_testspec:prepare_tests(TS, node()),
- do_run(Run, Skip, Opts1, []);
+ reformat_result(catch do_run(Run, Skip, Opts1, []));
{error,GCFReason} ->
exit(GCFReason)
end
@@ -1028,21 +1038,26 @@ delistify(E) -> E.
%%% @equiv ct:run/3
run(TestDir, Suite, Cases) ->
install([]),
- do_run(tests(TestDir, Suite, Cases), []).
+ reformat_result(catch do_run(tests(TestDir, Suite, Cases), [])).
%%%-----------------------------------------------------------------
%%% @hidden
%%% @equiv ct:run/2
run(TestDir, Suite) when is_list(TestDir), is_integer(hd(TestDir)) ->
install([]),
- do_run(tests(TestDir, Suite), []).
+ reformat_result(catch do_run(tests(TestDir, Suite), [])).
%%%-----------------------------------------------------------------
%%% @hidden
%%% @equiv ct:run/1
run(TestDirs) ->
install([]),
- do_run(tests(TestDirs), []).
+ reformat_result(catch do_run(tests(TestDirs), [])).
+
+reformat_result({user_error,Reason}) ->
+ {error,Reason};
+reformat_result(Result) ->
+ Result.
suite_to_test(Suite) ->
{filename:dirname(Suite),list_to_atom(filename:rootname(filename:basename(Suite)))}.
@@ -1161,10 +1176,19 @@ do_run(Tests, Skip, Opts, Args) ->
true ->
SavedErrors = save_make_errors(SuiteMakeErrors),
ct_repeat:log_loop_info(Args),
- {Tests1,Skip1} = final_tests(Tests,[],Skip,SavedErrors),
- R = do_run_test(Tests1, Skip1, Opts1),
- ct_util:stop(normal),
- R;
+
+ {Tests1,Skip1} = final_tests(Tests,Skip,SavedErrors),
+
+ R = (catch do_run_test(Tests1, Skip1, Opts1)),
+ case R of
+ {EType,_} = Error when EType == user_error ;
+ EType == error ->
+ ct_util:stop(clean),
+ exit(Error);
+ _ ->
+ ct_util:stop(normal),
+ R
+ end;
false ->
io:nl(),
ct_util:stop(clean),
@@ -1321,8 +1345,20 @@ suite_tuples([{TestDir,Suite,_} | Tests]) when is_atom(Suite) ->
suite_tuples([]) ->
[].
-final_tests([{TestDir,Suites,_}|Tests],
- Final, Skip, Bad) when is_list(Suites), is_atom(hd(Suites)) ->
+final_tests(Tests, Skip, Bad) ->
+
+%%! --- Thu Jun 24 15:47:27 2010 --- peppe was here!
+io:format(user, "FINAL0 = ~p~nSKIP0 = ~p~n", [Tests, Skip]),
+
+ {Tests1,Skip1} = final_tests1(Tests, [], Skip, Bad),
+
+%%! --- Thu Jun 24 15:47:27 2010 --- peppe was here!
+io:format(user, "FINAL1 = ~p~nSKIP1 = ~p~n", [Tests1, Skip1]),
+
+ {Tests1,final_skip(Skip1, [])}.
+
+final_tests1([{TestDir,Suites,_}|Tests], Final, Skip, Bad) when
+ is_list(Suites), is_atom(hd(Suites)) ->
% Separate =
% fun(S,{DoSuite,Dont}) ->
% case lists:keymember({TestDir,S},1,Bad) of
@@ -1341,9 +1377,9 @@ final_tests([{TestDir,Suites,_}|Tests],
Skip1 = [{TD,S,"Make failed"} || {{TD,S},_} <- Bad, S1 <- Suites,
S == S1, TD == TestDir],
Final1 = [{TestDir,S,all} || S <- Suites],
- final_tests(Tests, lists:reverse(Final1)++Final, Skip++Skip1, Bad);
+ final_tests1(Tests, lists:reverse(Final1)++Final, Skip++Skip1, Bad);
-final_tests([{TestDir,all,all}|Tests], Final, Skip, Bad) ->
+final_tests1([{TestDir,all,all}|Tests], Final, Skip, Bad) ->
MissingSuites =
case lists:keysearch({TestDir,all}, 1, Bad) of
{value,{_,Failed}} ->
@@ -1353,40 +1389,47 @@ final_tests([{TestDir,all,all}|Tests], Final, Skip, Bad) ->
end,
Missing = [{TestDir,S,"Make failed"} || S <- MissingSuites],
Final1 = [{TestDir,all,all}|Final],
- final_tests(Tests, Final1, Skip++Missing, Bad);
+ final_tests1(Tests, Final1, Skip++Missing, Bad);
-final_tests([{TestDir,Suite,Cases}|Tests],
- Final, Skip, Bad) when Cases==[]; Cases==all ->
- final_tests([{TestDir,[Suite],all}|Tests], Final, Skip, Bad);
+final_tests1([{TestDir,Suite,Cases}|Tests], Final, Skip, Bad) when
+ Cases==[]; Cases==all ->
+ final_tests1([{TestDir,[Suite],all}|Tests], Final, Skip, Bad);
-final_tests([{TestDir,Suite,Groups}|Tests], Final, Skip, Bad) when
- is_atom(element(1,hd(Groups))) ->
- Confs =
- lists:map(fun({Group,TCs}) ->
- ct_framework:make_conf(TestDir, Suite,
- Group, [], TCs)
- end, Groups),
- Do = {TestDir,Suite,Confs},
+final_tests1([{TestDir,Suite,GrsOrCs}|Tests], Final, Skip, Bad) when
+ is_list(GrsOrCs) ->
case lists:keymember({TestDir,Suite}, 1, Bad) of
- false ->
- final_tests(Tests, [Do|Final], Skip, Bad);
true ->
- Skip1 = Skip ++ [{TestDir,Suite,Confs,"Make failed"}],
- final_tests(Tests, [Do|Final], Skip1, Bad)
- end;
-
-final_tests([Do={TestDir,Suite,Cases}|Tests], Final, Skip, Bad) ->
- case lists:keymember({TestDir,Suite}, 1, Bad) of
- true ->
- Skip1 = Skip ++ [{TestDir,Suite,Cases,"Make failed"}],
- final_tests(Tests, [Do|Final], Skip1, Bad);
+ Skip1 = Skip ++ [{TestDir,Suite,all,"Make failed"}],
+ final_tests1(Tests, [{TestDir,Suite,all}|Final], Skip1, Bad);
false ->
- final_tests(Tests, [Do|Final], Skip, Bad)
+ GrsOrCs1 =
+ lists:map(
+ %% for now, only flat group defs are allowed as
+ %% start options and test spec terms
+ fun({Group,TCs}) ->
+ ct_framework:make_conf(TestDir, Suite,
+ Group, [], TCs);
+ (TC) ->
+ TC
+ end, GrsOrCs),
+ Do = {TestDir,Suite,GrsOrCs1},
+ final_tests1(Tests, [Do|Final], Skip, Bad)
end;
-final_tests([], Final, Skip, _Bad) ->
+final_tests1([], Final, Skip, _Bad) ->
{lists:reverse(Final),Skip}.
+final_skip([{TestDir,Suite,{Group,TCs},Reason}|Skips], Final) ->
+ Conf = ct_framework:make_conf(TestDir, Suite, Group, [], TCs),
+ Skip = {TestDir,Suite,Conf,Reason},
+ final_skip(Skips, [Skip|Final]);
+
+final_skip([Skip|Skips], Final) ->
+ final_skip(Skips, [Skip|Final]);
+
+final_skip([], Final) ->
+ lists:reverse(Final).
+
continue([]) ->
true;
continue(_MakeErrors) ->
@@ -1508,6 +1551,7 @@ do_run_test(Tests, Skip, Opts) ->
_ ->
false
end,
+
%% let test_server expand the test tuples and count no of cases
{Suites,NoOfCases} = count_test_cases(Tests, Skip),
Suites1 = delete_dups(Suites),
@@ -1563,19 +1607,30 @@ count_test_cases(Tests, Skip) ->
TSPid = test_server_ctrl:start_get_totals(SendResult),
Ref = erlang:monitor(process, TSPid),
add_jobs(Tests, Skip, #opts{}, []),
- {Suites,NoOfCases} = count_test_cases1(length(Tests), 0, [], Ref),
+ Counted = (catch count_test_cases1(length(Tests), 0, [], Ref)),
erlang:demonitor(Ref, [flush]),
- test_server_ctrl:stop_get_totals(),
- {Suites,NoOfCases}.
+ case Counted of
+ {error,{test_server_died}} = Error ->
+ throw(Error);
+ {error,Reason} ->
+ unlink(whereis(test_server_ctrl)),
+ test_server_ctrl:stop(),
+ throw({user_error,Reason});
+ Result ->
+ test_server_ctrl:stop_get_totals(),
+ Result
+ end.
count_test_cases1(0, N, Suites, _) ->
{lists:flatten(Suites), N};
count_test_cases1(Jobs, N, Suites, Ref) ->
receive
+ {_,{error,_Reason} = Error} ->
+ throw(Error);
{no_of_cases,{Ss,N1}} ->
count_test_cases1(Jobs-1, add_known(N,N1), [Ss|Suites], Ref);
- {'DOWN', Ref, _, _, _} ->
- {[],0}
+ {'DOWN', Ref, _, _, Info} ->
+ throw({error,{test_server_died,Info}})
end.
add_known(unknown, _) ->
diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl
index 0f68b062f6..9d2a791406 100644
--- a/lib/common_test/src/ct_testspec.erl
+++ b/lib/common_test/src/ct_testspec.erl
@@ -185,7 +185,15 @@ prepare_cases(Node,Dir,Suite,Cases) ->
{[{{Node,Dir},{Suite,all}}],SkipAll};
Skipped ->
%% note: this adds a test even if only skip is specified
- PrepC = lists:foldr(fun({C,{skip,_Cmt}},Acc) ->
+ PrepC = lists:foldr(fun({{G,_}=Group,{skip,_Cmt}}, Acc) when
+ is_atom(G) ->
+ case lists:keymember(G, 1, Cases) of
+ true ->
+ Acc;
+ false ->
+ [Group|Acc]
+ end;
+ ({C,{skip,_Cmt}},Acc) ->
case lists:member(C,Cases) of
true ->
Acc;
@@ -194,7 +202,7 @@ prepare_cases(Node,Dir,Suite,Cases) ->
end;
(C,Acc) -> [C|Acc]
end, [], Cases),
- {{{Node,Dir},{Suite,PrepC}},Skipped}
+ {{{Node,Dir},{Suite,PrepC}},Skipped}
end.
get_skipped_suites(Node,Dir,Suites) ->
@@ -878,8 +886,12 @@ skip_suites(_Node,_Dir,[],_Cmt,Tests) ->
skip_suites(Node,Dir,S,Cmt,Tests) ->
skip_suites(Node,Dir,[S],Cmt,Tests).
-skip_groups(Node,Dir,Suite,Group,Case,Cmt,Tests) when is_atom(Group) ->
- skip_groups(Node,Dir,Suite,[Group],[Case],Cmt,Tests);
+skip_groups(Node,Dir,Suite,Group,all,Cmt,Tests) when is_atom(Group) ->
+ skip_groups(Node,Dir,Suite,[Group],all,Cmt,Tests);
+skip_groups(Node,Dir,Suite,Group,Cases,Cmt,Tests) when is_atom(Group) ->
+ skip_groups(Node,Dir,Suite,[Group],Cases,Cmt,Tests);
+skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests) when is_atom(Case) ->
+ skip_groups(Node,Dir,Suite,Groups,[Case],Cmt,Tests);
skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests) when
((Cases == all) or is_list(Cases)) and is_list(Groups) ->
Suites =
diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile
index 97ded5eb9a..3fb0d627a0 100644
--- a/lib/common_test/test/Makefile
+++ b/lib/common_test/test/Makefile
@@ -32,6 +32,7 @@ MODULES= \
ct_event_handler_SUITE \
ct_groups_test_1_SUITE \
ct_groups_test_2_SUITE \
+ ct_testspec_1_SUITE \
ct_skip_SUITE \
ct_error_SUITE \
ct_test_server_if_1_SUITE \
diff --git a/lib/common_test/test/ct_groups_test_2_SUITE.erl b/lib/common_test/test/ct_groups_test_2_SUITE.erl
index 56e0ac30c7..c4371501b3 100644
--- a/lib/common_test/test/ct_groups_test_2_SUITE.erl
+++ b/lib/common_test/test/ct_groups_test_2_SUITE.erl
@@ -60,7 +60,7 @@ all(doc) ->
["Run smoke tests of Common Test."];
all(suite) ->
- [missing_conf, testspec_1, repeat_1].
+ [missing_conf, repeat_1].
%%--------------------------------------------------------------------
%% TEST CASES
@@ -88,25 +88,6 @@ missing_conf(Config) when is_list(Config) ->
%%%-----------------------------------------------------------------
%%%
-testspec_1(Config) when is_list(Config) ->
- DataDir = ?config(data_dir, Config),
-
- TestSpec = filename:join(DataDir, "specs/groups_2.1.spec"),
-
- {Opts,ERPid} = setup({spec,TestSpec}, Config),
- ok = ct_test_support:run(Opts, Config),
- Events = ct_test_support:get_events(ERPid, Config),
-
- ct_test_support:log_events(testspec_1,
- reformat(Events, ?eh),
- ?config(priv_dir, Config)),
-
- TestEvents = events_to_check(testspec_1),
- ok = ct_test_support:verify_events(TestEvents, Events, Config).
-
-%%%-----------------------------------------------------------------
-%%%
-
repeat_1(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
@@ -173,9 +154,6 @@ test_events(missing_conf) ->
{?eh,stop_logging,[]}
];
-test_events(testspec_1) ->
- [];
-
test_events(repeat_1) ->
[
{?eh,start_logging,{'DEF','RUNDIR'}},
diff --git a/lib/common_test/test/ct_groups_test_2_SUITE_data/specs/groups_2.1.spec b/lib/common_test/test/ct_groups_test_2_SUITE_data/specs/groups_2.1.spec
deleted file mode 100644
index 24d778ac44..0000000000
--- a/lib/common_test/test/ct_groups_test_2_SUITE_data/specs/groups_2.1.spec
+++ /dev/null
@@ -1,26 +0,0 @@
-
-{config, "../cfgs/groups_2.1.cfg"}.
-{alias, groups_2, "../groups_2"}.
-
-{suites, groups_2, groups_21_SUITE}.
-%{skip_groups, groups_2, groups_21_SUITE,
-% [test_group_1b, test_group_7], "Skip tg_1b & tg_7"}.
-{skip_cases, groups_2, groups_21_SUITE,
- [testcase_1b, testcase_3a], "Skip tc_1b & tc_3a"}.
-
-{groups, groups_2, groups_22_SUITE,
- test_group_1a}.
-{skip_cases, groups_2, groups_22_SUITE,
- testcase_1a, "Skip tc_1a"}.
-
-{groups, groups_2, groups_22_SUITE,
- test_group_1b}.
-{skip_cases, groups_2, groups_22_SUITE,
- testcase_1b, "Skip tc_1b"}.
-%{skip_groups, groups_2, groups_21_SUITE,
-% [test_group_3], "Skip tg_3"}.
-
-{groups, groups_2, groups_22_SUITE,
- test_group_5}.
-{skip_cases, groups_2, groups_22_SUITE,
- [testcase_7a, testcase_7b], "Skip tc_7a & tc_7b"}.
diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl
index 6a75c8f789..899c0bc0c3 100644
--- a/lib/common_test/test/ct_test_support.erl
+++ b/lib/common_test/test/ct_test_support.erl
@@ -23,7 +23,7 @@
%%%
-module(ct_test_support).
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
-include_lib("common_test/include/ct_event.hrl").
-export([init_per_suite/1, init_per_suite/2, end_per_suite/1,
diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl
new file mode 100644
index 0000000000..fb7f3fc9a9
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_1_SUITE.erl
@@ -0,0 +1,209 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_testspec_1_SUITE
+%%%
+%%% Description:
+%%% Test test specifications
+%%%
+%%% The suites used for the test are located in the data directory.
+%%%-------------------------------------------------------------------
+-module(ct_testspec_1_SUITE).
+
+-compile(export_all).
+
+-include_lib("test_server/include/test_server.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-define(eh, ct_test_support_eh).
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% Description: Since Common Test starts another Test Server
+%% instance, the tests need to be performed on a separate node (or
+%% there will be clashes with logging processes etc).
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config1 = ct_test_support:init_per_suite(Config),
+ Config1.
+
+end_per_suite(Config) ->
+ ct_test_support:end_per_suite(Config).
+
+init_per_testcase(TestCase, Config) ->
+ ct_test_support:init_per_testcase(TestCase, Config).
+
+end_per_testcase(TestCase, Config) ->
+ ct_test_support:end_per_testcase(TestCase, Config).
+
+all(doc) ->
+ ["Run smoke tests of Common Test."];
+
+all(suite) ->
+ [all_suites, skip_all_suites,
+ suite, skip_suite].
+% cases_1, skip_cases_1,
+% groups_1, skip_groups_1].
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+%%%-----------------------------------------------------------------
+%%%
+
+all_suites(Config) when is_list(Config) ->
+ Self = all_suites,
+ DataDir = ?config(data_dir, Config),
+
+ TestDir = filename:join(DataDir, "suites_1"),
+ TestSpec = [{suites,TestDir,all}],
+ SpecFile = create_spec_file(?config(priv_dir, Config),
+ Self, TestSpec),
+
+ {Opts,ERPid} = setup({spec,SpecFile}, Config),
+ ok = ct_test_support:run(Opts, Config),
+ ok = ct_test_support:run(ct, run_testspec, [TestSpec], Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(Self,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config)),
+
+ TestEvents = events_to_check(Self),
+ ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
+skip_all_suites(Config) when is_list(Config) ->
+ Self = skip_all_suites,
+ DataDir = ?config(data_dir, Config),
+
+ TestDir = filename:join(DataDir, "suites_1"),
+ TestSpec = [{suites,TestDir,all},
+ {skip_suites,TestDir,all,"SKIPPED!"}],
+ SpecFile = create_spec_file(?config(priv_dir, Config),
+ Self, TestSpec),
+
+ {Opts,ERPid} = setup({spec,SpecFile}, Config),
+ ok = ct_test_support:run(Opts, Config),
+ ok = ct_test_support:run(ct, run_testspec, [TestSpec], Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(Self,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config)),
+
+ TestEvents = events_to_check(Self),
+ ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+suite(Config) when is_list(Config) ->
+ Self = suite,
+ DataDir = ?config(data_dir, Config),
+
+ TestDir = filename:join(DataDir, "suites_1"),
+ TestSpec = [{suites,TestDir,simple_1_SUITE}],
+ SpecFile = create_spec_file(?config(priv_dir, Config),
+ Self, TestSpec),
+
+ {Opts,ERPid} = setup({spec,SpecFile}, Config),
+ ok = ct_test_support:run(Opts, Config),
+ ok = ct_test_support:run(ct, run_testspec, [TestSpec], Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(Self,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config)),
+
+ TestEvents = events_to_check(Self),
+ ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
+skip_suite(Config) when is_list(Config) ->
+ Self = skip_suite,
+ DataDir = ?config(data_dir, Config),
+
+ TestDir = filename:join(DataDir, "suites_1"),
+ TestSpec = [{suites,TestDir,[simple_1_SUITE,simple_2_SUITE]},
+ {skip_suites,TestDir,simple_1_SUITE,"SKIPPED!"}],
+ SpecFile = create_spec_file(?config(priv_dir, Config),
+ Self, TestSpec),
+
+ {Opts,ERPid} = setup({spec,SpecFile}, Config),
+ ok = ct_test_support:run(Opts, Config),
+ ok = ct_test_support:run(ct, run_testspec, [TestSpec], Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(Self,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config)),
+
+ TestEvents = events_to_check(Self),
+ ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
+
+%%%-----------------------------------------------------------------
+%%%
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+
+create_spec_file(SpecDir, TCName, TestSpec) ->
+ FileName = filename:join(SpecDir,
+ atom_to_list(TCName)++".spec"),
+ {ok,Dev} = file:open(FileName, [write]),
+ [io:format(Dev, "~p.~n", [Term]) || Term <- TestSpec],
+ file:close(Dev),
+ io:format("~nTest spec created here~n~n<a href=\"file://~s\">~s</a>~n",
+ [FileName,FileName]),
+ FileName.
+
+setup(Test, Config) ->
+ Opts0 = ct_test_support:get_opts(Config),
+ Level = ?config(trace_level, Config),
+ EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+ Opts = Opts0 ++ [Test,{event_handler,{?eh,EvHArgs}}],
+ ERPid = ct_test_support:start_event_receiver(Config),
+ {Opts,ERPid}.
+
+reformat(Events, EH) ->
+ ct_test_support:reformat(Events, EH).
+%reformat(Events, _EH) ->
+% Events.
+
+%%%-----------------------------------------------------------------
+%%% TEST EVENTS
+%%%-----------------------------------------------------------------
+events_to_check(Test) ->
+ %% 2 tests (ct:run_test + script_start) is default
+ events_to_check(Test, 2).
+
+events_to_check(_, 0) ->
+ [];
+events_to_check(Test, N) ->
+ test_events(Test) ++ events_to_check(Test, N-1).
+
+test_events(_) ->
+ [
+ ].
diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl
new file mode 100644
index 0000000000..4f11d8a0e8
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl
@@ -0,0 +1,281 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(groups_11_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%====================================================================
+%% COMMON TEST CALLBACK FUNCTIONS
+%%====================================================================
+
+suite() ->
+ [{timetrap,{minutes,1}}].
+
+groups() ->
+ [
+ {test_group_1a, [testcase_1a,testcase_1b]},
+
+ {test_group_1b, [], [testcase_1a,testcase_1b]},
+
+ {test_group_2, [], [testcase_2a,
+
+ {test_group_3, [], [testcase_3a,
+ testcase_3b]},
+ testcase_2b]},
+
+ {test_group_4, [{test_group_5, [], [testcase_5a,
+
+ {group, test_group_6},
+
+ testcase_5b]}]},
+ {test_group_6, [{group, test_group_7}]},
+
+ {test_group_7, [testcase_7a,testcase_7b]}
+ ].
+
+all() ->
+ [testcase_1,
+ {group, test_group_1a},
+ {group, test_group_1b},
+ testcase_2,
+ {group, test_group_2},
+ testcase_3,
+ {group, test_group_4}].
+
+%% this func only for internal test purposes
+grs_and_tcs() ->
+ {[
+ test_group_1a, test_group_1b,
+ test_group_2, test_group_3,
+ test_group_4, test_group_5,
+ test_group_6, test_group_7
+ ],
+ [
+ testcase_1,
+ testcase_1a, testcase_1b,
+ testcase_2,
+ testcase_2a, testcase_2b,
+ testcase_3a, testcase_3b,
+ testcase_3,
+ testcase_5a, testcase_5b,
+ testcase_7a, testcase_7b
+ ]}.
+
+%%--------------------------------------------------------------------
+%% Suite Configuration
+%%--------------------------------------------------------------------
+
+init_per_suite(Config) ->
+ [{suite,init}|Config].
+
+end_per_suite(Config) ->
+ init = ?config(suite,Config).
+
+%%--------------------------------------------------------------------
+%% Group Configuration
+%%--------------------------------------------------------------------
+
+init_per_group(Group, Config) ->
+ [{name,Group}] = ?config(tc_group_properties,Config),
+ {Grs,_} = grs_and_tcs(),
+ case lists:member(Group, Grs) of
+ true ->
+ ct:comment(Group),
+ init = ?config(suite,Config),
+ [{Group,Group} | Config];
+ false ->
+ ct:fail({bad_group,Group})
+ end.
+
+end_per_group(Group, Config) ->
+ {Grs,_} = grs_and_tcs(),
+ case lists:member(Group, Grs) of
+ true ->
+ ct:comment(Group),
+ init = ?config(suite,Config),
+ Group = ?config(Group,Config),
+ ok;
+ false ->
+ ct:fail({bad_group,Group})
+ end.
+
+%%--------------------------------------------------------------------
+%% Testcase Configuration
+%%--------------------------------------------------------------------
+
+init_per_testcase(TestCase, Config) ->
+ {_,TCs} = grs_and_tcs(),
+ case lists:member(TestCase, TCs) of
+ true ->
+ init = ?config(suite,Config),
+ [{TestCase,TestCase} | Config];
+ false ->
+ ct:fail({unknown_testcase,TestCase})
+ end.
+
+end_per_testcase(TestCase, Config) ->
+ {_,TCs} = grs_and_tcs(),
+ case lists:member(TestCase, TCs) of
+ true ->
+ init = ?config(suite,Config),
+ TestCase = ?config(TestCase,Config),
+ ok;
+ false ->
+ ct:fail({unknown_testcase,TestCase})
+ end.
+
+
+%%--------------------------------------------------------------------
+%% Testcases
+%%--------------------------------------------------------------------
+
+testcase_1() ->
+ [].
+testcase_1(Config) ->
+ init = ?config(suite,Config),
+ testcase_1 = ?config(testcase_1,Config),
+ ok.
+
+testcase_1a() ->
+ [].
+testcase_1a(Config) ->
+ init = ?config(suite,Config),
+ case ?config(test_group_1a,Config) of
+ test_group_1a -> ok;
+ _ ->
+ case ?config(test_group_1b,Config) of
+ test_group_1b -> ok;
+ _ -> ct:fail(no_group_data)
+ end
+ end,
+ testcase_1a = ?config(testcase_1a,Config),
+ ok.
+testcase_1b() ->
+ [].
+testcase_1b(Config) ->
+ init = ?config(suite,Config),
+ case ?config(test_group_1a,Config) of
+ test_group_1a -> ok;
+ _ ->
+ case ?config(test_group_1b,Config) of
+ test_group_1b -> ok;
+ _ -> ct:fail(no_group_data)
+ end
+ end,
+ undefined = ?config(testcase_1a,Config),
+ testcase_1b = ?config(testcase_1b,Config),
+ ok.
+
+testcase_2() ->
+ [].
+testcase_2(Config) ->
+ init = ?config(suite,Config),
+ undefined = ?config(test_group_1a,Config),
+ undefined = ?config(test_group_1b,Config),
+ testcase_2 = ?config(testcase_2,Config),
+ ok.
+
+testcase_2a() ->
+ [].
+testcase_2a(Config) ->
+ init = ?config(suite,Config),
+ test_group_2 = ?config(test_group_2,Config),
+ testcase_2a = ?config(testcase_2a,Config),
+ ok.
+testcase_2b() ->
+ [].
+testcase_2b(Config) ->
+ init = ?config(suite,Config),
+ test_group_2 = ?config(test_group_2,Config),
+ undefined = ?config(testcase_2a,Config),
+ testcase_2b = ?config(testcase_2b,Config),
+ ok.
+
+testcase_3a() ->
+ [].
+testcase_3a(Config) ->
+ init = ?config(suite,Config),
+ test_group_2 = ?config(test_group_2,Config),
+ test_group_3 = ?config(test_group_3,Config),
+ undefined = ?config(testcase_2b,Config),
+ testcase_3a = ?config(testcase_3a,Config),
+ ok.
+testcase_3b() ->
+ [].
+testcase_3b(Config) ->
+ init = ?config(suite,Config),
+ test_group_2 = ?config(test_group_2,Config),
+ test_group_3 = ?config(test_group_3,Config),
+ undefined = ?config(testcase_3a,Config),
+ testcase_3b = ?config(testcase_3b,Config),
+ ok.
+
+testcase_3() ->
+ [].
+testcase_3(Config) ->
+ init = ?config(suite,Config),
+ undefined = ?config(test_group_2,Config),
+ undefined = ?config(test_group_3,Config),
+ testcase_3 = ?config(testcase_3,Config),
+ ok.
+
+testcase_5a() ->
+ [].
+testcase_5a(Config) ->
+ init = ?config(suite,Config),
+ undefined = ?config(test_group_3,Config),
+ test_group_4 = ?config(test_group_4,Config),
+ test_group_5 = ?config(test_group_5,Config),
+ undefined = ?config(testcase_3,Config),
+ testcase_5a = ?config(testcase_5a,Config),
+ ok.
+testcase_5b() ->
+ [].
+testcase_5b(Config) ->
+ init = ?config(suite,Config),
+ test_group_4 = ?config(test_group_4,Config),
+ test_group_5 = ?config(test_group_5,Config),
+ undefined = ?config(testcase_5a,Config),
+ testcase_5b = ?config(testcase_5b,Config),
+ ok.
+
+testcase_7a() ->
+ [].
+testcase_7a(Config) ->
+ init = ?config(suite,Config),
+ undefined = ?config(test_group_3,Config),
+ test_group_4 = ?config(test_group_4,Config),
+ test_group_5 = ?config(test_group_5,Config),
+ test_group_6 = ?config(test_group_6,Config),
+ test_group_7 = ?config(test_group_7,Config),
+ testcase_7a = ?config(testcase_7a,Config),
+ ok.
+testcase_7b() ->
+ [].
+testcase_7b(Config) ->
+ init = ?config(suite,Config),
+ test_group_4 = ?config(test_group_4,Config),
+ test_group_5 = ?config(test_group_5,Config),
+ test_group_6 = ?config(test_group_6,Config),
+ test_group_7 = ?config(test_group_7,Config),
+ undefined = ?config(testcase_7a,Config),
+ testcase_7b = ?config(testcase_7b,Config),
+ ok.
diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl
new file mode 100644
index 0000000000..dbf1576c76
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl
@@ -0,0 +1,314 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(groups_12_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%====================================================================
+%% COMMON TEST CALLBACK FUNCTIONS
+%%====================================================================
+
+suite() ->
+ [{timetrap,{minutes,1}}].
+
+groups() ->
+ [
+ {test_group_1a, [shuffle], [testcase_1a,testcase_1b,testcase_1c]},
+
+ {test_group_1b, [parallel], [testcase_1a,testcase_1b]},
+
+ {test_group_2, [parallel], [testcase_2a,
+
+ {test_group_3, [{repeat,2}],
+ [testcase_3a, testcase_3b]},
+
+ testcase_2b]},
+
+ {test_group_4, [{test_group_5, [parallel], [testcase_5a,
+
+ {group, test_group_6},
+
+ testcase_5b]}]},
+
+ {test_group_6, [parallel], [{group, test_group_7}]},
+
+ {test_group_7, [sequence], [testcase_7a,testcase_7b]}
+ ].
+
+all() ->
+ [{group, test_group_1a},
+ {group, test_group_1b},
+ testcase_1,
+ testcase_2,
+ {group, test_group_2},
+ testcase_3,
+ {group, test_group_4}].
+
+%% this func only for internal test purposes
+grs_and_tcs() ->
+ {[
+ test_group_1a, test_group_1b,
+ test_group_2, test_group_3,
+ test_group_4, test_group_5,
+ test_group_6, test_group_7
+ ],
+ [
+ testcase_1a, testcase_1b, testcase_1c,
+ testcase_1,
+ testcase_2,
+ testcase_2a, testcase_2b,
+ testcase_3a, testcase_3b,
+ testcase_3,
+ testcase_5a, testcase_5b,
+ testcase_7a, testcase_7b
+ ]}.
+
+%%--------------------------------------------------------------------
+%% Suite Configuration
+%%--------------------------------------------------------------------
+
+init_per_suite(Config) ->
+ [{suite,init}|Config].
+
+end_per_suite(Config) ->
+ init = ?config(suite,Config).
+
+%%--------------------------------------------------------------------
+%% Group Configuration
+%%--------------------------------------------------------------------
+
+init_per_group(Group, Config) ->
+ Cmt =
+ case {Group,?config(tc_group_properties,Config)} of
+ {test_group_1a,[{shuffle,S},{name,test_group_1a}]} ->
+ io_lib:format("shuffled, ~w", [S]);
+ {test_group_1b,[{name,test_group_1b},parallel]} -> "parallel";
+ {test_group_2,[{name,test_group_2},parallel]} -> "parallel";
+ {test_group_3,[{name,test_group_3},{repeat,2}]} -> "repeat 2";
+ {test_group_3,[{name,test_group_3}]} -> "repeat 1";
+ {test_group_4,[{name,test_group_4}]} -> ok;
+ {test_group_5,[{name,test_group_5},parallel]} -> "parallel";
+ {test_group_6,[{name,test_group_6},parallel]} -> "parallel";
+ {test_group_7,[{name,test_group_7},sequence]} -> "sequence"
+ end,
+ {Grs,_} = grs_and_tcs(),
+ case lists:member(Group, Grs) of
+ true ->
+ init = ?config(suite,Config),
+ ct:comment(io_lib:format("~w, ~s", [Group,Cmt])),
+ [{Group,Group} | Config];
+ false ->
+ ct:fail({bad_group,Group})
+ end.
+
+end_per_group(Group, Config) ->
+ {Grs,_} = grs_and_tcs(),
+ case lists:member(Group, Grs) of
+ true ->
+ init = ?config(suite,Config),
+ Group = ?config(Group,Config),
+ ok;
+ false ->
+ ct:fail({bad_group,Group})
+ end.
+
+%%--------------------------------------------------------------------
+%% Testcase Configuration
+%%--------------------------------------------------------------------
+
+init_per_testcase(TestCase, Config) ->
+ {_,TCs} = grs_and_tcs(),
+ case lists:member(TestCase, TCs) of
+ true ->
+ init = ?config(suite,Config),
+ [{TestCase,TestCase} | Config];
+ false ->
+ ct:fail({unknown_testcase,TestCase})
+ end.
+
+end_per_testcase(TestCase, Config) ->
+ {_,TCs} = grs_and_tcs(),
+ case lists:member(TestCase, TCs) of
+ true ->
+ init = ?config(suite,Config),
+ TestCase = ?config(TestCase,Config),
+ ok;
+ false ->
+ ct:fail({unknown_testcase,TestCase})
+ end.
+
+
+%%--------------------------------------------------------------------
+%% Testcases
+%%--------------------------------------------------------------------
+
+testcase_1a() ->
+ [].
+testcase_1a(Config) ->
+ init = ?config(suite,Config),
+ case ?config(test_group_1a,Config) of
+ test_group_1a -> ok;
+ _ ->
+ case ?config(test_group_1b,Config) of
+ test_group_1b -> ok;
+ _ -> ct:fail(no_group_data)
+ end
+ end,
+ testcase_1a = ?config(testcase_1a,Config),
+ ok.
+testcase_1b() ->
+ [].
+testcase_1b(Config) ->
+ init = ?config(suite,Config),
+ case ?config(test_group_1a,Config) of
+ test_group_1a -> ok;
+ _ ->
+ case ?config(test_group_1b,Config) of
+ test_group_1b -> ok;
+ _ -> ct:fail(no_group_data)
+ end
+ end,
+ undefined = ?config(testcase_1a,Config),
+ testcase_1b = ?config(testcase_1b,Config),
+ ok.
+
+testcase_1c() ->
+ [].
+testcase_1c(Config) ->
+ init = ?config(suite,Config),
+ case ?config(test_group_1a,Config) of
+ test_group_1a -> ok;
+ _ ->
+ case ?config(test_group_1b,Config) of
+ test_group_1b -> ok;
+ _ -> ct:fail(no_group_data)
+ end
+ end,
+ undefined = ?config(testcase_1b,Config),
+ testcase_1c = ?config(testcase_1c,Config),
+ ok.
+
+testcase_1() ->
+ [].
+testcase_1(Config) ->
+ init = ?config(suite,Config),
+ testcase_1 = ?config(testcase_1,Config),
+ ok.
+
+testcase_2() ->
+ [].
+testcase_2(Config) ->
+ init = ?config(suite,Config),
+ undefined = ?config(test_group_1a,Config),
+ undefined = ?config(test_group_1b,Config),
+ testcase_2 = ?config(testcase_2,Config),
+ ok.
+
+testcase_2a() ->
+ [].
+testcase_2a(Config) ->
+ init = ?config(suite,Config),
+ test_group_2 = ?config(test_group_2,Config),
+ testcase_2a = ?config(testcase_2a,Config),
+ ok.
+testcase_2b() ->
+ [].
+testcase_2b(Config) ->
+ init = ?config(suite,Config),
+ test_group_2 = ?config(test_group_2,Config),
+ undefined = ?config(testcase_2a,Config),
+ testcase_2b = ?config(testcase_2b,Config),
+ ok.
+
+testcase_3a() ->
+ [].
+testcase_3a(Config) ->
+ init = ?config(suite,Config),
+ test_group_2 = ?config(test_group_2,Config),
+ test_group_3 = ?config(test_group_3,Config),
+ undefined = ?config(testcase_2b,Config),
+ testcase_3a = ?config(testcase_3a,Config),
+ ok.
+testcase_3b() ->
+ [].
+testcase_3b(Config) ->
+ init = ?config(suite,Config),
+ test_group_2 = ?config(test_group_2,Config),
+ test_group_3 = ?config(test_group_3,Config),
+ undefined = ?config(testcase_3a,Config),
+ testcase_3b = ?config(testcase_3b,Config),
+ ok.
+
+testcase_3() ->
+ [].
+testcase_3(Config) ->
+ init = ?config(suite,Config),
+ undefined = ?config(test_group_2,Config),
+ undefined = ?config(test_group_3,Config),
+ testcase_3 = ?config(testcase_3,Config),
+ ok.
+
+testcase_5a() ->
+ [].
+testcase_5a(Config) ->
+ init = ?config(suite,Config),
+ undefined = ?config(test_group_3,Config),
+ test_group_4 = ?config(test_group_4,Config),
+ test_group_5 = ?config(test_group_5,Config),
+ undefined = ?config(testcase_3,Config),
+ testcase_5a = ?config(testcase_5a,Config),
+ %% increase chance the done event will come
+ %% during execution of subgroup (could be
+ %% tricky to handle)
+ timer:sleep(3),
+ ok.
+testcase_5b() ->
+ [].
+testcase_5b(Config) ->
+ init = ?config(suite,Config),
+ test_group_4 = ?config(test_group_4,Config),
+ test_group_5 = ?config(test_group_5,Config),
+ undefined = ?config(testcase_5a,Config),
+ testcase_5b = ?config(testcase_5b,Config),
+ ok.
+
+testcase_7a() ->
+ [].
+testcase_7a(Config) ->
+ init = ?config(suite,Config),
+ undefined = ?config(test_group_3,Config),
+ test_group_4 = ?config(test_group_4,Config),
+ test_group_5 = ?config(test_group_5,Config),
+ test_group_6 = ?config(test_group_6,Config),
+ test_group_7 = ?config(test_group_7,Config),
+ testcase_7a = ?config(testcase_7a,Config),
+ ok.
+testcase_7b() ->
+ [].
+testcase_7b(Config) ->
+ init = ?config(suite,Config),
+ test_group_4 = ?config(test_group_4,Config),
+ test_group_5 = ?config(test_group_5,Config),
+ test_group_6 = ?config(test_group_6,Config),
+ test_group_7 = ?config(test_group_7,Config),
+ undefined = ?config(testcase_7a,Config),
+ testcase_7b = ?config(testcase_7b,Config),
+ ok.
diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_2/groups_21_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_2/groups_21_SUITE.erl
new file mode 100644
index 0000000000..2533ac8e84
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_2/groups_21_SUITE.erl
@@ -0,0 +1,281 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(groups_21_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%====================================================================
+%% COMMON TEST CALLBACK FUNCTIONS
+%%====================================================================
+
+suite() ->
+ [{timetrap,{minutes,1}}].
+
+groups() ->
+ [
+ {test_group_1a, [testcase_1a,testcase_1b]},
+
+ {test_group_1b, [], [testcase_1a,testcase_1b]},
+
+ {test_group_2, [], [testcase_2a,
+
+ {test_group_3, [], [testcase_3a,
+ testcase_3b]},
+ testcase_2b]},
+
+ {test_group_4, [{test_group_5, [], [testcase_5a,
+
+ {group, test_group_6},
+
+ testcase_5b]}]},
+
+ {test_group_6, [{group, test_group_7}]},
+
+ {test_group_7, [testcase_7a,testcase_7b]}
+ ].
+
+all() ->
+ [testcase_1,
+ {group, test_group_1a},
+ {group, test_group_1b},
+ testcase_2,
+ {group, test_group_2},
+ testcase_3,
+ {group, test_group_4}].
+
+%% this func only for internal test purposes
+grs_and_tcs() ->
+ {[
+ test_group_1a, test_group_1b,
+ test_group_2, test_group_3,
+ test_group_4, test_group_5,
+ test_group_6, test_group_7
+ ],
+ [
+ testcase_1,
+ testcase_1a, testcase_1b,
+ testcase_2,
+ testcase_2a, testcase_2b,
+ testcase_3a, testcase_3b,
+ testcase_3,
+ testcase_5a, testcase_5b,
+ testcase_7a, testcase_7b
+ ]}.
+
+%%--------------------------------------------------------------------
+%% Suite Configuration
+%%--------------------------------------------------------------------
+
+init_per_suite(Config) ->
+ [{suite,init}|Config].
+
+end_per_suite(Config) ->
+ init = ?config(suite,Config).
+
+%%--------------------------------------------------------------------
+%% Group Configuration
+%%--------------------------------------------------------------------
+
+init_per_group(Group, Config) ->
+ [{name,Group}] = ?config(tc_group_properties,Config),
+ {Grs,_} = grs_and_tcs(),
+ case lists:member(Group, Grs) of
+ true ->
+ ct:comment(io_lib:format("~w", [Group])),
+ init = ?config(suite,Config),
+ [{Group,Group} | Config];
+ false ->
+ ct:fail({bad_group,Group})
+ end.
+
+end_per_group(Group, Config) ->
+ {Grs,_} = grs_and_tcs(),
+ case lists:member(Group, Grs) of
+ true ->
+ init = ?config(suite,Config),
+ Group = ?config(Group,Config),
+ ok;
+ false ->
+ ct:fail({bad_group,Group})
+ end.
+
+%%--------------------------------------------------------------------
+%% Testcase Configuration
+%%--------------------------------------------------------------------
+
+init_per_testcase(TestCase, Config) ->
+ {_,TCs} = grs_and_tcs(),
+ case lists:member(TestCase, TCs) of
+ true ->
+ init = ?config(suite,Config),
+ [{TestCase,TestCase} | Config];
+ false ->
+ ct:fail({unknown_testcase,TestCase})
+ end.
+
+end_per_testcase(TestCase, Config) ->
+ {_,TCs} = grs_and_tcs(),
+ case lists:member(TestCase, TCs) of
+ true ->
+ init = ?config(suite,Config),
+ TestCase = ?config(TestCase,Config),
+ ok;
+ false ->
+ ct:fail({unknown_testcase,TestCase})
+ end.
+
+
+%%--------------------------------------------------------------------
+%% Testcases
+%%--------------------------------------------------------------------
+
+testcase_1() ->
+ [].
+testcase_1(Config) ->
+ init = ?config(suite,Config),
+ testcase_1 = ?config(testcase_1,Config),
+ ok.
+
+testcase_1a() ->
+ [].
+testcase_1a(Config) ->
+ init = ?config(suite,Config),
+ case ?config(test_group_1a,Config) of
+ test_group_1a -> ok;
+ _ ->
+ case ?config(test_group_1b,Config) of
+ test_group_1b -> ok;
+ _ -> ct:fail(no_group_data)
+ end
+ end,
+ testcase_1a = ?config(testcase_1a,Config),
+ ok.
+testcase_1b() ->
+ [].
+testcase_1b(Config) ->
+ init = ?config(suite,Config),
+ case ?config(test_group_1a,Config) of
+ test_group_1a -> ok;
+ _ ->
+ case ?config(test_group_1b,Config) of
+ test_group_1b -> ok;
+ _ -> ct:fail(no_group_data)
+ end
+ end,
+ undefined = ?config(testcase_1a,Config),
+ testcase_1b = ?config(testcase_1b,Config),
+ ok.
+
+testcase_2() ->
+ [].
+testcase_2(Config) ->
+ init = ?config(suite,Config),
+ undefined = ?config(test_group_1a,Config),
+ undefined = ?config(test_group_1b,Config),
+ testcase_2 = ?config(testcase_2,Config),
+ ok.
+
+testcase_2a() ->
+ [].
+testcase_2a(Config) ->
+ init = ?config(suite,Config),
+ test_group_2 = ?config(test_group_2,Config),
+ testcase_2a = ?config(testcase_2a,Config),
+ ok.
+testcase_2b() ->
+ [].
+testcase_2b(Config) ->
+ init = ?config(suite,Config),
+ test_group_2 = ?config(test_group_2,Config),
+ undefined = ?config(testcase_2a,Config),
+ testcase_2b = ?config(testcase_2b,Config),
+ ok.
+
+testcase_3a() ->
+ [].
+testcase_3a(Config) ->
+ init = ?config(suite,Config),
+ test_group_2 = ?config(test_group_2,Config),
+ test_group_3 = ?config(test_group_3,Config),
+ undefined = ?config(testcase_2b,Config),
+ testcase_3a = ?config(testcase_3a,Config),
+ ok.
+testcase_3b() ->
+ [].
+testcase_3b(Config) ->
+ init = ?config(suite,Config),
+ test_group_2 = ?config(test_group_2,Config),
+ test_group_3 = ?config(test_group_3,Config),
+ undefined = ?config(testcase_3a,Config),
+ testcase_3b = ?config(testcase_3b,Config),
+ ok.
+
+testcase_3() ->
+ [].
+testcase_3(Config) ->
+ init = ?config(suite,Config),
+ undefined = ?config(test_group_2,Config),
+ undefined = ?config(test_group_3,Config),
+ testcase_3 = ?config(testcase_3,Config),
+ ok.
+
+testcase_5a() ->
+ [].
+testcase_5a(Config) ->
+ init = ?config(suite,Config),
+ undefined = ?config(test_group_3,Config),
+ test_group_4 = ?config(test_group_4,Config),
+ test_group_5 = ?config(test_group_5,Config),
+ undefined = ?config(testcase_3,Config),
+ testcase_5a = ?config(testcase_5a,Config),
+ ok.
+testcase_5b() ->
+ [].
+testcase_5b(Config) ->
+ init = ?config(suite,Config),
+ test_group_4 = ?config(test_group_4,Config),
+ test_group_5 = ?config(test_group_5,Config),
+ undefined = ?config(testcase_5a,Config),
+ testcase_5b = ?config(testcase_5b,Config),
+ ok.
+
+testcase_7a() ->
+ [].
+testcase_7a(Config) ->
+ init = ?config(suite,Config),
+ undefined = ?config(test_group_3,Config),
+ test_group_4 = ?config(test_group_4,Config),
+ test_group_5 = ?config(test_group_5,Config),
+ test_group_6 = ?config(test_group_6,Config),
+ test_group_7 = ?config(test_group_7,Config),
+ testcase_7a = ?config(testcase_7a,Config),
+ ok.
+testcase_7b() ->
+ [].
+testcase_7b(Config) ->
+ init = ?config(suite,Config),
+ test_group_4 = ?config(test_group_4,Config),
+ test_group_5 = ?config(test_group_5,Config),
+ test_group_6 = ?config(test_group_6,Config),
+ test_group_7 = ?config(test_group_7,Config),
+ undefined = ?config(testcase_7a,Config),
+ testcase_7b = ?config(testcase_7b,Config),
+ ok.
diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_2/groups_22_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_2/groups_22_SUITE.erl
new file mode 100644
index 0000000000..cd517876df
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_2/groups_22_SUITE.erl
@@ -0,0 +1,314 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(groups_22_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%====================================================================
+%% COMMON TEST CALLBACK FUNCTIONS
+%%====================================================================
+
+suite() ->
+ [{timetrap,{minutes,1}}].
+
+groups() ->
+ [
+ {test_group_1a, [shuffle], [testcase_1a,testcase_1b,testcase_1c]},
+
+ {test_group_1b, [parallel], [testcase_1a,testcase_1b]},
+
+ {test_group_2, [parallel], [testcase_2a,
+
+ {test_group_3, [{repeat,1}],
+ [testcase_3a, testcase_3b]},
+
+ testcase_2b]},
+
+ {test_group_4, [{test_group_5, [parallel], [testcase_5a,
+
+ {group, test_group_6},
+
+ testcase_5b]}]},
+
+ {test_group_6, [parallel], [{group, test_group_7}]},
+
+ {test_group_7, [sequence], [testcase_7a,testcase_7b]}
+ ].
+
+all() ->
+ [{group, test_group_1a},
+ {group, test_group_1b},
+ testcase_1,
+ testcase_2,
+ {group, test_group_2},
+ testcase_3,
+ {group, test_group_4}].
+
+%% this func only for internal test purposes
+grs_and_tcs() ->
+ {[
+ test_group_1a, test_group_1b,
+ test_group_2, test_group_3,
+ test_group_4, test_group_5,
+ test_group_6, test_group_7
+ ],
+ [
+ testcase_1a, testcase_1b, testcase_1c,
+ testcase_1,
+ testcase_2,
+ testcase_2a, testcase_2b,
+ testcase_3a, testcase_3b,
+ testcase_3,
+ testcase_5a, testcase_5b,
+ testcase_7a, testcase_7b
+ ]}.
+
+%%--------------------------------------------------------------------
+%% Suite Configuration
+%%--------------------------------------------------------------------
+
+init_per_suite(Config) ->
+ [{suite,init}|Config].
+
+end_per_suite(Config) ->
+ init = ?config(suite,Config).
+
+%%--------------------------------------------------------------------
+%% Group Configuration
+%%--------------------------------------------------------------------
+
+init_per_group(Group, Config) ->
+ Cmt =
+ case {Group,?config(tc_group_properties,Config)} of
+ {test_group_1a,[{shuffle,S},{name,test_group_1a}]} ->
+ io_lib:format("shuffled, ~w", [S]);
+ {test_group_1b,[{name,test_group_1b},parallel]} -> "parallel";
+ {test_group_2,[{name,test_group_2},parallel]} -> "parallel";
+ {test_group_3,[{name,test_group_3},{repeat,1}]} -> "repeat 1";
+ {test_group_3,[{name,test_group_3}]} -> "repeat 0";
+ {test_group_4,[{name,test_group_4}]} -> ok;
+ {test_group_5,[{name,test_group_5},parallel]} -> "parallel";
+ {test_group_6,[{name,test_group_6},parallel]} -> "parallel";
+ {test_group_7,[{name,test_group_7},sequence]} -> "sequence"
+ end,
+ {Grs,_} = grs_and_tcs(),
+ case lists:member(Group, Grs) of
+ true ->
+ init = ?config(suite,Config),
+ ct:comment(io_lib:format("~w, ~s", [Group,Cmt])),
+ [{Group,Group} | Config];
+ false ->
+ ct:fail({bad_group,Group})
+ end.
+
+end_per_group(Group, Config) ->
+ {Grs,_} = grs_and_tcs(),
+ case lists:member(Group, Grs) of
+ true ->
+ init = ?config(suite,Config),
+ Group = ?config(Group,Config),
+ ok;
+ false ->
+ ct:fail({bad_group,Group})
+ end.
+
+%%--------------------------------------------------------------------
+%% Testcase Configuration
+%%--------------------------------------------------------------------
+
+init_per_testcase(TestCase, Config) ->
+ {_,TCs} = grs_and_tcs(),
+ case lists:member(TestCase, TCs) of
+ true ->
+ init = ?config(suite,Config),
+ [{TestCase,TestCase} | Config];
+ false ->
+ ct:fail({unknown_testcase,TestCase})
+ end.
+
+end_per_testcase(TestCase, Config) ->
+ {_,TCs} = grs_and_tcs(),
+ case lists:member(TestCase, TCs) of
+ true ->
+ init = ?config(suite,Config),
+ TestCase = ?config(TestCase,Config),
+ ok;
+ false ->
+ ct:fail({unknown_testcase,TestCase})
+ end.
+
+
+%%--------------------------------------------------------------------
+%% Testcases
+%%--------------------------------------------------------------------
+
+testcase_1a() ->
+ [].
+testcase_1a(Config) ->
+ init = ?config(suite,Config),
+ case ?config(test_group_1a,Config) of
+ test_group_1a -> ok;
+ _ ->
+ case ?config(test_group_1b,Config) of
+ test_group_1b -> ok;
+ _ -> ct:fail(no_group_data)
+ end
+ end,
+ testcase_1a = ?config(testcase_1a,Config),
+ ok.
+testcase_1b() ->
+ [].
+testcase_1b(Config) ->
+ init = ?config(suite,Config),
+ case ?config(test_group_1a,Config) of
+ test_group_1a -> ok;
+ _ ->
+ case ?config(test_group_1b,Config) of
+ test_group_1b -> ok;
+ _ -> ct:fail(no_group_data)
+ end
+ end,
+ undefined = ?config(testcase_1a,Config),
+ testcase_1b = ?config(testcase_1b,Config),
+ ok.
+
+testcase_1c() ->
+ [].
+testcase_1c(Config) ->
+ init = ?config(suite,Config),
+ case ?config(test_group_1a,Config) of
+ test_group_1a -> ok;
+ _ ->
+ case ?config(test_group_1b,Config) of
+ test_group_1b -> ok;
+ _ -> ct:fail(no_group_data)
+ end
+ end,
+ undefined = ?config(testcase_1b,Config),
+ testcase_1c = ?config(testcase_1c,Config),
+ ok.
+
+testcase_1() ->
+ [].
+testcase_1(Config) ->
+ init = ?config(suite,Config),
+ testcase_1 = ?config(testcase_1,Config),
+ ok.
+
+testcase_2() ->
+ [].
+testcase_2(Config) ->
+ init = ?config(suite,Config),
+ undefined = ?config(test_group_1a,Config),
+ undefined = ?config(test_group_1b,Config),
+ testcase_2 = ?config(testcase_2,Config),
+ ok.
+
+testcase_2a() ->
+ [].
+testcase_2a(Config) ->
+ init = ?config(suite,Config),
+ test_group_2 = ?config(test_group_2,Config),
+ testcase_2a = ?config(testcase_2a,Config),
+ ok.
+testcase_2b() ->
+ [].
+testcase_2b(Config) ->
+ init = ?config(suite,Config),
+ test_group_2 = ?config(test_group_2,Config),
+ undefined = ?config(testcase_2a,Config),
+ testcase_2b = ?config(testcase_2b,Config),
+ ok.
+
+testcase_3a() ->
+ [].
+testcase_3a(Config) ->
+ init = ?config(suite,Config),
+ test_group_2 = ?config(test_group_2,Config),
+ test_group_3 = ?config(test_group_3,Config),
+ undefined = ?config(testcase_2b,Config),
+ testcase_3a = ?config(testcase_3a,Config),
+ ok.
+testcase_3b() ->
+ [].
+testcase_3b(Config) ->
+ init = ?config(suite,Config),
+ test_group_2 = ?config(test_group_2,Config),
+ test_group_3 = ?config(test_group_3,Config),
+ undefined = ?config(testcase_3a,Config),
+ testcase_3b = ?config(testcase_3b,Config),
+ ok.
+
+testcase_3() ->
+ [].
+testcase_3(Config) ->
+ init = ?config(suite,Config),
+ undefined = ?config(test_group_2,Config),
+ undefined = ?config(test_group_3,Config),
+ testcase_3 = ?config(testcase_3,Config),
+ ok.
+
+testcase_5a() ->
+ [].
+testcase_5a(Config) ->
+ init = ?config(suite,Config),
+ undefined = ?config(test_group_3,Config),
+ test_group_4 = ?config(test_group_4,Config),
+ test_group_5 = ?config(test_group_5,Config),
+ undefined = ?config(testcase_3,Config),
+ testcase_5a = ?config(testcase_5a,Config),
+ %% increase chance the done event will come
+ %% during execution of subgroup (could be
+ %% tricky to handle)
+ timer:sleep(3),
+ ok.
+testcase_5b() ->
+ [].
+testcase_5b(Config) ->
+ init = ?config(suite,Config),
+ test_group_4 = ?config(test_group_4,Config),
+ test_group_5 = ?config(test_group_5,Config),
+ undefined = ?config(testcase_5a,Config),
+ testcase_5b = ?config(testcase_5b,Config),
+ ok.
+
+testcase_7a() ->
+ [].
+testcase_7a(Config) ->
+ init = ?config(suite,Config),
+ undefined = ?config(test_group_3,Config),
+ test_group_4 = ?config(test_group_4,Config),
+ test_group_5 = ?config(test_group_5,Config),
+ test_group_6 = ?config(test_group_6,Config),
+ test_group_7 = ?config(test_group_7,Config),
+ testcase_7a = ?config(testcase_7a,Config),
+ ok.
+testcase_7b() ->
+ [].
+testcase_7b(Config) ->
+ init = ?config(suite,Config),
+ test_group_4 = ?config(test_group_4,Config),
+ test_group_5 = ?config(test_group_5,Config),
+ test_group_6 = ?config(test_group_6,Config),
+ test_group_7 = ?config(test_group_7,Config),
+ undefined = ?config(testcase_7a,Config),
+ testcase_7b = ?config(testcase_7b,Config),
+ ok.
diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/suites_1/simple_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/suites_1/simple_1_SUITE.erl
new file mode 100644
index 0000000000..b789851134
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_1_SUITE_data/suites_1/simple_1_SUITE.erl
@@ -0,0 +1,146 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(simple_1_SUITE).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% COMMON TEST CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% Function: suite() -> Info
+%%
+%% Info = [tuple()]
+%% List of key/value pairs.
+%%
+%% Description: Returns list of tuples to set default properties
+%% for the suite.
+%%
+%% Note: The suite/0 function is only meant to be used to return
+%% default data values, not perform any other operations.
+%%--------------------------------------------------------------------
+suite() ->
+ [
+ {timetrap,{seconds,10}}
+ ].
+
+%%--------------------------------------------------------------------
+%% Function: init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%%
+%% Config0 = Config1 = [tuple()]
+%% A list of key/value pairs, holding the test case configuration.
+%% Reason = term()
+%% The reason for skipping the suite.
+%%
+%% Description: Initialization before the suite.
+%%
+%% Note: This function is free to add any key/value pairs to the Config
+%% variable, but should NOT alter/remove any existing entries.
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ [{ips,ips_data} | Config].
+
+%%--------------------------------------------------------------------
+%% Function: end_per_suite(Config0) -> void() | {save_config,Config1}
+%%
+%% Config0 = Config1 = [tuple()]
+%% A list of key/value pairs, holding the test case configuration.
+%%
+%% Description: Cleanup after the suite.
+%%--------------------------------------------------------------------
+end_per_suite(Config) ->
+ ips_data = ?config(ips, Config).
+
+%%--------------------------------------------------------------------
+%% Function: init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%%
+%% TestCase = atom()
+%% Name of the test case that is about to run.
+%% Config0 = Config1 = [tuple()]
+%% A list of key/value pairs, holding the test case configuration.
+%% Reason = term()
+%% The reason for skipping the test case.
+%%
+%% Description: Initialization before each test case.
+%%
+%% Note: This function is free to add any key/value pairs to the Config
+%% variable, but should NOT alter/remove any existing entries.
+%%--------------------------------------------------------------------
+init_per_testcase(TestCase, Config) ->
+ [{TestCase,{TestCase,data}} | Config].
+
+%%--------------------------------------------------------------------
+%% Function: end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1}
+%%
+%% TestCase = atom()
+%% Name of the test case that is finished.
+%% Config0 = Config1 = [tuple()]
+%% A list of key/value pairs, holding the test case configuration.
+%%
+%% Description: Cleanup after each test case.
+%%--------------------------------------------------------------------
+end_per_testcase(TestCase, Config) ->
+ {TestCase,data} = ?config(TestCase, Config).
+
+%%--------------------------------------------------------------------
+%% Function: all() -> TestCases | {skip,Reason}
+%%
+%% TestCases = [TestCase | {sequence,SeqName}]
+%% TestCase = atom()
+%% Name of a test case.
+%% SeqName = atom()
+%% Name of a test case sequence.
+%% Reason = term()
+%% The reason for skipping all test cases.
+%%
+%% Description: Returns the list of test cases that are to be executed.
+%%--------------------------------------------------------------------
+all() ->
+ [tc1,
+ tc2].
+
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+tc1() ->
+ [{userdata,{info, "This is a testcase"}}].
+
+tc1(Config) ->
+ ips_data = ?config(ips, Config),
+ {tc1,data} = ?config(tc1, Config),
+ ok.
+
+tc2() ->
+ [{timetrap,5000}].
+
+tc2(Config) ->
+ ips_data = ?config(ips, Config),
+ undefined = ?config(tc1, Config),
+ {tc2,data} = ?config(tc2, Config),
+ ok.
diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/suites_1/simple_2_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/suites_1/simple_2_SUITE.erl
new file mode 100644
index 0000000000..eb7e9cdf7b
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_1_SUITE_data/suites_1/simple_2_SUITE.erl
@@ -0,0 +1,146 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(simple_2_SUITE).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% COMMON TEST CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% Function: suite() -> Info
+%%
+%% Info = [tuple()]
+%% List of key/value pairs.
+%%
+%% Description: Returns list of tuples to set default properties
+%% for the suite.
+%%
+%% Note: The suite/0 function is only meant to be used to return
+%% default data values, not perform any other operations.
+%%--------------------------------------------------------------------
+suite() ->
+ [
+ {timetrap,{seconds,10}}
+ ].
+
+%%--------------------------------------------------------------------
+%% Function: init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%%
+%% Config0 = Config1 = [tuple()]
+%% A list of key/value pairs, holding the test case configuration.
+%% Reason = term()
+%% The reason for skipping the suite.
+%%
+%% Description: Initialization before the suite.
+%%
+%% Note: This function is free to add any key/value pairs to the Config
+%% variable, but should NOT alter/remove any existing entries.
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ [{ips,ips_data} | Config].
+
+%%--------------------------------------------------------------------
+%% Function: end_per_suite(Config0) -> void() | {save_config,Config1}
+%%
+%% Config0 = Config1 = [tuple()]
+%% A list of key/value pairs, holding the test case configuration.
+%%
+%% Description: Cleanup after the suite.
+%%--------------------------------------------------------------------
+end_per_suite(Config) ->
+ ips_data = ?config(ips, Config).
+
+%%--------------------------------------------------------------------
+%% Function: init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%%
+%% TestCase = atom()
+%% Name of the test case that is about to run.
+%% Config0 = Config1 = [tuple()]
+%% A list of key/value pairs, holding the test case configuration.
+%% Reason = term()
+%% The reason for skipping the test case.
+%%
+%% Description: Initialization before each test case.
+%%
+%% Note: This function is free to add any key/value pairs to the Config
+%% variable, but should NOT alter/remove any existing entries.
+%%--------------------------------------------------------------------
+init_per_testcase(TestCase, Config) ->
+ [{TestCase,{TestCase,data}} | Config].
+
+%%--------------------------------------------------------------------
+%% Function: end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1}
+%%
+%% TestCase = atom()
+%% Name of the test case that is finished.
+%% Config0 = Config1 = [tuple()]
+%% A list of key/value pairs, holding the test case configuration.
+%%
+%% Description: Cleanup after each test case.
+%%--------------------------------------------------------------------
+end_per_testcase(TestCase, Config) ->
+ {TestCase,data} = ?config(TestCase, Config).
+
+%%--------------------------------------------------------------------
+%% Function: all() -> TestCases | {skip,Reason}
+%%
+%% TestCases = [TestCase | {sequence,SeqName}]
+%% TestCase = atom()
+%% Name of a test case.
+%% SeqName = atom()
+%% Name of a test case sequence.
+%% Reason = term()
+%% The reason for skipping all test cases.
+%%
+%% Description: Returns the list of test cases that are to be executed.
+%%--------------------------------------------------------------------
+all() ->
+ [tc1,
+ tc2].
+
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+tc1() ->
+ [{userdata,{info, "This is a testcase"}}].
+
+tc1(Config) ->
+ ips_data = ?config(ips, Config),
+ {tc1,data} = ?config(tc1, Config),
+ ok.
+
+tc2() ->
+ [{timetrap,5000}].
+
+tc2(Config) ->
+ ips_data = ?config(ips, Config),
+ undefined = ?config(tc1, Config),
+ {tc2,data} = ?config(tc2, Config),
+ ok.
diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl
index d1f53f3c39..a28a73a8ed 100644
--- a/lib/test_server/src/test_server_ctrl.erl
+++ b/lib/test_server/src/test_server_ctrl.erl
@@ -1564,8 +1564,8 @@ temp_nodename([Chr|Base], Acc) ->
%% of cases can not be calculated and NoOfCases = unknown.
count_test_cases(TopCases, SkipCases) when is_list(TopCases) ->
case collect_all_cases(TopCases, SkipCases) of
- {error,_Why} ->
- error;
+ {error,_Why} = Error ->
+ Error;
TestSpec ->
{get_suites(TestSpec, []),
case remove_conf(TestSpec) of
@@ -2067,10 +2067,6 @@ run_test_cases(TestSpec, Config, TimetrapData) ->
html_convert_modules(TestSpec, Config),
- %%! For readable tracing...
- %%! Config1 = [{data_dir,""},{priv_dir,""},{nodes,[]}],
- %%! run_test_cases_loop(TestSpec, [[]], TimetrapData, [], []),
-
run_test_cases_loop(TestSpec, [Config], TimetrapData, [], []),
maybe_get_privdir(),
@@ -3552,7 +3548,7 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, Where,
case test_server_sup:framework_call(warn, [nodes], true) of
true ->
case catch controller_call(kill_slavenodes) of
- {'EXIT',_}=Exit ->
+ {'EXIT',_} = Exit ->
print(minor,
"WARNING: There might be slavenodes left in the"
" system. I tried to kill them, but I failed: ~p\n",
@@ -4444,9 +4440,9 @@ collect_cases([Case|Cs0], St0) ->
case collect_cases(Cs0, St1) of
{ok,FlatCases2,St} ->
{ok,FlatCases1 ++ FlatCases2,St};
- {error,_Reason}=Error -> Error
+ {error,_Reason} = Error -> Error
end;
- {error,_Reason}=Error -> Error
+ {error,_Reason} = Error -> Error
end;
@@ -4482,39 +4478,55 @@ collect_cases({conf,Props,InitMF,CaseList,FinF}, St) when is_atom(FinF) ->
Props1 ->
collect_cases({conf,Props1,InitMF,CaseList,{St#cc.mod,FinF}}, St)
end;
-collect_cases({conf,Props,InitMF,CaseList,FinMF}, St0) ->
- case collect_cases(CaseList, St0) of
- {ok,[],_St}=Empty ->
- Empty;
- {ok,FlatCases,St} ->
+collect_cases({conf,Props,InitMF,CaseList,FinMF} = Conf, St) ->
+ case init_props(Props) of
+ {error,_} ->
+ {ok,[],St};
+ Props1 ->
Ref = make_ref(),
- case in_skip_list(InitMF, St#cc.skip) of
- {true,Comment} ->
+ Skips = St#cc.skip,
+ case in_skip_list({St#cc.mod,Conf}, Skips) of
+ {true,Comment} -> % conf init skipped
{ok,[{skip_case,{conf,Ref,InitMF,Comment}} |
- FlatCases ++ [{conf,Ref,[],FinMF}]],St};
+ [] ++ [{conf,Ref,[],FinMF}]],St};
+ {true,Name,Comment} when is_atom(Name) -> % all cases skipped
+ {ok,[{skip_case,{{St#cc.mod,{group,Name}},Comment}}],St};
+ {true,ToSkip,_} when is_list(ToSkip) -> % some cases skipped
+ case collect_cases(CaseList,
+ St#cc{skip=ToSkip++Skips}) of
+ {ok,[],_St} = Empty ->
+ Empty;
+ {ok,FlatCases,St1} ->
+ {ok,[{conf,Ref,Props1,InitMF} |
+ FlatCases ++ [{conf,Ref,
+ keep_name(Props1),
+ FinMF}]],St1#cc{skip=Skips}};
+ {error,_Reason} = Error ->
+ Error
+ end;
false ->
- case init_props(Props) of
- {error,_} ->
- {ok,[],St};
- Props1 ->
+ case collect_cases(CaseList, St) of
+ {ok,[],_St} = Empty ->
+ Empty;
+ {ok,FlatCases,St1} ->
{ok,[{conf,Ref,Props1,InitMF} |
FlatCases ++ [{conf,Ref,
keep_name(Props1),
- FinMF}]],St}
+ FinMF}]],St1};
+ {error,_Reason} = Error ->
+ Error
end
- end;
- {error,_Reason}=Error ->
- Error
+ end
end;
collect_cases({make,InitMFA,CaseList,FinMFA}, St0) ->
case collect_cases(CaseList, St0) of
- {ok,[],_St}=Empty -> Empty;
+ {ok,[],_St} = Empty -> Empty;
{ok,FlatCases,St} ->
Ref = make_ref(),
{ok,[{make,Ref,InitMFA}|FlatCases ++
[{make,Ref,FinMFA}]],St};
- {error,_Reason}=Error -> Error
+ {error,_Reason} = Error -> Error
end;
collect_cases({Module, Cases}, St) when is_list(Cases) ->
@@ -4535,6 +4547,9 @@ collect_cases(Case, St) when is_atom(Case), is_atom(St#cc.mod) ->
collect_cases(Other, St) ->
{error,{bad_subtest_spec,St#cc.mod,Other}}.
+collect_case({_Mod,{conf,_,_,_,_}=Conf}, St) ->
+ collect_cases(Conf, St);
+
collect_case(MFA, St) ->
case in_skip_list(MFA, St#cc.skip) of
{true,Comment} ->
@@ -4580,6 +4595,8 @@ collect_subcases(Mod, Case, MFA, St, Suite) ->
%%%! --- END Kept for backwards compatibilty ---
{Skip,Reason} when Skip==skip; Skip==skipped ->
{ok,[{skip_case,{MFA,Reason}}],St};
+ {error,Reason} ->
+ throw(Reason);
SubCases ->
collect_case_subcases(Mod, Case, SubCases, St)
end.
@@ -4641,6 +4658,45 @@ check_deny_req(Req, DenyList) ->
false -> granted
end.
+in_skip_list({Mod,{conf,Props,InitMF,_CaseList,_FinMF}}, SkipList) ->
+ case in_skip_list(InitMF, SkipList) of
+ {true,_} = Yes ->
+ Yes;
+ _ ->
+ case proplists:get_value(name, Props) of
+ undefined ->
+ false;
+ Name ->
+ ToSkip =
+ lists:flatmap(
+ fun({M,{conf,SProps,_,SCaseList,_},Cmt}) when
+ M == Mod ->
+ case proplists:get_value(name, SProps) of
+ Name ->
+ case SCaseList of
+ all ->
+ [{M,all,Cmt}];
+ _ ->
+ [{M,F,Cmt} || F <- SCaseList]
+ end;
+ _ ->
+ []
+ end;
+ (_) ->
+ []
+ end, SkipList),
+ case ToSkip of
+ [] ->
+ false;
+ _ ->
+ case lists:keysearch(all, 2, ToSkip) of
+ {value,{_,_,Cmt}} -> {true,Name,Cmt};
+ _ -> {true,ToSkip,""}
+ end
+ end
+ end
+ end;
+
in_skip_list({Mod,Func,_Args}, SkipList) ->
in_skip_list({Mod,Func}, SkipList);
in_skip_list({Mod,Func}, [{Mod,Funcs,Comment}|SkipList]) when is_list(Funcs) ->