aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPeter Andersson <[email protected]>2012-11-20 12:25:23 +0100
committerPeter Andersson <[email protected]>2012-11-20 12:26:18 +0100
commitb0acd41f90490d38156cf9c8615859def630596b (patch)
tree29d9c0235efc35f50bbd09794aecf1dbab7e0588
parent8b2dcbaf0acd2350f8b3259701384e6d1a858824 (diff)
parent44fa71f32c0e901c73808657af7b8ab38158817b (diff)
downloadotp-b0acd41f90490d38156cf9c8615859def630596b.tar.gz
otp-b0acd41f90490d38156cf9c8615859def630596b.tar.bz2
otp-b0acd41f90490d38156cf9c8615859def630596b.zip
Merge remote branch 'origin/peppe/common_test/group_search_r15b03' into maint
* origin/peppe/common_test/group_search_r15b03: Fix problem with test case order in group specifications Finish the test suite and correct remaining bugs Implement new group search functionality OTP-10466
-rw-r--r--lib/common_test/src/Makefile3
-rw-r--r--lib/common_test/src/ct_framework.erl368
-rw-r--r--lib/common_test/src/ct_groups.erl599
-rw-r--r--lib/common_test/src/ct_run.erl146
-rw-r--r--lib/common_test/src/ct_testspec.erl24
-rw-r--r--lib/common_test/test/Makefile3
-rw-r--r--lib/common_test/test/ct_groups_search_SUITE.erl1245
-rw-r--r--lib/common_test/test/ct_groups_search_SUITE_data/groups_search_dummy_1_SUITE.erl83
-rw-r--r--lib/common_test/test/ct_groups_search_SUITE_data/groups_search_dummy_2_SUITE.erl102
9 files changed, 2160 insertions, 413 deletions
diff --git a/lib/common_test/src/Makefile b/lib/common_test/src/Makefile
index f7dce195d7..dd2923ece9 100644
--- a/lib/common_test/src/Makefile
+++ b/lib/common_test/src/Makefile
@@ -73,7 +73,8 @@ MODULES= \
cth_surefire \
ct_netconfc \
ct_conn_log_h \
- cth_conn_log
+ cth_conn_log \
+ ct_groups
TARGET_MODULES= $(MODULES:%=$(EBIN)/%)
BEAM_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR))
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index 4d47731239..403eab66cb 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -32,8 +32,6 @@
-export([error_in_suite/1, init_per_suite/1, end_per_suite/1,
init_per_group/2, end_per_group/2]).
--export([make_all_conf/3, make_conf/5]).
-
-include("ct_event.hrl").
-include("ct_util.hrl").
@@ -876,13 +874,13 @@ get_suite(Mod, all) ->
{'EXIT',_} ->
get_all(Mod, []);
GroupDefs when is_list(GroupDefs) ->
- case catch find_groups(Mod, all, all, GroupDefs) of
+ case catch ct_groups:find_groups(Mod, all, all, GroupDefs) of
{error,_} = Error ->
%% this makes test_server call error_in_suite as first
%% (and only) test case so we can report Error properly
[{?MODULE,error_in_suite,[[Error]]}];
ConfTests ->
- get_all(Mod, ConfTests)
+ get_all(Mod, ConfTests)
end;
_ ->
E = "Bad return value from "++atom_to_list(Mod)++":groups/0",
@@ -901,7 +899,7 @@ get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) ->
{'EXIT',_} ->
[Group];
GroupDefs when is_list(GroupDefs) ->
- case catch find_groups(Mod, Name, TCs, GroupDefs) of
+ case catch ct_groups:find_groups(Mod, Name, TCs, GroupDefs) of
{error,_} = Error ->
%% this makes test_server call error_in_suite as first
%% (and only) test case so we can report Error properly
@@ -916,12 +914,13 @@ get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) ->
%% init/end functions for top groups will be executed
case catch ?val(name, element(2, hd(ConfTests))) of
Name -> % top group
- delete_subs(ConfTests, ConfTests);
+ ct_groups:delete_subs(ConfTests, ConfTests);
_ ->
[]
end;
false ->
- ConfTests1 = delete_subs(ConfTests, ConfTests),
+ ConfTests1 = ct_groups:delete_subs(ConfTests,
+ ConfTests),
case ?val(override, Props) of
undefined ->
ConfTests1;
@@ -930,9 +929,9 @@ get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) ->
ORSpec ->
ORSpec1 = if is_tuple(ORSpec) -> [ORSpec];
true -> ORSpec end,
- search_and_override(ConfTests1,
- ORSpec1, Mod)
- end
+ ct_groups:search_and_override(ConfTests1,
+ ORSpec1, Mod)
+ end
end
end;
_ ->
@@ -976,234 +975,6 @@ get_all_cases1(_, []) ->
%%%-----------------------------------------------------------------
-find_groups(Mod, Name, TCs, GroupDefs) ->
- Found = find(Mod, Name, TCs, GroupDefs, [], GroupDefs, false),
- trim(Found).
-
-find(Mod, all, _TCs, [{Name,Props,Tests} | Gs], Known, Defs, _)
- when is_atom(Name), is_list(Props), is_list(Tests) ->
- cyclic_test(Mod, Name, Known),
- [make_conf(Mod, Name, Props,
- find(Mod, all, all, Tests, [Name | Known], Defs, true)) |
- find(Mod, all, all, Gs, [], Defs, true)];
-
-find(Mod, Name, TCs, [{Name,Props,Tests} | _Gs], Known, Defs, false)
- when is_atom(Name), is_list(Props), is_list(Tests) ->
- cyclic_test(Mod, Name, Known),
- case TCs of
- all ->
- [make_conf(Mod, Name, Props,
- find(Mod, Name, TCs, Tests, [Name | Known], Defs, true))];
- _ ->
- Tests1 = [TC || TC <- TCs,
- lists:member(TC, Tests) == true],
- [make_conf(Mod, Name, Props, Tests1)]
- end;
-
-find(Mod, Name, TCs, [{Name1,Props,Tests} | Gs], Known, Defs, false)
- when is_atom(Name1), is_list(Props), is_list(Tests) ->
- cyclic_test(Mod, Name1, Known),
- [make_conf(Mod,Name1,Props,
- find(Mod, Name, TCs, Tests, [Name1 | Known], Defs, false)) |
- find(Mod, Name, TCs, Gs, [], Defs, false)];
-
-find(Mod, Name, _TCs, [{Name,_Props,_Tests} | _Gs], _Known, _Defs, true)
- when is_atom(Name) ->
- E = "Duplicate groups named "++atom_to_list(Name)++" in "++
- atom_to_list(Mod)++":groups/0",
- throw({error,list_to_atom(E)});
-
-find(Mod, Name, all, [{Name1,Props,Tests} | Gs], Known, Defs, true)
- when is_atom(Name1), is_list(Props), is_list(Tests) ->
- cyclic_test(Mod, Name1, Known),
- [make_conf(Mod, Name1, Props,
- find(Mod, Name, all, Tests, [Name1 | Known], Defs, true)) |
- find(Mod, Name, all, Gs, [], Defs, true)];
-
-find(Mod, Name, TCs, [{group,Name1} | Gs], Known, Defs, Found)
- when is_atom(Name1) ->
- find(Mod, Name, TCs, [expand(Mod, Name1, Defs) | Gs], Known, Defs, Found);
-
-%% Undocumented remote group feature, use with caution
-find(Mod, Name, TCs, [{group, ExtMod, ExtGrp} | Gs], Known, Defs, true)
- when is_atom(ExtMod), is_atom(ExtGrp) ->
- ExternalDefs = ExtMod:groups(),
- ExternalTCs = find(ExtMod, ExtGrp, TCs, [{group, ExtGrp}],
- [], ExternalDefs, false),
- ExternalTCs ++ find(Mod, Name, TCs, Gs, Known, Defs, true);
-
-find(Mod, Name, TCs, [{Name1,Tests} | Gs], Known, Defs, Found)
- when is_atom(Name1), is_list(Tests) ->
- find(Mod, Name, TCs, [{Name1,[],Tests} | Gs], Known, Defs, Found);
-
-find(Mod, Name, TCs, [_TC | Gs], Known, Defs, false) ->
- find(Mod, Name, TCs, Gs, Known, Defs, false);
-
-find(Mod, Name, TCs, [TC | Gs], Known, Defs, true) when is_atom(TC) ->
- [{Mod, TC} | find(Mod, Name, TCs, Gs, Known, Defs, true)];
-
-find(Mod, Name, TCs, [{ExternalTC, Case} = TC | Gs], Known, Defs, true)
- when is_atom(ExternalTC),
- is_atom(Case) ->
- [TC | find(Mod, Name, TCs, Gs, Known, Defs, true)];
-
-find(Mod, _Name, _TCs, [BadTerm | _Gs], Known, _Defs, _Found) ->
- Where = if length(Known) == 0 ->
- atom_to_list(Mod)++":groups/0";
- true ->
- "group "++atom_to_list(lists:last(Known))++
- " in "++atom_to_list(Mod)++":groups/0"
- end,
- Term = io_lib:format("~p", [BadTerm]),
- E = "Bad term "++lists:flatten(Term)++" in "++Where,
- throw({error,list_to_atom(E)});
-
-find(_Mod, _Name, _TCs, [], _Known, _Defs, false) ->
- ['$NOMATCH'];
-
-find(_Mod, _Name, _TCs, [], _Known, _Defs, _Found) ->
- [].
-
-delete_subs([{conf, _,_,_,_} = Conf | Confs], All) ->
- All1 = delete_conf(Conf, All),
- case is_sub(Conf, All1) of
- true ->
- delete_subs(Confs, All1);
- false ->
- delete_subs(Confs, All)
- end;
-delete_subs([_Else | Confs], All) ->
- delete_subs(Confs, All);
-delete_subs([], All) ->
- All.
-
-delete_conf({conf,Props,_,_,_}, Confs) ->
- Name = ?val(name, Props),
- [Conf || Conf = {conf,Props0,_,_,_} <- Confs,
- Name =/= ?val(name, Props0)].
-
-is_sub({conf,Props,_,_,_}=Conf, [{conf,_,_,Tests,_} | Confs]) ->
- Name = ?val(name, Props),
- case lists:any(fun({conf,Props0,_,_,_}) ->
- case ?val(name, Props0) of
- N when N == Name ->
- true;
- _ ->
- false
- end;
- (_) ->
- false
- end, Tests) of
- true ->
- true;
- false ->
- is_sub(Conf, Tests) or is_sub(Conf, Confs)
- end;
-
-is_sub(Conf, [_TC | Tests]) ->
- is_sub(Conf, Tests);
-
-is_sub(_Conf, []) ->
- false.
-
-trim(['$NOMATCH' | Tests]) ->
- trim(Tests);
-
-trim([{conf,Props,Init,Tests,End} | Confs]) ->
- case trim(Tests) of
- [] ->
- trim(Confs);
- Trimmed ->
- [{conf,Props,Init,Trimmed,End} | trim(Confs)]
- end;
-
-trim([TC | Tests]) ->
- [TC | trim(Tests)];
-
-trim([]) ->
- [].
-
-cyclic_test(Mod, Name, Names) ->
- case lists:member(Name, Names) of
- true ->
- E = "Cyclic reference to group "++atom_to_list(Name)++
- " in "++atom_to_list(Mod)++":groups/0",
- throw({error,list_to_atom(E)});
- false ->
- ok
- end.
-
-expand(Mod, Name, Defs) ->
- case lists:keysearch(Name, 1, Defs) of
- {value,Def} ->
- Def;
- false ->
- E = "Invalid group "++atom_to_list(Name)++
- " in "++atom_to_list(Mod)++":groups/0",
- throw({error,list_to_atom(E)})
- end.
-
-make_all_conf(Dir, Mod, _Props) ->
- case code:is_loaded(Mod) of
- false ->
- code:load_abs(filename:join(Dir,atom_to_list(Mod)));
- _ ->
- ok
- end,
- make_all_conf(Mod).
-
-make_all_conf(Mod) ->
- case catch apply(Mod, groups, []) of
- {'EXIT',_} ->
- {error,{invalid_group_definition,Mod}};
- GroupDefs when is_list(GroupDefs) ->
- case catch find_groups(Mod, all, all, GroupDefs) of
- {error,_} = Error ->
- %% this makes test_server call error_in_suite as first
- %% (and only) test case so we can report Error properly
- [{?MODULE,error_in_suite,[[Error]]}];
- [] ->
- {error,{invalid_group_spec,Mod}};
- ConfTests ->
- [{conf,Props,Init,all,End} ||
- {conf,Props,Init,_,End}
- <- delete_subs(ConfTests, ConfTests)]
- end
- end.
-
-make_conf(Dir, Mod, Name, Props, TestSpec) ->
- case code:is_loaded(Mod) of
- false ->
- code:load_abs(filename:join(Dir,atom_to_list(Mod)));
- _ ->
- ok
- end,
- make_conf(Mod, Name, Props, TestSpec).
-
-make_conf(Mod, Name, Props, TestSpec) ->
- case code:is_loaded(Mod) of
- false ->
- code:load_file(Mod);
- _ ->
- ok
- end,
- {InitConf,EndConf,ExtraProps} =
- case erlang:function_exported(Mod,init_per_group,2) of
- true ->
- {{Mod,init_per_group},{Mod,end_per_group},[]};
- false ->
- ct_logs:log("TEST INFO", "init_per_group/2 and "
- "end_per_group/2 missing for group "
- "~p in ~p, using default.",
- [Name,Mod]),
- {{?MODULE,init_per_group},
- {?MODULE,end_per_group},
- [{suite,Mod}]}
- end,
- {conf,[{name,Name}|Props++ExtraProps],InitConf,TestSpec,EndConf}.
-
-%%%-----------------------------------------------------------------
-
get_all(Mod, ConfTests) ->
case catch apply(Mod, all, []) of
{'EXIT',_} ->
@@ -1218,133 +989,24 @@ get_all(Mod, ConfTests) ->
[{?MODULE,error_in_suite,[[{error,What}]]}];
SeqsAndTCs ->
%% expand group references in all() using ConfTests
- case catch expand_groups(SeqsAndTCs, ConfTests, Mod) of
+ case catch ct_groups:expand_groups(SeqsAndTCs,
+ ConfTests,
+ Mod) of
{error,_} = Error ->
[{?MODULE,error_in_suite,[[Error]]}];
Tests ->
- delete_subs(Tests, Tests)
+ ct_groups:delete_subs(Tests, Tests)
end
end;
Skip = {skip,_Reason} ->
Skip;
_ ->
Reason =
- list_to_atom("Bad return value from "++atom_to_list(Mod)++":all/0"),
+ list_to_atom("Bad return value from "++
+ atom_to_list(Mod)++":all/0"),
[{?MODULE,error_in_suite,[[{error,Reason}]]}]
end.
-expand_groups([H | T], ConfTests, Mod) ->
- [expand_groups(H, ConfTests, Mod) | expand_groups(T, ConfTests, Mod)];
-expand_groups([], _ConfTests, _Mod) ->
- [];
-expand_groups({group,Name}, ConfTests, Mod) ->
- expand_groups({group,Name,default,[]}, ConfTests, Mod);
-expand_groups({group,Name,default}, ConfTests, Mod) ->
- expand_groups({group,Name,default,[]}, ConfTests, Mod);
-expand_groups({group,Name,ORProps}, ConfTests, Mod) when is_list(ORProps) ->
- expand_groups({group,Name,ORProps,[]}, ConfTests, Mod);
-expand_groups({group,Name,ORProps,SubORSpec}, ConfTests, Mod) ->
- FindConf =
- fun(Conf = {conf,Props,Init,Ts,End}) ->
- case ?val(name, Props) of
- Name when ORProps == default ->
- [Conf];
- Name ->
- [{conf,[{name,Name}|ORProps],Init,Ts,End}];
- _ ->
- []
- end
- end,
- case lists:flatmap(FindConf, ConfTests) of
- [] ->
- throw({error,invalid_ref_msg(Name, Mod)});
- Matching when SubORSpec == [] ->
- Matching;
- Matching ->
- override_props(Matching, SubORSpec, Name,Mod)
- end;
-expand_groups(SeqOrTC, _ConfTests, _Mod) ->
- SeqOrTC.
-
-%% search deep for the matching conf test and modify it and any
-%% sub tests according to the override specification
-search_and_override([Conf = {conf,Props,Init,Tests,End}], ORSpec, Mod) ->
- Name = ?val(name, Props),
- case lists:keysearch(Name, 1, ORSpec) of
- {value,{Name,default}} ->
- [Conf];
- {value,{Name,ORProps}} ->
- [{conf,[{name,Name}|ORProps],Init,Tests,End}];
- {value,{Name,default,[]}} ->
- [Conf];
- {value,{Name,default,SubORSpec}} ->
- override_props([Conf], SubORSpec, Name,Mod);
- {value,{Name,ORProps,SubORSpec}} ->
- override_props([{conf,[{name,Name}|ORProps],
- Init,Tests,End}], SubORSpec, Name,Mod);
- _ ->
- [{conf,Props,Init,search_and_override(Tests,ORSpec,Mod),End}]
- end.
-
-%% Modify the Tests element according to the override specification
-override_props([{conf,Props,Init,Tests,End} | Confs], SubORSpec, Name,Mod) ->
- {Subs,SubORSpec1} = override_sub_props(Tests, [], SubORSpec, Mod),
- [{conf,Props,Init,Subs,End} | override_props(Confs, SubORSpec1, Name,Mod)];
-override_props([], [], _,_) ->
- [];
-override_props([], SubORSpec, Name,Mod) ->
- Es = [invalid_ref_msg(Name, element(1,Spec), Mod) || Spec <- SubORSpec],
- throw({error,Es}).
-
-override_sub_props([], New, ORSpec, _) ->
- {?rev(New),ORSpec};
-override_sub_props([T = {conf,Props,Init,Tests,End} | Ts],
- New, ORSpec, Mod) ->
- Name = ?val(name, Props),
- case lists:keysearch(Name, 1, ORSpec) of
- {value,Spec} -> % group found in spec
- Props1 =
- case element(2, Spec) of
- default -> Props;
- ORProps -> [{name,Name} | ORProps]
- end,
- case catch element(3, Spec) of
- Undef when Undef == [] ; 'EXIT' == element(1, Undef) ->
- override_sub_props(Ts, [{conf,Props1,Init,Tests,End} | New],
- lists:keydelete(Name, 1, ORSpec), Mod);
- SubORSpec when is_list(SubORSpec) ->
- case override_sub_props(Tests, [], SubORSpec, Mod) of
- {Subs,[]} ->
- override_sub_props(Ts, [{conf,Props1,Init,
- Subs,End} | New],
- lists:keydelete(Name, 1, ORSpec),
- Mod);
- {_,NonEmptySpec} ->
- Es = [invalid_ref_msg(Name, element(1, GrRef),
- Mod) || GrRef <- NonEmptySpec],
- throw({error,Es})
- end;
- BadGrSpec ->
- throw({error,{invalid_form,BadGrSpec}})
- end;
- _ -> % not a group in spec
- override_sub_props(Ts, [T | New], ORSpec, Mod)
- end;
-override_sub_props([TC | Ts], New, ORSpec, Mod) ->
- override_sub_props(Ts, [TC | New], ORSpec, Mod).
-
-invalid_ref_msg(Name, Mod) ->
- E = "Invalid reference to group "++
- atom_to_list(Name)++" in "++
- atom_to_list(Mod)++":all/0",
- list_to_atom(E).
-
-invalid_ref_msg(Name0, Name1, Mod) ->
- E = "Invalid reference to group "++
- atom_to_list(Name1)++" from "++atom_to_list(Name0)++
- " in "++atom_to_list(Mod)++":all/0",
- list_to_atom(E).
-
%%!============================================================
%%! The support for sequences by means of using sequences/0
%%! will be removed in OTP R15. The code below is only kept
diff --git a/lib/common_test/src/ct_groups.erl b/lib/common_test/src/ct_groups.erl
new file mode 100644
index 0000000000..24ca3826a8
--- /dev/null
+++ b/lib/common_test/src/ct_groups.erl
@@ -0,0 +1,599 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2004-2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%% @doc Common Test Framework callback module.
+%%%
+%%% <p>This module contains CT internal help functions for searching
+%%% through groups specification trees and producing resulting
+%%% tests.</p>
+
+-module(ct_groups).
+
+-export([find_groups/4]).
+-export([make_all_conf/3, make_all_conf/4, make_conf/5]).
+-export([delete_subs/2]).
+-export([expand_groups/3, search_and_override/3]).
+
+-define(val(Key, List), proplists:get_value(Key, List)).
+-define(val(Key, List, Def), proplists:get_value(Key, List, Def)).
+-define(rev(L), lists:reverse(L)).
+
+find_groups(Mod, GrNames, TCs, GroupDefs) when is_atom(GrNames) ;
+ (length(GrNames) == 1) ->
+ find_groups1(Mod, GrNames, TCs, GroupDefs);
+
+find_groups(Mod, Groups, TCs, GroupDefs) when Groups /= [] ->
+ lists:append([find_groups1(Mod, [GrNames], TCs, GroupDefs) ||
+ GrNames <- Groups]);
+
+find_groups(_Mod, [], _TCs, _GroupDefs) ->
+ [].
+
+%% GrNames == atom(): Single group name, perform full search
+%% GrNames == list(): List of groups, find all matching paths
+%% GrNames == [list()]: Search path terminated by last group in GrNames
+find_groups1(Mod, GrNames, TCs, GroupDefs) ->
+ {GrNames1,FindAll} =
+ case GrNames of
+ Name when is_atom(Name), Name /= all ->
+ {[Name],true};
+ [Path] when is_list(Path) ->
+ {Path,false};
+ Path ->
+ {Path,true}
+ end,
+ TCs1 = if (is_atom(TCs) and (TCs /= all)) or is_tuple(TCs) ->
+ [TCs];
+ true ->
+ TCs
+ end,
+ Found = find(Mod, GrNames1, TCs1, GroupDefs, [],
+ GroupDefs, FindAll),
+ [Conf || Conf <- Found, Conf /= 'NOMATCH'].
+
+%% Locate all groups
+find(Mod, all, all, [{Name,Props,Tests} | Gs], Known, Defs, _)
+ when is_atom(Name), is_list(Props), is_list(Tests) ->
+ cyclic_test(Mod, Name, Known),
+ trim(make_conf(Mod, Name, Props,
+ find(Mod, all, all, Tests, [Name | Known],
+ Defs, true))) ++
+ find(Mod, all, all, Gs, Known, Defs, true);
+
+%% Locate particular TCs in all groups
+find(Mod, all, TCs, [{Name,Props,Tests} | Gs], Known, Defs, _)
+ when is_atom(Name), is_list(Props), is_list(Tests) ->
+ cyclic_test(Mod, Name, Known),
+ Tests1 = rm_unwanted_tcs(Tests, TCs, []),
+ trim(make_conf(Mod, Name, Props,
+ find(Mod, all, TCs, Tests1, [Name | Known],
+ Defs, true))) ++
+ find(Mod, all, TCs, Gs, Known, Defs, true);
+
+%% Found next group is in search path
+find(Mod, [Name|GrNames]=SPath, TCs, [{Name,Props,Tests} | Gs], Known,
+ Defs, FindAll) when is_atom(Name), is_list(Props), is_list(Tests) ->
+ cyclic_test(Mod, Name, Known),
+ Tests1 = rm_unwanted_tcs(Tests, TCs, GrNames),
+ trim(make_conf(Mod, Name, Props,
+ find(Mod, GrNames, TCs, Tests1, [Name|Known],
+ Defs, FindAll))) ++
+ find(Mod, SPath, TCs, Gs, Known, Defs, FindAll);
+
+%% Group path terminated, stop the search
+find(Mod, [], TCs, Tests, _Known, _Defs, false) ->
+ Cases = lists:flatmap(fun(TC) when is_atom(TC), TCs == all ->
+ [{Mod,TC}];
+ ({group,_}) ->
+ [];
+ ({_,_}=TC) when TCs == all ->
+ [TC];
+ (TC) ->
+ if is_atom(TC) ->
+ Tuple = {Mod,TC},
+ case lists:member(Tuple, TCs) of
+ true ->
+ [Tuple];
+ false ->
+ case lists:member(TC, TCs) of
+ true -> [{Mod,TC}];
+ false -> []
+ end
+ end;
+ true ->
+ []
+ end
+ end, Tests),
+ if Cases == [] -> ['NOMATCH'];
+ true -> Cases
+ end;
+
+%% No more groups
+find(_Mod, [_|_], _TCs, [], _Known, _Defs, _) ->
+ ['NOMATCH'];
+
+%% Found group not next in search path
+find(Mod, GrNames, TCs, [{Name,Props,Tests} | Gs], Known,
+ Defs, FindAll) when is_atom(Name), is_list(Props), is_list(Tests) ->
+ cyclic_test(Mod, Name, Known),
+ Tests1 = rm_unwanted_tcs(Tests, TCs, GrNames),
+ trim(make_conf(Mod, Name, Props,
+ find(Mod, GrNames, TCs, Tests1, [Name|Known],
+ Defs, FindAll))) ++
+ find(Mod, GrNames, TCs, Gs, Known, Defs, FindAll);
+
+%% A nested group defined on top level found
+find(Mod, GrNames, TCs, [{group,Name1} | Gs], Known, Defs, FindAll)
+ when is_atom(Name1) ->
+ find(Mod, GrNames, TCs, [expand(Mod, Name1, Defs) | Gs], Known,
+ Defs, FindAll);
+
+%% Undocumented remote group feature, use with caution
+find(Mod, GrNames, TCs, [{group, ExtMod, ExtGrp} | Gs], Known,
+ Defs, FindAll) when is_atom(ExtMod), is_atom(ExtGrp) ->
+ ExternalDefs = ExtMod:groups(),
+ ExternalTCs = find(ExtMod, ExtGrp, TCs, [{group, ExtGrp}],
+ [], ExternalDefs, FindAll),
+ ExternalTCs ++ find(Mod, GrNames, TCs, Gs, Known, Defs, FindAll);
+
+%% Group definition without properties, add an empty property list
+find(Mod, GrNames, TCs, [{Name1,Tests} | Gs], Known, Defs, FindAll)
+ when is_atom(Name1), is_list(Tests) ->
+ find(Mod, GrNames, TCs, [{Name1,[],Tests} | Gs], Known, Defs, FindAll);
+
+%% Save, and keep searching
+find(Mod, GrNames, TCs, [{ExternalTC, Case} = TC | Gs], Known,
+ Defs, FindAll) when is_atom(ExternalTC),
+ is_atom(Case) ->
+ [TC | find(Mod, GrNames, TCs, Gs, Known, Defs, FindAll)];
+
+%% Save test case
+find(Mod, GrNames, all, [TC | Gs], Known,
+ Defs, FindAll) when is_atom(TC) ->
+ [{Mod,TC} | find(Mod, GrNames, all, Gs, Known, Defs, FindAll)];
+
+%% Save test case
+find(Mod, GrNames, all, [{M,TC} | Gs], Known,
+ Defs, FindAll) when is_atom(M), M /= group, is_atom(TC) ->
+ [{M,TC} | find(Mod, GrNames, all, Gs, Known, Defs, FindAll)];
+
+%% Check if test case should be saved
+find(Mod, GrNames, TCs, [TC | Gs], Known,
+ Defs, FindAll) when is_atom(TC) orelse
+ ((size(TC) == 2) and (hd(TC) /= group)) ->
+ Case =
+ if is_atom(TC) ->
+ Tuple = {Mod,TC},
+ case lists:member(Tuple, TCs) of
+ true ->
+ Tuple;
+ false ->
+ case lists:member(TC, TCs) of
+ true -> {Mod,TC};
+ false -> []
+ end
+ end;
+ true ->
+ case lists:member(TC, TCs) of
+ true -> {Mod,TC};
+ false -> []
+ end
+ end,
+ if Case == [] ->
+ find(Mod, GrNames, TCs, Gs, Known, Defs, FindAll);
+ true ->
+ [Case | find(Mod, GrNames, TCs, Gs, Known, Defs, FindAll)]
+ end;
+
+%% Unexpeted term in group list
+find(Mod, _GrNames, _TCs, [BadTerm | _Gs], Known, _Defs, _FindAll) ->
+ Where = if length(Known) == 0 ->
+ atom_to_list(Mod)++":groups/0";
+ true ->
+ "group "++atom_to_list(lists:last(Known))++
+ " in "++atom_to_list(Mod)++":groups/0"
+ end,
+ Term = io_lib:format("~p", [BadTerm]),
+ E = "Bad term "++lists:flatten(Term)++" in "++Where,
+ throw({error,list_to_atom(E)});
+
+%% No more groups
+find(_Mod, _GrNames, _TCs, [], _Known, _Defs, _) ->
+ [].
+
+%%%-----------------------------------------------------------------
+
+%% We have to always search bottom up to only remove a branch
+%% if there's 'NOMATCH' in the leaf (otherwise, the branch should
+%% be kept)
+
+trim({conf,Props,Init,Tests,End}) ->
+ try trim(Tests) of
+ [] -> [];
+ Tests1 -> [{conf,Props,Init,Tests1,End}]
+ catch
+ throw:_ -> []
+ end;
+
+trim(Tests) when is_list(Tests) ->
+ %% we need to compare the result of trimming each test on this
+ %% level, and only let a 'NOMATCH' fail the level if no
+ %% successful sub group can be found
+ Tests1 =
+ lists:flatmap(fun(Test) ->
+ IsConf = case Test of
+ {conf,_,_,_,_} ->
+ true;
+ _ ->
+ false
+ end,
+ try trim_test(Test) of
+ [] -> [];
+ Test1 when IsConf -> [{conf,Test1}];
+ Test1 -> [Test1]
+ catch
+ throw:_ -> ['NOMATCH']
+ end
+ end, Tests),
+ case lists:keymember(conf, 1, Tests1) of
+ true -> % at least one successful group
+ lists:flatmap(fun({conf,Test}) -> [Test];
+ ('NOMATCH') -> []; % ignore any 'NOMATCH'
+ (Test) -> [Test]
+ end, Tests1);
+ false ->
+ case lists:member('NOMATCH', Tests1) of
+ true ->
+ throw('NOMATCH');
+ false ->
+ Tests1
+ end
+ end.
+
+trim_test({conf,Props,Init,Tests,End}) ->
+ case trim(Tests) of
+ [] ->
+ [];
+ Tests1 ->
+ {conf,Props,Init,Tests1,End}
+ end;
+
+trim_test('NOMATCH') ->
+ throw('NOMATCH');
+
+trim_test(Test) ->
+ Test.
+
+%% GrNames is [] if the terminating group has been found. From
+%% that point, all specified test should be included (as well as
+%% sub groups for deeper search).
+rm_unwanted_tcs(Tests, all, []) ->
+ Tests;
+
+rm_unwanted_tcs(Tests, TCs, []) ->
+ sort_tests(lists:flatmap(fun(Test) when is_tuple(Test),
+ (size(Test) > 2) ->
+ [Test];
+ (Test={group,_}) ->
+ [Test];
+ (Test={_M,TC}) ->
+ case lists:member(TC, TCs) of
+ true -> [Test];
+ false -> []
+ end;
+ (Test) when is_atom(Test) ->
+ case lists:keysearch(Test, 2, TCs) of
+ {value,_} ->
+ [Test];
+ _ ->
+ case lists:member(Test, TCs) of
+ true -> [Test];
+ false -> []
+ end
+ end;
+ (Test) -> [Test]
+ end, Tests), TCs);
+
+rm_unwanted_tcs(Tests, _TCs, _) ->
+ [Test || Test <- Tests, not is_atom(Test)].
+
+%% make sure the order of tests is according to the order in TCs
+sort_tests(Tests, TCs) when is_list(TCs)->
+ lists:sort(fun(T1, T2) ->
+ case {is_tc(T1),is_tc(T2)} of
+ {true,true} ->
+ (position(T1, TCs) =<
+ position(T2, TCs));
+ {false,true} ->
+ (position(T2, TCs) == (length(TCs)+1));
+ _ -> true
+
+ end
+ end, Tests);
+sort_tests(Tests, _) ->
+ Tests.
+
+is_tc(T) when is_atom(T) -> true;
+is_tc({group,_}) -> false;
+is_tc({_M,T}) when is_atom(T) -> true;
+is_tc(_) -> false.
+
+position(T, TCs) ->
+ position(T, TCs, 1).
+
+position(T, [T|_TCs], Pos) ->
+ Pos;
+position(T, [{_,T}|_TCs], Pos) ->
+ Pos;
+position({M,T}, [T|_TCs], Pos) when M /= group ->
+ Pos;
+position(T, [_|TCs], Pos) ->
+ position(T, TCs, Pos+1);
+position(_, [], Pos) ->
+ Pos.
+
+%%%-----------------------------------------------------------------
+
+delete_subs([{conf, _,_,_,_} = Conf | Confs], All) ->
+ All1 = delete_conf(Conf, All),
+ case is_sub(Conf, All1) of
+ true ->
+ delete_subs(Confs, All1);
+ false ->
+ delete_subs(Confs, All)
+ end;
+delete_subs([_Else | Confs], All) ->
+ delete_subs(Confs, All);
+delete_subs([], All) ->
+ All.
+
+delete_conf({conf,Props,_,_,_}, Confs) ->
+ Name = ?val(name, Props),
+ [Conf || Conf = {conf,Props0,_,_,_} <- Confs,
+ Name =/= ?val(name, Props0)].
+
+is_sub({conf,Props,_,_,_}=Conf, [{conf,_,_,Tests,_} | Confs]) ->
+ Name = ?val(name, Props),
+ case lists:any(fun({conf,Props0,_,_,_}) ->
+ case ?val(name, Props0) of
+ N when N == Name ->
+ true;
+ _ ->
+ false
+ end;
+ (_) ->
+ false
+ end, Tests) of
+ true ->
+ true;
+ false ->
+ is_sub(Conf, Tests) orelse is_sub(Conf, Confs)
+ end;
+
+is_sub(Conf, [_TC | Tests]) ->
+ is_sub(Conf, Tests);
+
+is_sub(_Conf, []) ->
+ false.
+
+
+cyclic_test(Mod, Name, Names) ->
+ case lists:member(Name, Names) of
+ true ->
+ E = "Cyclic reference to group "++atom_to_list(Name)++
+ " in "++atom_to_list(Mod)++":groups/0",
+ throw({error,list_to_atom(E)});
+ false ->
+ ok
+ end.
+
+expand(Mod, Name, Defs) ->
+ case lists:keysearch(Name, 1, Defs) of
+ {value,Def} ->
+ Def;
+ false ->
+ E = "Invalid group "++atom_to_list(Name)++
+ " in "++atom_to_list(Mod)++":groups/0",
+ throw({error,list_to_atom(E)})
+ end.
+
+make_all_conf(Dir, Mod, Props, TestSpec) ->
+ case code:is_loaded(Mod) of
+ false ->
+ code:load_abs(filename:join(Dir,atom_to_list(Mod)));
+ _ ->
+ ok
+ end,
+ make_all_conf(Mod, Props, TestSpec).
+
+make_all_conf(Mod, Props, TestSpec) ->
+ case catch apply(Mod, groups, []) of
+ {'EXIT',_} ->
+ exit({invalid_group_definition,Mod});
+ GroupDefs when is_list(GroupDefs) ->
+ case catch find_groups(Mod, all, TestSpec, GroupDefs) of
+ {error,_} = Error ->
+ %% this makes test_server call error_in_suite as first
+ %% (and only) test case so we can report Error properly
+ [{ct_framework,error_in_suite,[[Error]]}];
+ [] ->
+ exit({invalid_group_spec,Mod});
+ _ConfTests ->
+ make_conf(Mod, all, Props, TestSpec)
+ end
+ end.
+
+make_conf(Dir, Mod, Name, Props, TestSpec) ->
+ case code:is_loaded(Mod) of
+ false ->
+ code:load_abs(filename:join(Dir,atom_to_list(Mod)));
+ _ ->
+ ok
+ end,
+ make_conf(Mod, Name, Props, TestSpec).
+
+make_conf(Mod, Name, Props, TestSpec) ->
+ case code:is_loaded(Mod) of
+ false ->
+ code:load_file(Mod);
+ _ ->
+ ok
+ end,
+ {InitConf,EndConf,ExtraProps} =
+ case erlang:function_exported(Mod,init_per_group,2) of
+ true ->
+ {{Mod,init_per_group},{Mod,end_per_group},[]};
+ false ->
+ ct_logs:log("TEST INFO", "init_per_group/2 and "
+ "end_per_group/2 missing for group "
+ "~p in ~p, using default.",
+ [Name,Mod]),
+ {{ct_framework,init_per_group},
+ {ct_framework,end_per_group},
+ [{suite,Mod}]}
+ end,
+ {conf,[{name,Name}|Props++ExtraProps],InitConf,TestSpec,EndConf}.
+
+%%%-----------------------------------------------------------------
+
+expand_groups([H | T], ConfTests, Mod) ->
+ [expand_groups(H, ConfTests, Mod) | expand_groups(T, ConfTests, Mod)];
+expand_groups([], _ConfTests, _Mod) ->
+ [];
+expand_groups({group,Name}, ConfTests, Mod) ->
+ expand_groups({group,Name,default,[]}, ConfTests, Mod);
+expand_groups({group,Name,default}, ConfTests, Mod) ->
+ expand_groups({group,Name,default,[]}, ConfTests, Mod);
+expand_groups({group,Name,ORProps}, ConfTests, Mod) when is_list(ORProps) ->
+ expand_groups({group,Name,ORProps,[]}, ConfTests, Mod);
+expand_groups({group,Name,ORProps,SubORSpec}, ConfTests, Mod) ->
+ FindConf =
+ fun(Conf = {conf,Props,Init,Ts,End}) ->
+ case ?val(name, Props) of
+ Name when ORProps == default ->
+ [Conf];
+ Name ->
+ Props1 = case ?val(suite, Props) of
+ undefined ->
+ ORProps;
+ SuiteName ->
+ [{suite,SuiteName}|ORProps]
+ end,
+ [{conf,[{name,Name}|Props1],Init,Ts,End}];
+ _ ->
+ []
+ end
+ end,
+ case lists:flatmap(FindConf, ConfTests) of
+ [] ->
+ throw({error,invalid_ref_msg(Name, Mod)});
+ Matching when SubORSpec == [] ->
+ Matching;
+ Matching ->
+ override_props(Matching, SubORSpec, Name,Mod)
+ end;
+expand_groups(SeqOrTC, _ConfTests, _Mod) ->
+ SeqOrTC.
+
+%% search deep for the matching conf test and modify it and any
+%% sub tests according to the override specification
+search_and_override([Conf = {conf,Props,Init,Tests,End}], ORSpec, Mod) ->
+ InsProps = fun(GrName, undefined, Ps) ->
+ [{name,GrName} | Ps];
+ (GrName, Suite, Ps) ->
+ [{name,GrName}, {suite,Suite} | Ps]
+ end,
+ Name = ?val(name, Props),
+ Suite = ?val(suite, Props),
+ case lists:keysearch(Name, 1, ORSpec) of
+ {value,{Name,default}} ->
+ [Conf];
+ {value,{Name,ORProps}} ->
+ [{conf,InsProps(Name,Suite,ORProps),Init,Tests,End}];
+ {value,{Name,default,[]}} ->
+ [Conf];
+ {value,{Name,default,SubORSpec}} ->
+ override_props([Conf], SubORSpec, Name,Mod);
+ {value,{Name,ORProps,SubORSpec}} ->
+ override_props([{conf,InsProps(Name,Suite,ORProps),
+ Init,Tests,End}], SubORSpec, Name,Mod);
+ _ ->
+ [{conf,Props,Init,search_and_override(Tests,ORSpec,Mod),End}]
+ end.
+
+%% Modify the Tests element according to the override specification
+override_props([{conf,Props,Init,Tests,End} | Confs], SubORSpec, Name,Mod) ->
+ {Subs,SubORSpec1} = override_sub_props(Tests, [], SubORSpec, Mod),
+ [{conf,Props,Init,Subs,End} | override_props(Confs, SubORSpec1, Name,Mod)];
+override_props([], [], _,_) ->
+ [];
+override_props([], SubORSpec, Name,Mod) ->
+ Es = [invalid_ref_msg(Name, element(1,Spec), Mod) || Spec <- SubORSpec],
+ throw({error,Es}).
+
+override_sub_props([], New, ORSpec, _) ->
+ {?rev(New),ORSpec};
+override_sub_props([T = {conf,Props,Init,Tests,End} | Ts],
+ New, ORSpec, Mod) ->
+ Name = ?val(name, Props),
+ Suite = ?val(suite, Props),
+ case lists:keysearch(Name, 1, ORSpec) of
+ {value,Spec} -> % group found in spec
+ Props1 =
+ case element(2, Spec) of
+ default -> Props;
+ ORProps when Suite == undefined -> [{name,Name} | ORProps];
+ ORProps -> [{name,Name}, {suite,Suite} | ORProps]
+ end,
+ case catch element(3, Spec) of
+ Undef when Undef == [] ; 'EXIT' == element(1, Undef) ->
+ override_sub_props(Ts, [{conf,Props1,Init,Tests,End} | New],
+ lists:keydelete(Name, 1, ORSpec), Mod);
+ SubORSpec when is_list(SubORSpec) ->
+ case override_sub_props(Tests, [], SubORSpec, Mod) of
+ {Subs,[]} ->
+ override_sub_props(Ts, [{conf,Props1,Init,
+ Subs,End} | New],
+ lists:keydelete(Name, 1, ORSpec),
+ Mod);
+ {_,NonEmptySpec} ->
+ Es = [invalid_ref_msg(Name, element(1, GrRef),
+ Mod) || GrRef <- NonEmptySpec],
+ throw({error,Es})
+ end;
+ BadGrSpec ->
+ throw({error,{invalid_form,BadGrSpec}})
+ end;
+ _ -> % not a group in spec
+ override_sub_props(Ts, [T | New], ORSpec, Mod)
+ end;
+override_sub_props([TC | Ts], New, ORSpec, Mod) ->
+ override_sub_props(Ts, [TC | New], ORSpec, Mod).
+
+invalid_ref_msg(Name, Mod) ->
+ E = "Invalid reference to group "++
+ atom_to_list(Name)++" in "++
+ atom_to_list(Mod)++":all/0",
+ list_to_atom(E).
+
+invalid_ref_msg(Name0, Name1, Mod) ->
+ E = "Invalid reference to group "++
+ atom_to_list(Name1)++" from "++atom_to_list(Name0)++
+ " in "++atom_to_list(Mod)++":all/0",
+ list_to_atom(E).
diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index ffa2a21ea9..96b2934382 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -1272,7 +1272,8 @@ run_dir(Opts = #opts{logdir = LogDir,
reformat_result(catch do_run(tests(Dir2, Mod),
[], Opts1, StartOpts));
_ ->
- reformat_result(catch do_run(tests(Dir2, Mod, GsAndCs),
+ reformat_result(catch do_run(tests(Dir2, Mod,
+ GsAndCs),
[], Opts1, StartOpts))
end;
@@ -1281,7 +1282,8 @@ run_dir(Opts = #opts{logdir = LogDir,
[_,_|_] when GsAndCs /= [] ->
exit({error,multiple_suites_and_cases});
[{Dir2,Mod}] when GsAndCs /= [] ->
- reformat_result(catch do_run(tests(Dir2, Mod, GsAndCs),
+ reformat_result(catch do_run(tests(Dir2, Mod,
+ GsAndCs),
[], Opts1, StartOpts));
DirMods ->
reformat_result(catch do_run(tests(DirMods),
@@ -1536,17 +1538,36 @@ groups_and_cases(Gs, Cs) when ((Gs == undefined) or (Gs == [])) and
((Cs == undefined) or (Cs == [])) ->
[];
groups_and_cases(Gs, Cs) when Gs == undefined ; Gs == [] ->
- [ensure_atom(C) || C <- listify(Cs)];
-groups_and_cases(Gs, Cs) when Cs == undefined ; Cs == [] ->
- [{ensure_atom(G),all} || G <- listify(Gs)];
-groups_and_cases(G, Cs) when is_atom(G) ->
- [{G,[ensure_atom(C) || C <- listify(Cs)]}];
-groups_and_cases([G], Cs) ->
- [{ensure_atom(G),[ensure_atom(C) || C <- listify(Cs)]}];
-groups_and_cases([_,_|_] , Cs) when Cs =/= [] ->
- {error,multiple_groups_and_cases};
-groups_and_cases(_Gs, _Cs) ->
- {error,incorrect_group_or_case_option}.
+ if (Cs == all) or (Cs == [all]) or (Cs == ["all"]) -> all;
+ true -> [ensure_atom(C) || C <- listify(Cs)]
+ end;
+groups_and_cases(GOrGs, Cs) when (is_atom(GOrGs) orelse
+ (is_list(GOrGs) andalso
+ (is_atom(hd(GOrGs)) orelse
+ (is_list(hd(GOrGs)) andalso
+ is_atom(hd(hd(GOrGs))))))) ->
+ if (Cs == undefined) or (Cs == []) or
+ (Cs == all) or (Cs == [all]) or (Cs == ["all"]) ->
+ [{GOrGs,all}];
+ true ->
+ [{GOrGs,[ensure_atom(C) || C <- listify(Cs)]}]
+ end;
+groups_and_cases(Gs, Cs) when is_integer(hd(hd(Gs))) ->
+ %% if list of strings, this comes from 'ct_run -group G1 G2 ...' and
+ %% we need to parse the strings
+ Gs1 =
+ if (Gs == [all]) or (Gs == ["all"]) ->
+ all;
+ true ->
+ lists:map(fun(G) ->
+ {ok,Ts,_} = erl_scan:string(G++"."),
+ {ok,Term} = erl_parse:parse_term(Ts),
+ Term
+ end, Gs)
+ end,
+ groups_and_cases(Gs1, Cs);
+groups_and_cases(Gs, Cs) ->
+ {error,{incorrect_group_or_case_option,Gs,Cs}}.
tests(TestDir, Suites, []) when is_list(TestDir), is_integer(hd(TestDir)) ->
[{?testdir(TestDir,Suites),ensure_atom(Suites),all}];
@@ -1687,11 +1708,15 @@ compile_and_run(Tests, Skip, Opts, Args) ->
SavedErrors = save_make_errors(SuiteMakeErrors),
ct_repeat:log_loop_info(Args),
- {Tests1,Skip1} = final_tests(Tests,Skip,SavedErrors),
-
- ReleaseSh = proplists:get_value(release_shell, Args),
- ct_util:set_testdata({release_shell,ReleaseSh}),
- possibly_spawn(ReleaseSh == true, Tests1, Skip1, Opts);
+ try final_tests(Tests,Skip,SavedErrors) of
+ {Tests1,Skip1} ->
+ ReleaseSh = proplists:get_value(release_shell, Args),
+ ct_util:set_testdata({release_shell,ReleaseSh}),
+ possibly_spawn(ReleaseSh == true, Tests1, Skip1, Opts)
+ catch
+ _:BadFormat ->
+ {error,BadFormat}
+ end;
false ->
io:nl(),
ct_util:stop(clean),
@@ -1961,22 +1986,21 @@ final_tests1([{TestDir,Suite,GrsOrCs}|Tests], Final, Skip, Bad) when
%% for now, only flat group defs are allowed as
%% start options and test spec terms
fun({all,all}) ->
- ct_framework:make_all_conf(TestDir,
- Suite, []);
+ [ct_groups:make_conf(TestDir, Suite, all, [], all)];
({skipped,Group,TCs}) ->
- [ct_framework:make_conf(TestDir, Suite,
- Group, [skipped], TCs)];
- ({GrSpec = {Group,_},TCs}) ->
+ [ct_groups:make_conf(TestDir, Suite,
+ Group, [skipped], TCs)];
+ ({GrSpec = {GroupName,_},TCs}) ->
Props = [{override,GrSpec}],
- [ct_framework:make_conf(TestDir, Suite,
- Group, Props, TCs)];
- ({GrSpec = {Group,_,_},TCs}) ->
+ [ct_groups:make_conf(TestDir, Suite,
+ GroupName, Props, TCs)];
+ ({GrSpec = {GroupName,_,_},TCs}) ->
Props = [{override,GrSpec}],
- [ct_framework:make_conf(TestDir, Suite,
- Group, Props, TCs)];
- ({Group,TCs}) ->
- [ct_framework:make_conf(TestDir, Suite,
- Group, [], TCs)];
+ [ct_groups:make_conf(TestDir, Suite,
+ GroupName, Props, TCs)];
+ ({GroupOrGroups,TCs}) ->
+ [ct_groups:make_conf(TestDir, Suite,
+ GroupOrGroups, [], TCs)];
(TC) ->
[TC]
end, GrsOrCs),
@@ -1988,12 +2012,12 @@ final_tests1([], Final, Skip, _Bad) ->
{lists:reverse(Final),Skip}.
final_skip([{TestDir,Suite,{all,all},Reason}|Skips], Final) ->
- SkipConf = ct_framework:make_conf(TestDir, Suite, all, [], all),
+ SkipConf = ct_groups:make_conf(TestDir, Suite, all, [], all),
Skip = {TestDir,Suite,SkipConf,Reason},
final_skip(Skips, [Skip|Final]);
final_skip([{TestDir,Suite,{Group,TCs},Reason}|Skips], Final) ->
- Conf = ct_framework:make_conf(TestDir, Suite, Group, [], TCs),
+ Conf = ct_groups:make_conf(TestDir, Suite, Group, [], TCs),
Skip = {TestDir,Suite,Conf,Reason},
final_skip(Skips, [Skip|Final]);
@@ -2256,9 +2280,11 @@ add_jobs([{TestDir,all,_}|Tests], Skip, Opts, CleanUp) ->
wait_for_idle(),
add_jobs(Tests, Skip, Opts, CleanUp)
end;
-add_jobs([{TestDir,[Suite],all}|Tests], Skip, Opts, CleanUp) when is_atom(Suite) ->
+add_jobs([{TestDir,[Suite],all}|Tests], Skip,
+ Opts, CleanUp) when is_atom(Suite) ->
add_jobs([{TestDir,Suite,all}|Tests], Skip, Opts, CleanUp);
-add_jobs([{TestDir,Suites,all}|Tests], Skip, Opts, CleanUp) when is_list(Suites) ->
+add_jobs([{TestDir,Suites,all}|Tests], Skip,
+ Opts, CleanUp) when is_list(Suites) ->
Name = get_name(TestDir) ++ ".suites",
case catch test_server_ctrl:add_module_with_skip(Name, Suites,
skiplist(TestDir,Skip)) of
@@ -2273,7 +2299,8 @@ add_jobs([{TestDir,Suite,all}|Tests], Skip, Opts, CleanUp) ->
ok ->
Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite),
case catch test_server_ctrl:add_module_with_skip(Name, [Suite],
- skiplist(TestDir,Skip)) of
+ skiplist(TestDir,
+ Skip)) of
{'EXIT',_} ->
CleanUp;
_ ->
@@ -2296,15 +2323,24 @@ add_jobs([{TestDir,Suite,Confs}|Tests], Skip, Opts, CleanUp) when
GrTestName =
case Confs of
[Conf] ->
- "." ++ atom_to_list(Group(Conf)) ++ TCTestName(TestCases(Conf));
+ case Group(Conf) of
+ GrName when is_atom(GrName) ->
+ "." ++ atom_to_list(GrName) ++
+ TCTestName(TestCases(Conf));
+ _ ->
+ ".groups" ++ TCTestName(TestCases(Conf))
+ end;
_ ->
".groups"
end,
TestName = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ GrTestName,
case maybe_interpret(Suite, init_per_group, Opts) of
ok ->
- case catch test_server_ctrl:add_conf_with_skip(TestName, Suite, Confs,
- skiplist(TestDir,Skip)) of
+ case catch test_server_ctrl:add_conf_with_skip(TestName,
+ Suite,
+ Confs,
+ skiplist(TestDir,
+ Skip)) of
{'EXIT',_} ->
CleanUp;
_ ->
@@ -2316,18 +2352,21 @@ add_jobs([{TestDir,Suite,Confs}|Tests], Skip, Opts, CleanUp) when
end;
%% test case
-add_jobs([{TestDir,Suite,[Case]}|Tests], Skip, Opts, CleanUp) when is_atom(Case) ->
+add_jobs([{TestDir,Suite,[Case]}|Tests],
+ Skip, Opts, CleanUp) when is_atom(Case) ->
add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opts, CleanUp);
-add_jobs([{TestDir,Suite,Cases}|Tests], Skip, Opts, CleanUp) when is_list(Cases) ->
+add_jobs([{TestDir,Suite,Cases}|Tests],
+ Skip, Opts, CleanUp) when is_list(Cases) ->
Cases1 = lists:map(fun({GroupName,_}) when is_atom(GroupName) -> GroupName;
(Case) -> Case
end, Cases),
case maybe_interpret(Suite, Cases1, Opts) of
ok ->
- Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ ".cases",
+ Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ ".cases",
case catch test_server_ctrl:add_cases_with_skip(Name, Suite, Cases1,
- skiplist(TestDir,Skip)) of
+ skiplist(TestDir,
+ Skip)) of
{'EXIT',_} ->
CleanUp;
_ ->
@@ -2343,7 +2382,8 @@ add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opts, CleanUp) when is_atom(Case) -
Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ "." ++
atom_to_list(Case),
case catch test_server_ctrl:add_case_with_skip(Name, Suite, Case,
- skiplist(TestDir,Skip)) of
+ skiplist(TestDir,
+ Skip)) of
{'EXIT',_} ->
CleanUp;
_ ->
@@ -2378,7 +2418,8 @@ skiplist(Dir, [{Dir,all,Cmt}|Skip]) ->
%% we need to turn 'all' into list of modules since
%% test_server doesn't do skips on Dir level
Ss = filelib:wildcard(filename:join(Dir, "*_SUITE.beam")),
- [{list_to_atom(filename:basename(S,".beam")),Cmt} || S <- Ss] ++ skiplist(Dir,Skip);
+ [{list_to_atom(filename:basename(S,".beam")),Cmt} || S <- Ss] ++
+ skiplist(Dir,Skip);
skiplist(Dir, [{Dir,S,Cmt}|Skip]) ->
[{S,Cmt} | skiplist(Dir, Skip)];
skiplist(Dir, [{Dir,S,C,Cmt}|Skip]) ->
@@ -2438,8 +2479,10 @@ run_make(Targets, TestDir0, Mod, UserInclude) ->
FileTest = fun(F, suites) -> is_suite(F);
(F, helpmods) -> not is_suite(F)
end,
- Files = lists:flatmap(fun({F,out_of_date}) ->
- case FileTest(F, Targets) of
+ Files =
+ lists:flatmap(fun({F,out_of_date}) ->
+ case FileTest(F,
+ Targets) of
true -> [F];
false -> []
end;
@@ -2783,7 +2826,8 @@ opts2args(EnvStartOpts) ->
lists:flatmap(fun({exit_status,ExitStatusOpt}) when is_atom(ExitStatusOpt) ->
[{exit_status,[atom_to_list(ExitStatusOpt)]}];
({halt_with,{HaltM,HaltF}}) ->
- [{halt_with,[atom_to_list(HaltM),atom_to_list(HaltF)]}];
+ [{halt_with,[atom_to_list(HaltM),
+ atom_to_list(HaltF)]}];
({interactive_mode,true}) ->
[{shell,[]}];
({config,CfgFile}) when is_integer(hd(CfgFile)) ->
@@ -2807,6 +2851,12 @@ opts2args(EnvStartOpts) ->
end, UserCfg),
[_LastAnd|StrsR] = lists:reverse(lists:flatten(Strs)),
[{userconfig,lists:reverse(StrsR)}];
+ ({group,G}) when is_atom(G) ->
+ [{group,[atom_to_list(G)]}];
+ ({group,Gs}) when is_list(Gs) ->
+ LOfGStrs = [lists:flatten(io_lib:format("~w",[G])) ||
+ G <- Gs],
+ [{group,LOfGStrs}];
({testcase,Case}) when is_atom(Case) ->
[{'case',[atom_to_list(Case)]}];
({testcase,Cases}) ->
diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl
index a8b67d0329..5ce095e38e 100644
--- a/lib/common_test/src/ct_testspec.erl
+++ b/lib/common_test/src/ct_testspec.erl
@@ -1026,20 +1026,24 @@ insert_groups(Node,Dir,Suite,Group,Cases,Tests,MergeTests)
insert_groups(Node,Dir,Suite,[Group],Cases,Tests,MergeTests);
insert_groups(Node,Dir,Suite,Groups,Cases,Tests,false) when
((Cases == all) or is_list(Cases)) and is_list(Groups) ->
- Groups1 = [{Gr,Cases} || Gr <- Groups],
+ Groups1 = [if is_list(Gr) -> % preserve group path
+ {[Gr],Cases};
+ true ->
+ {Gr,Cases} end || Gr <- Groups],
append({{Node,Dir},[{Suite,Groups1}]},Tests);
insert_groups(Node,Dir,Suite,Groups,Cases,Tests,true) when
((Cases == all) or is_list(Cases)) and is_list(Groups) ->
+ Groups1 = [if is_list(Gr) -> % preserve group path
+ {[Gr],Cases};
+ true ->
+ {Gr,Cases} end || Gr <- Groups],
case lists:keysearch({Node,Dir},1,Tests) of
{value,{{Node,Dir},[{all,_}]}} ->
Tests;
{value,{{Node,Dir},Suites0}} ->
- Suites1 = insert_groups1(Suite,
- [{Gr,Cases} || Gr <- Groups],
- Suites0),
+ Suites1 = insert_groups1(Suite,Groups1,Suites0),
insert_in_order({{Node,Dir},Suites1},Tests);
false ->
- Groups1 = [{Gr,Cases} || Gr <- Groups],
insert_in_order({{Node,Dir},[{Suite,Groups1}]},Tests)
end;
insert_groups(Node,Dir,Suite,Groups,Case,Tests, MergeTests)
@@ -1062,13 +1066,13 @@ insert_groups1(Suite,Groups,Suites0) ->
insert_groups2(_Groups,all) ->
all;
-insert_groups2([Group={GrName,Cases}|Groups],GrAndCases) ->
- case lists:keysearch(GrName,1,GrAndCases) of
- {value,{GrName,all}} ->
+insert_groups2([Group={Gr,Cases}|Groups],GrAndCases) ->
+ case lists:keysearch(Gr,1,GrAndCases) of
+ {value,{Gr,all}} ->
GrAndCases;
- {value,{GrName,Cases0}} ->
+ {value,{Gr,Cases0}} ->
Cases1 = insert_in_order(Cases,Cases0),
- insert_groups2(Groups,insert_in_order({GrName,Cases1},GrAndCases));
+ insert_groups2(Groups,insert_in_order({Gr,Cases1},GrAndCases));
false ->
insert_groups2(Groups,insert_in_order(Group,GrAndCases))
end;
diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile
index 686ee43aa3..374fd8a824 100644
--- a/lib/common_test/test/Makefile
+++ b/lib/common_test/test/Makefile
@@ -51,7 +51,8 @@ MODULES= \
ct_basic_html_SUITE \
ct_auto_compile_SUITE \
ct_verbosity_SUITE \
- ct_shell_SUITE
+ ct_shell_SUITE \
+ ct_groups_search_SUITE
ERL_FILES= $(MODULES:%=%.erl)
diff --git a/lib/common_test/test/ct_groups_search_SUITE.erl b/lib/common_test/test/ct_groups_search_SUITE.erl
new file mode 100644
index 0000000000..6b1c1f4634
--- /dev/null
+++ b/lib/common_test/test/ct_groups_search_SUITE.erl
@@ -0,0 +1,1245 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File:
+%%%
+%%% Description:
+%%%
+%%%
+%%% The suites used for the test are located in the data directory.
+%%%
+%%% The group(s) and case(s) are specified according to this:
+%%%
+%%% Tests = ct_groups:find_groups(Mod, GroupPaths, TestCases, GroupDef)
+%%%
+%%% GroupPaths = GroupPath | [GroupPath]
+%%% GroupPath = atom() | [atom()]
+%%%
+%%% CT will find all paths that include GroupPath. GroupPath can be a
+%%% single group, or a list of groups along the path to TestCases.
+%%% If GroupPath is the latter, the last group in the list must be
+%%% the "terminating" group in the path, or it will be impossible to
+%%% execute test cases in higher level groups *only*, as in this case:
+%%% groups() -> [{g1,[],[tc1,{g2,[],[tc2]}]}].
+%%% Compare: find_groups(x, g1, all, groups()), and
+%%% find_groups(x, [[g1]], all, groups())
+%%%
+%%% Some examples:
+%%%
+%%% GroupPaths = g1, means find all paths with g1 included
+%%% GroupPaths = [g1], -''-
+%%% GroupPaths = [g1,g2], search twice - once for g1 and once for g2
+%%% GroupPaths = [[g1,g2]], find cases under group g1 and sub group g2
+%%% GroupPaths = [[g1,g2],[g1,g3]], find cases for g1-g2 AND g1-g3
+%%%
+%%% TestCases = all | atom() | [atom()]
+%%%
+%%%-------------------------------------------------------------------
+
+-module(ct_groups_search_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/src/ct_util.hrl").
+
+
+-define(eh, ct_test_support_eh).
+
+-define(M1, groups_search_dummy_1_SUITE).
+-define(M2, groups_search_dummy_2_SUITE).
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+init_per_suite(Config) ->
+ DataDir = proplists:get_value(data_dir, Config),
+ code:add_patha(DataDir),
+ M1Erl = filename:join(DataDir, atom_to_list(?M1)++".erl"),
+ M2Erl = filename:join(DataDir, atom_to_list(?M2)++".erl"),
+ {ok,?M1} = compile:file(M1Erl, [{outdir,DataDir}]),
+ {ok,?M2} = compile:file(M2Erl, [{outdir,DataDir}]),
+ {module,?M1} = code:load_file(?M1),
+ {module,?M2} = code:load_file(?M2),
+
+ Config1 = ct_test_support:init_per_suite(Config),
+ Config1.
+
+end_per_suite(Config) ->
+ ct_test_support:end_per_suite(Config).
+
+init_per_testcase(TestCase, Config) ->
+ ct_test_support:init_per_testcase(TestCase, Config).
+
+end_per_testcase(TestCase, Config) ->
+ ct_test_support:end_per_testcase(TestCase, Config).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+groups() ->
+ [
+ {find_groups,[],[all_groups,
+ testcases_in_all_groups,
+ all_in_top_group1,
+ all_in_top_group2,
+ all_in_sub_group1,
+ all_in_sub_group2,
+ testcase_in_top_group1,
+ testcase_in_top_group2,
+ testcase_in_sub_group1,
+ testcase_in_sub_group2,
+ testcase_in_top_groups1,
+ testcase_in_top_groups2,
+ testcase_in_top_groups3,
+ testcase_in_top_groups4,
+ testcase_in_top_groups5,
+ testcase_in_top_groups6,
+ testcase_in_top_groups7,
+ testcase_in_sub_groups1,
+ testcase_in_sub_groups2,
+ testcase_in_sub_groups3,
+ testcase_in_sub_groups4,
+ testcase_in_sub_groups5,
+ testcase_in_sub_groups6,
+ testcase_in_sub_groups7,
+ testcase_in_sub_groups8,
+ testcase_in_sub_groups9,
+ testcase_in_sub_groups10,
+ testcase_in_sub_groups11,
+ testcase_in_sub_groups12,
+ testcase_in_sub_groups13,
+ bad_testcase_in_sub_groups1]},
+
+ {run_groups,[sequence],[run_groups_with_options,
+ run_groups_with_testspec]}
+ ].
+
+all() ->
+ [{group,find_groups,[parallel]},
+ {group,run_groups}].
+
+
+
+%%--------------------------------------------------------------------
+%% TEST CASES CHECKING RETURN VALUE ONLY
+%%--------------------------------------------------------------------
+
+all_groups(_) ->
+ GPath = all, TCs = all,
+
+ Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()),
+
+ Top1 = ct_groups:find_groups(?M1, top1, TCs, groups1()),
+ Top2 = ct_groups:find_groups(?M1, top2, TCs, groups1()),
+
+ All = Top1 ++ Top2 ++ [{conf,[{name,sub2}],
+ {?M1,init_per_group},
+ [{?M1,sub2_tc1},{?M1,sub2_tc2}],
+ {?M1,end_per_group}}],
+
+ All = Found,
+
+ {?M1,GPath,TCs,Top1++Top2}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcases_in_all_groups(_) ->
+ GPath = all, TCs = [tc3,sub_tc2],
+
+ Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()),
+
+ [Top1 =
+ {conf,[{name,top1}],{?M2,init_per_group},
+ [{?M2,tc3},
+ {conf,[{name,sub11}],
+ {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}],
+ {?M2,end_per_group}},
+ {conf,[{name,sub12}],
+ {?M2,init_per_group},
+ [{?M2,tc3},{?M2,sub_tc2},
+ {conf,[{name,sub121}],
+ {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}},
+
+ Top2 =
+ {conf,[{name,top2}],{?M2,init_per_group},
+ [{?M2,tc3},
+ {conf,[{name,sub21}],
+ {?M2,init_per_group},
+ [{?M2,tc3},{?M2,sub_tc2},
+ {conf,[{name,sub2xx}],
+ {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}},
+
+ {conf,[{name,sub22}],
+ {?M2,init_per_group},
+ [{?M2,tc3},{?M2,sub_tc2},
+ {conf,[{name,sub221}],
+ {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}],
+ {?M2,end_per_group}},
+ {conf,[{name,sub2xx}],
+ {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}},
+
+ {conf,[{name,sub21}],
+ {?M2,init_per_group},
+ [{?M2,tc3},{?M2,sub_tc2},
+ {conf,[{name,sub2xx}],
+ {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}],{?M2,end_per_group}}],
+ {?M2,end_per_group}},
+
+ {conf,[{name,sub22}],
+ {?M2,init_per_group},
+ [{?M2,tc3},{?M2,sub_tc2},
+ {conf,[{name,sub221}],
+ {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}],{?M2,end_per_group}},
+ {conf,[{name,sub2xx}],
+ {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}],{?M2,end_per_group}}],
+ {?M2,end_per_group}},
+
+ {conf,[{name,sub221}],
+ {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}],{?M2,end_per_group}},
+
+ {conf,[{name,sub2xx}],
+ {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}],{?M2,end_per_group}}]
+
+ = Found,
+
+ {?M2,GPath,TCs,[Top1,Top2]}.
+
+%%%-----------------------------------------------------------------
+%%%
+all_in_top_group1(_) ->
+ GPath= top1, TCs = all,
+
+ Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()),
+
+ [{conf,[{name,top1}],
+ {?M1,init_per_group},
+ [{?M1,top1_tc1},{?M1,top1_tc2},
+ {conf,[{name,sub1}],
+ {?M1,init_per_group},
+ [{?M1,sub1_tc1},{?M1,sub1_tc2}],
+ {?M1,end_per_group}}],
+ {?M1,end_per_group}}] = Found,
+
+ {?M1,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+all_in_top_group2(_) ->
+ GPath= top2, TCs = all,
+
+ Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()),
+
+ [{conf,[{name,top2}],
+ {?M1,init_per_group},
+ [{conf,[{name,sub2}],
+ {?M1,init_per_group},
+ [{?M1,sub2_tc1},{?M1,sub2_tc2}],
+ {?M1,end_per_group}},
+ {?M1,top2_tc1},{?M1,top2_tc2}],
+ {?M1,end_per_group}}] = Found,
+
+ {?M1,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+all_in_sub_group1(_) ->
+ GPath = sub1, TCs = all,
+
+ Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()),
+
+ [{conf,[{name,top1}],
+ {?M1,init_per_group},
+ [{conf,[{name,sub1}],
+ {?M1,init_per_group},
+ [{?M1,sub1_tc1},{?M1,sub1_tc2}],
+ {?M1,end_per_group}}],
+ {?M1,end_per_group}}] = Found,
+
+ {?M1,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+all_in_sub_group2(_) ->
+ GPath = sub2, TCs = all,
+
+ Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()),
+
+ [Top2 =
+ {conf,[{name,top2}],
+ {?M1,init_per_group},
+ [{conf,[{name,sub2}],
+ {?M1,init_per_group},
+ [{?M1,sub2_tc1},{?M1,sub2_tc2}],
+ {?M1,end_per_group}}],
+ {?M1,end_per_group}},
+
+ {conf,[{name,sub2}],
+ {?M1,init_per_group},
+ [{?M1,sub2_tc1},{?M1,sub2_tc2}],
+ {?M1,end_per_group}}] = Found,
+
+ {?M1,GPath,TCs,Top2}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_top_group1(_) ->
+ GPath = top1, TCs = [top1_tc2],
+
+ Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()),
+
+ [{conf,[{name,top1}],
+ {?M1,init_per_group},
+ [{?M1,top1_tc2}],
+ {?M1,end_per_group}}] = Found,
+
+ {?M1,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_top_group2(_) ->
+ GPath = top2, TCs = [top2_tc2],
+
+ Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()),
+
+ [{conf,[{name,top2}],
+ {?M1,init_per_group},
+ [{?M1,top2_tc2}],
+ {?M1,end_per_group}}] = Found,
+
+ {?M1,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_sub_group1(_) ->
+ GPath = sub1, TCs = [sub1_tc2],
+
+ Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()),
+
+ [{conf,[{name,top1}],
+ {?M1,init_per_group},
+ [{conf,[{name,sub1}],
+ {?M1,init_per_group},
+ [{?M1,sub1_tc2}],
+ {?M1,end_per_group}}],
+ {?M1,end_per_group}}] = Found,
+
+ {?M1,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_sub_group2(_) ->
+ GPath = sub2, TCs = [sub2_tc2],
+
+ Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()),
+
+ [Top2 =
+ {conf,[{name,top2}],
+ {?M1,init_per_group},
+ [{conf,[{name,sub2}],
+ {?M1,init_per_group},
+ [{?M1,sub2_tc2}],
+ {?M1,end_per_group}}],
+ {?M1,end_per_group}},
+
+ {conf,[{name,sub2}],
+ {?M1,init_per_group},
+ [{?M1,sub2_tc2}],
+ {?M1,end_per_group}}] = Found,
+
+ {?M1,GPath,TCs,Top2}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_top_groups1(_) ->
+ GPath = [top1,top2], TCs = all,
+
+ Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()),
+
+ [{conf,[{name,top1}],
+ {?M2,init_per_group},
+ [{?M2,top1_tc1},{?M2,top_tc2},{?M2,tc3},
+ {conf,[{name,sub11}],
+ {?M2,init_per_group},
+ [{?M2,sub11_tc1},{?M2,sub_tc2},{?M2,tc3}],
+ {?M2,end_per_group}},
+ {conf,[{name,sub12}],
+ {?M2,init_per_group},
+ [{?M2,sub12_tc1},{?M2,sub_tc2},{?M2,tc3},
+ {conf,[{name,sub121}],
+ {?M2,init_per_group},
+ [{?M2,sub121_tc1},{?M2,sub_tc2},{?M2,tc3}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}},
+
+ {conf,[{name,top2}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub21}],
+ {?M2,init_per_group},
+ [{?M2,sub21_tc1},{?M2,sub_tc2},{?M2,tc3},
+ {conf,[{name,sub2xx}],
+ {?M2,init_per_group},
+ [{?M2,sub2xx_tc1},{?M2,sub_tc2},{?M2,tc3}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}},
+ {?M2,top2_tc1},{?M2,top_tc2},{?M2,tc3},
+ {conf,[{name,sub22}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub221}],
+ {?M2,init_per_group},
+ [{?M2,sub221_tc1},{?M2,sub_tc2},{?M2,tc3}],
+ {?M2,end_per_group}},
+ {?M2,sub22_tc1},{?M2,sub_tc2},{?M2,tc3},
+ {conf,[{name,sub2xx}],
+ {?M2,init_per_group},
+ [{?M2,sub2xx_tc1},{?M2,sub_tc2},{?M2,tc3}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}] = Found,
+
+ {?M2,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_top_groups2(_) ->
+ GPath = [top1,top2], TCs = tc3,
+
+ Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()),
+
+ [{conf,[{name,top1}],
+ {?M2,init_per_group},
+ [{?M2,tc3},
+ {conf,[{name,sub11}],
+ {?M2,init_per_group},
+ [{?M2,tc3}],
+ {?M2,end_per_group}},
+ {conf,[{name,sub12}],
+ {?M2,init_per_group},
+ [{?M2,tc3},
+ {conf,[{name,sub121}],
+ {?M2,init_per_group},
+ [{?M2,tc3}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}},
+
+ {conf,[{name,top2}],
+ {?M2,init_per_group},
+ [{?M2,tc3},
+ {conf,[{name,sub21}],
+ {?M2,init_per_group},
+ [{?M2,tc3},
+ {conf,[{name,sub2xx}],
+ {?M2,init_per_group},
+ [{?M2,tc3}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}},
+
+ {conf,[{name,sub22}],
+ {?M2,init_per_group},
+ [{?M2,tc3},
+ {conf,[{name,sub221}],
+ {?M2,init_per_group},
+ [{?M2,tc3}],
+ {?M2,end_per_group}},
+ {conf,[{name,sub2xx}],
+ {?M2,init_per_group},
+ [{?M2,tc3}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}] = Found,
+
+ {?M2,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_top_groups3(_) ->
+ GPath = [top1,top2], TCs = top1_tc1,
+
+ Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()),
+
+ [{conf,[{name,top1}],
+ {?M2,init_per_group},
+ [{?M2,top1_tc1}],
+ {?M2,end_per_group}}] = Found,
+
+ {?M2,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_top_groups4(_) ->
+ GPath = [top1,top2], TCs = sub2xx_tc1,
+
+ Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()),
+
+ [{conf,[{name,top2}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub21}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub2xx}],
+ {?M2,init_per_group},
+ [{?M2,sub2xx_tc1}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}},
+ {conf,[{name,sub22}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub2xx}],
+ {?M2,init_per_group},
+ [{?M2,sub2xx_tc1}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}] = Found,
+
+ {?M2,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_top_groups5(_) ->
+ GPath = [top1,top2], TCs = [sub21_tc1,sub22_tc1],
+
+ Found = ct_groups:find_groups(?M2, [top1,top2], [sub21_tc1,sub22_tc1],
+ groups2()),
+
+ [{conf,[{name,top2}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub21}],
+ {?M2,init_per_group},
+ [{?M2,sub21_tc1}],
+ {?M2,end_per_group}},
+ {conf,[{name,sub22}],
+ {?M2,init_per_group},
+ [{?M2,sub22_tc1}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}] = Found,
+
+ {?M2,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_top_groups6(_) ->
+ GPath = [[top1],[top2]], TCs = tc3,
+
+ Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()),
+
+ [{conf,[{name,top1}],
+ {?M2,init_per_group},
+ [{?M2,tc3}],
+ {?M2,end_per_group}},
+ {conf,[{name,top2}],
+ {?M2,init_per_group},
+ [{?M2,tc3}],
+ {?M2,end_per_group}}] = Found,
+
+ {?M2,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_top_groups7(_) ->
+ GPath = [[top1],[top2]], TCs = all,
+
+ Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()),
+
+ [{conf,[{name,top1}],
+ {?M2,init_per_group},
+ [{?M2,top1_tc1},
+ {?M2,top_tc2},
+ {?M2,tc3}],
+ {?M2,end_per_group}},
+ {conf,[{name,top2}],
+ {?M2,init_per_group},
+ [{?M2,top2_tc1},
+ {?M2,top_tc2},
+ {?M2,tc3}],
+ {?M2,end_per_group}}] = Found,
+
+ {?M2,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_sub_groups1(_) ->
+ GPath = [sub121], TCs = tc3,
+
+ Found = ct_groups:find_groups(?M2, sub121, tc3, groups2()),
+ Found = ct_groups:find_groups(?M2, [sub121], tc3, groups2()),
+
+ [{conf,[{name,top1}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub12}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub121}],
+ {?M2,init_per_group},
+ [{?M2,tc3}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}] = Found,
+
+ {?M2,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_sub_groups2(_) ->
+ GPath = sub12, TCs = tc3,
+
+ Found = ct_groups:find_groups(?M2, sub12, tc3, groups2()),
+ Found = ct_groups:find_groups(?M2, [sub12], tc3, groups2()),
+
+ [{conf,[{name,top1}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub12}],
+ {?M2,init_per_group},
+ [{?M2,tc3},
+ {conf,[{name,sub121}],
+ {?M2,init_per_group},
+ [{?M2,tc3}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}] = Found,
+
+ FoundX = ct_groups:find_groups(?M2, [[sub12]], tc3, groups2()),
+
+ [{conf,[{name,top1}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub12}],
+ {?M2,init_per_group},
+ [{?M2,tc3}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}] = FoundX,
+
+ {?M2,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_sub_groups3(_) ->
+ GPath = [sub121,sub221], TCs = all,
+
+ Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()),
+
+ [Top1 =
+ {conf,[{name,top1}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub12}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub121}],
+ {?M2,init_per_group},
+ [{?M2,sub121_tc1},
+ {?M2,sub_tc2},
+ {?M2,tc3}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}},
+
+ Top2 =
+ {conf,[{name,top2}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub22}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub221}],
+ {?M2,init_per_group},
+ [{?M2,sub221_tc1},
+ {?M2,sub_tc2},
+ {?M2,tc3}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}},
+
+ {conf,[{name,sub22}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub221}],
+ {?M2,init_per_group},
+ [{?M2,sub221_tc1},
+ {?M2,sub_tc2},
+ {?M2,tc3}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}},
+
+ {conf,[{name,sub221}],
+ {?M2,init_per_group},
+ [{?M2,sub221_tc1},
+ {?M2,sub_tc2},
+ {?M2,tc3}],
+ {?M2,end_per_group}}] = Found,
+
+ {?M2,GPath,TCs,[Top1,Top2]}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_sub_groups4(_) ->
+ GPath = [top1,sub21], TCs = sub_tc2,
+
+ Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()),
+
+ [Top1 =
+ {conf,[{name,top1}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub11}],
+ {?M2,init_per_group},
+ [{?M2,sub_tc2}],
+ {?M2,end_per_group}},
+ {conf,[{name,sub12}],
+ {?M2,init_per_group},
+ [{?M2,sub_tc2},
+ {conf,[{name,sub121}],
+ {?M2,init_per_group},
+ [{?M2,sub_tc2}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}},
+
+ Top2 =
+ {conf,[{name,top2}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub21}],
+ {?M2,init_per_group},
+ [{?M2,sub_tc2},
+ {conf,[{name,sub2xx}],
+ {?M2,init_per_group},
+ [{?M2,sub_tc2}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}},
+
+ {conf,[{name,sub21}],
+ {?M2,init_per_group},
+ [{?M2,sub_tc2},
+ {conf,[{name,sub2xx}],
+ {?M2,init_per_group},
+ [{?M2,sub_tc2}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}] = Found,
+
+ {?M2,GPath,TCs,[Top1,Top2]}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_sub_groups5(_) ->
+ GPath = [[top1,sub12]], TCs = sub12_tc1,
+
+ Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()),
+
+ [{conf,[{name,top1}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub12}],
+ {?M2,init_per_group},
+ [{?M2,sub12_tc1}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}] = Found,
+
+ {?M2,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_sub_groups6(_) ->
+ GPath = [[top1,sub12]], TCs = [sub_tc2],
+
+ Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()),
+
+ [{conf,[{name,top1}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub12}],
+ {?M2,init_per_group},
+ [{?M2,sub_tc2}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}] = Found,
+
+ {?M2,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_sub_groups7(_) ->
+ GPath = [[top1,sub12]], TCs = [sub12_tc1,sub_tc2],
+
+ Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()),
+
+ [{conf,[{name,top1}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub12}],
+ {?M2,init_per_group},
+ [{?M2,sub12_tc1},
+ {?M2,sub_tc2}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}] = Found,
+
+ {?M2,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_sub_groups8(_) ->
+ GPath = [[top2,sub22]], TCs = [sub22_tc1,sub_tc2],
+
+ Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()),
+
+ [{conf,[{name,top2}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub22}],
+ {?M2,init_per_group},
+ [{?M2,sub22_tc1},
+ {?M2,sub_tc2}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}] = Found,
+
+ {?M2,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_sub_groups9(_) ->
+ GPath = [[sub2xx]], TCs = tc3,
+
+ Found = ct_groups:find_groups(?M2, sub2xx, tc3, groups2()),
+ Found = ct_groups:find_groups(?M2, [[sub2xx]], tc3, groups2()),
+
+ [Top2 =
+ {conf,[{name,top2}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub21}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub2xx}],
+ {?M2,init_per_group},
+ [{?M2,tc3}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}},
+ {conf,[{name,sub22}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub2xx}],
+ {?M2,init_per_group},
+ [{?M2,tc3}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}},
+
+ {conf,[{name,sub21}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub2xx}],
+ {?M2,init_per_group},
+ [{?M2,tc3}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}},
+
+ {conf,[{name,sub22}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub2xx}],
+ {?M2,init_per_group},
+ [{?M2,tc3}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}},
+
+ {conf,[{name,sub2xx}],
+ {?M2,init_per_group},
+ [{?M2,tc3}],
+ {?M2,end_per_group}}] = Found,
+
+ {?M2,GPath,TCs,Top2}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_sub_groups10(_) ->
+ GPath = [[sub22,sub2xx]], TCs = tc3,
+
+ Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()),
+
+ [Top2 =
+ {conf,[{name,top2}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub22}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub2xx}],
+ {?M2,init_per_group},
+ [{?M2,tc3}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}},
+
+ {conf,[{name,sub22}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub2xx}],
+ {?M2,init_per_group},
+ [{?M2,tc3}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}] = Found,
+
+ {?M2,GPath,TCs,Top2}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_sub_groups11(_) ->
+ GPath = [[top1,sub12,sub121]], TCs = all,
+
+ Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()),
+
+ [{conf,[{name,top1}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub12}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub121}],
+ {?M2,init_per_group},
+ [{?M2,sub121_tc1},
+ {?M2,sub_tc2},
+ {?M2,tc3}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}] = Found,
+
+ {?M2,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_sub_groups12(_) ->
+ GPath = [[top2,sub2xx]], TCs = [sub2xx_tc1,tc3],
+
+ Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()),
+
+ [{conf,[{name,top2}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub21}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub2xx}],
+ {?M2,init_per_group},
+ [{?M2,sub2xx_tc1},
+ {?M2,tc3}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}},
+ {conf,[{name,sub22}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub2xx}],
+ {?M2,init_per_group},
+ [{?M2,sub2xx_tc1},
+ {?M2,tc3}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}] = Found,
+
+ {?M2,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+testcase_in_sub_groups13(_) ->
+ GPath = [[top2,sub22,sub2xx]], TCs = [top2_tc1,sub2xx_tc1,tc3],
+
+ Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()),
+
+ [{conf,[{name,top2}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub22}],
+ {?M2,init_per_group},
+ [{conf,[{name,sub2xx}],
+ {?M2,init_per_group},
+ [{?M2,sub2xx_tc1},
+ {?M2,tc3}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}],
+ {?M2,end_per_group}}] = Found,
+
+ {?M2,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+bad_testcase_in_sub_groups1(_) ->
+ GPath = [sub2xx], TCs = [top2_tc1],
+
+ Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()),
+
+ [] = Found,
+
+ {?M2,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%%
+bad_testcase_in_sub_groups2(_) ->
+ GPath = [sub12,sub2xx], TCs = [top1_tc1,top2_tc1],
+
+ Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()),
+
+ [] = Found,
+
+ {?M2,GPath,TCs,Found}.
+
+%%%-----------------------------------------------------------------
+%%% CASES EXECUTING THE TESTS
+%%%-----------------------------------------------------------------
+
+run_groups_with_options(Config) ->
+ DataDir = ?config(data_dir, Config),
+
+ {M1All,M1Rest,M2All,M2Rest} = get_all_groups_and_cases(Config),
+
+ M1AllGrs = lists:flatmap(fun({Path,_,_}) when is_atom(hd(Path)) -> Path;
+ ({Path,_,_}) when is_list(hd(Path)) -> Path;
+ ({Path,_,_}) -> [Path]
+ end, M1All),
+
+ %% ct:pal("NOW RUNNING M1 TEST: ~p", [M1All]),
+
+ {OptsM11,ERPidM11} = setup([{dir,DataDir},{suite,?M1},
+ {group,M1AllGrs},{label,m1_all_cases}], Config),
+ M1AllGrInfo = {M1AllGrs,lists:flatten([Found || {_,_,Found} <- M1All])},
+ ok = execute(m1_all_cases, M1AllGrInfo, OptsM11, ERPidM11, Config),
+
+ lists:foldl(
+ fun({GrPath,TCs,Found}, N) ->
+ TestName = list_to_atom("m1_spec_cases_" ++ integer_to_list(N)),
+ %% ct:pal("NOW RUNNING M1 TEST ~p: ~p + ~p",
+ %% [TestName,GrPath,TCs]),
+ {OptsM12,ERPidM12} = setup([{dir,DataDir},{suite,?M1},
+ {group,GrPath},{testcase,TCs},
+ {label,TestName}], Config),
+ ok = execute(TestName, {GrPath,TCs,Found},
+ OptsM12, ERPidM12, Config),
+ N+1
+ end, 1, M1Rest),
+
+ %% ct:pal("NOW RUNNING M2 TEST: ~p", [M2All]),
+
+ M2AllGrs = lists:flatmap(fun({Path,_,_}) when is_atom(hd(Path)) -> Path;
+ ({Path,_,_}) when is_list(hd(Path)) -> Path;
+ ({Path,_,_}) -> [Path]
+ end, M2All),
+
+
+ {OptsM21,ERPidM21} = setup([{dir,DataDir},{suite,?M2},
+ {group,M2AllGrs},{testcase,all},
+ {label,m2_all_cases}], Config),
+ M2AllGrInfo = {M2AllGrs,lists:flatten([Found || {_,_,Found} <- M2All])},
+ ok = execute(m2_all_cases, M2AllGrInfo, OptsM21, ERPidM21, Config),
+
+ lists:foldl(
+ fun({GrPath,TCs,Found}, N) ->
+ TestName = list_to_atom("m2_spec_cases_" ++ integer_to_list(N)),
+ %% ct:pal("NOW RUNNING M2 TEST ~p: ~p + ~p", [TestName,GrPath,TCs]),
+ {OptsM22,ERPidM22} = setup([{dir,DataDir},{suite,?M2},
+ {group,GrPath},{testcase,TCs},
+ {label,TestName}], Config),
+ ok = execute(TestName, {GrPath,TCs,Found},
+ OptsM22, ERPidM22, Config),
+ N+1
+ end, 1, M2Rest),
+ ok.
+
+
+%%%-----------------------------------------------------------------
+%%%
+run_groups_with_testspec(Config) ->
+ Name = run_groups_with_testspec,
+ DataDir = ?config(data_dir, Config),
+ PrivDir = ?config(priv_dir, Config),
+
+ {M1All,M1Rest,M2All,M2Rest} = get_all_groups_and_cases(Config),
+
+ M1AllGrs = lists:flatmap(fun({Path,_,_}) when is_atom(hd(Path)) -> Path;
+ ({Path,_,_}) when is_list(hd(Path)) -> Path;
+ ({Path,_,_}) -> [Path]
+ end, M1All),
+ M1AllTerm = {groups,DataDir,?M1,M1AllGrs},
+
+ M1RestTerms = lists:map(
+ fun({GrPath,TCs,_}) ->
+ {groups,DataDir,?M1,GrPath,{cases,TCs}}
+ end, M1Rest),
+
+ M2AllGrs = lists:flatmap(fun({Path,_,_}) when is_atom(hd(Path)) -> Path;
+ ({Path,_,_}) when is_list(hd(Path)) -> Path;
+ ({Path,_,_}) -> [Path]
+ end, M2All),
+ M2AllTerm = {groups,DataDir,?M2,M2AllGrs,{cases,all}},
+
+ M2RestTerms = lists:map(
+ fun({GrPath,TCs,_}) ->
+ {groups,DataDir,?M2,GrPath,{cases,TCs}}
+ end, M2Rest),
+
+ GroupTerms = lists:flatten([M1AllTerm,
+ M1RestTerms,
+ M2AllTerm,
+ M2RestTerms]),
+
+ TestSpec = [{merge_tests,false},
+ {label,Name}] ++ GroupTerms,
+
+ ct:pal("Here's the test spec:~n~p", [TestSpec]),
+
+ TestSpecName = ct_test_support:write_testspec(TestSpec, PrivDir,
+ "groups_search_spec"),
+
+ {Opts,ERPid} = setup([{spec,TestSpecName}], Config),
+ GroupInfo =
+ [{M1AllTerm,lists:flatten([Found || {_,_,Found} <- M1All])} |
+ M1Rest] ++
+ [{M2AllTerm,lists:flatten([Found || {_,_,Found} <- M2All])} |
+ M2Rest],
+ ok = execute(Name, GroupInfo, Opts, ERPid, Config).
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+
+groups1() ->
+ [{top1,[],[top1_tc1,top1_tc2,{sub1,[],[sub1_tc1,sub1_tc2]}]},
+ {top2,[],[{group,sub2},top2_tc1,top2_tc2]},
+ {sub2,[],[sub2_tc1,sub2_tc2]}].
+
+groups2() ->
+ [{top1,[],[top1_tc1,top_tc2,tc3,
+ {sub11,[],[sub11_tc1,sub_tc2,tc3]},
+ {sub12,[],[sub12_tc1,sub_tc2,tc3,
+ {sub121,[],[sub121_tc1,sub_tc2,tc3]}]}]},
+ {top2,[],[{group,sub21},top2_tc1,top_tc2,tc3,{group,sub22}]},
+ {sub21,[],[sub21_tc1,sub_tc2,tc3,{group,sub2xx}]},
+ {sub22,[],[{group,sub221},sub22_tc1,sub_tc2,tc3,{group,sub2xx}]},
+ {sub221,[],[sub221_tc1,sub_tc2,tc3]},
+ {sub2xx,[],[sub2xx_tc1,sub_tc2,tc3]}].
+
+get_all_groups_and_cases(Config) ->
+ {value,{_,_,FindGrTCs}} = lists:keysearch(find_groups, 1, groups()),
+
+ MGTFs = [apply(?MODULE, TC, [Config]) || TC <- FindGrTCs],
+
+ ct:pal("Extracted data from ~p test cases", [length(MGTFs)]),
+
+ lists:foldr(fun({M,Gs,TCs,F},
+ {M11,M12,M21,M22}) ->
+ case {M,Gs,TCs} of
+ {?M1,all,_} -> {M11,[{Gs,TCs,F}|M12],M21,M22};
+ {?M1,_,all} -> {[{Gs,all,F}|M11],M12,M21,M22};
+ {?M1,_,_} -> {M11,[{Gs,TCs,F}|M12],M21,M22};
+ {?M2,all,_} -> {M11,M12,M21,[{Gs,TCs,F}|M22]};
+ {?M2,_,all} -> {M11,M12,[{Gs,all,F}|M21],M22};
+ {?M2,_,_} -> {M11,M12,M21,[{Gs,TCs,F}|M22]}
+ end
+ end, {[],[],[],[]}, MGTFs).
+
+%%%-----------------------------------------------------------------
+
+setup(Test, Config) ->
+ Opts0 = ct_test_support:get_opts(Config),
+ Level = ?config(trace_level, Config),
+ EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+ Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test],
+ ERPid = ct_test_support:start_event_receiver(Config),
+ {Opts,ERPid}.
+
+execute(Name, TestParams, Opts, ERPid, Config) ->
+ ok = ct_test_support:run(Opts, Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+ Events1 = reformat(Events, ?eh),
+ ct_test_support:log_events(Name,
+ Events1,
+ ?config(priv_dir, Config),
+ Opts),
+ verify_events(Name, TestParams, Events1).
+
+reformat(Events, EH) ->
+ ct_test_support:reformat(Events, EH).
+
+%%%-----------------------------------------------------------------
+%%% TEST EVENTS
+verify_events(Name, Params, Events) ->
+ %% 2 tests (ct:run_test + script_start) is default
+ verify_events(Name, Params, Events, 2).
+
+verify_events(_, _, _, 0) ->
+ ok;
+verify_events(Name, Params, Events, N) ->
+ test_events(Name, Params, Events),
+ verify_events(Name, Params, Events, N-1).
+
+%%%-----------------------------------------------------------------
+%%% check run_groups_with_options
+
+test_events(TestName, {GrPath,Found}, Events) ->
+ test_events(TestName, {GrPath,all,Found}, Events);
+
+test_events(TestName, {GrPath,TCs,Found}, Events)
+ when TestName /= run_groups_with_testspec ->
+ try check_events(Events, flatten_tests(Found)) of
+ ok -> ok
+ catch
+ throw:Reason ->
+ ct:pal("Test failed for ~p with group path ~p and cases ~p"
+ "~nReason: ~p", [TestName,GrPath,TCs,Reason]),
+ throw(failed)
+ end;
+
+%%%-----------------------------------------------------------------
+%%% check run_groups_with_testspec
+
+test_events(run_groups_with_testspec, Params, Events) ->
+ AllFound = lists:flatmap(fun({_All,Found}) when is_tuple(Found) ->
+ [Found];
+ ({_All,Found}) ->
+ Found;
+ ({_Gr,_TCs,Found}) when is_tuple(Found) ->
+ [Found];
+ ({_Gr,_TCs,Found}) ->
+ Found
+ end, Params),
+ try check_events(Events, flatten_tests(AllFound)) of
+ ok -> ok
+ catch
+ throw:Reason ->
+ ct:pal("Test failed for run_groups_with_testspec."
+ "~nReason: ~p", [Reason]),
+ throw(failed)
+ end.
+
+flatten_tests({conf,[{name,G}|_],{Mod,_I},Tests,_E}) ->
+ lists:flatten([{group,Mod,G} | flatten_tests(Tests)]);
+flatten_tests([{conf,[{name,G}|_],{Mod,_I},Tests,_E} | Confs]) ->
+ lists:flatten([{group,Mod,G} | flatten_tests(Tests)]) ++
+ lists:flatten(flatten_tests(Confs));
+flatten_tests([{_Mod,_TC} = Case | Tests]) ->
+ lists:flatten([Case | flatten_tests(Tests)]);
+flatten_tests([]) ->
+ [].
+
+check_events([{_,tc_start,{Mod,{init_per_group,G,_}}} | Evs],
+ [{group,Mod,G} | Check]) ->
+ check_events(Evs, Check);
+check_events([{_,tc_start,{Mod,TC}} | Evs],
+ [{Mod,TC} | Check]) when is_atom(TC) ->
+ check_events(Evs, Check);
+check_events([{_,tc_start,{Mod,{init_per_group,G,_}}} | _Evs], Check) ->
+ ct:pal("CHECK FAILED!~nGroup ~p in ~p not found in ~p.",
+ [G,Mod,Check]),
+ throw({test_not_found,{Mod,G}});
+check_events([{_,tc_start,{Mod,TC}} | _Evs], Check)
+ when is_atom(TC), TC /= init_per_suite, TC /= end_per_suite ->
+ ct:pal("CHECK FAILED!~nCase ~p in ~p not found in ~p.",
+ [TC,Mod,Check]),
+ throw({test_not_found,{Mod,TC}});
+check_events([Group | Evs], Check) when is_list(Group) ->
+ Check1 = check_events(Group, Check),
+ check_events(Evs, Check1);
+check_events(_, []) ->
+ ok;
+check_events([Elem | Evs], Check) when is_tuple(Elem) ->
+ check_events(Evs, Check);
+check_events([], Check = [_|_]) ->
+ ct:pal("CHECK FAILED!~nTests remain: ~p", [Check]),
+ throw({tests_remain,Check});
+check_events([Wut | _],_) ->
+ throw({unexpected,Wut}).
+
diff --git a/lib/common_test/test/ct_groups_search_SUITE_data/groups_search_dummy_1_SUITE.erl b/lib/common_test/test/ct_groups_search_SUITE_data/groups_search_dummy_1_SUITE.erl
new file mode 100644
index 0000000000..6f6922e686
--- /dev/null
+++ b/lib/common_test/test/ct_groups_search_SUITE_data/groups_search_dummy_1_SUITE.erl
@@ -0,0 +1,83 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(groups_search_dummy_1_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+
+all() ->
+ [{group,top1},
+ {group,top2}].
+
+groups() ->
+ [{top1,[],[top1_tc1,top1_tc2,{sub1,[],[sub1_tc1,sub1_tc2]}]},
+ {top2,[],[{group,sub2},top2_tc1,top2_tc2]},
+ {sub2,[],[sub2_tc1,sub2_tc2]}].
+
+%%%-----------------------------------------------------------------
+%%% CONFIG FUNCS
+%%%-----------------------------------------------------------------
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+init_per_group(_, Config) ->
+ Config.
+
+end_per_group(_, _Config) ->
+ ok.
+
+%%%-----------------------------------------------------------------
+%%% TEST CASES
+%%%-----------------------------------------------------------------
+
+top1_tc1(_) ->
+ ok.
+
+top1_tc2(_) ->
+ ok.
+
+sub1_tc1(_) ->
+ ok.
+
+sub1_tc2(_) ->
+ ok.
+
+top2_tc1(_) ->
+ ok.
+
+top2_tc2(_) ->
+ ok.
+
+sub2_tc1(_) ->
+ ok.
+
+sub2_tc2(_) ->
+ ok.
diff --git a/lib/common_test/test/ct_groups_search_SUITE_data/groups_search_dummy_2_SUITE.erl b/lib/common_test/test/ct_groups_search_SUITE_data/groups_search_dummy_2_SUITE.erl
new file mode 100644
index 0000000000..ac3c000079
--- /dev/null
+++ b/lib/common_test/test/ct_groups_search_SUITE_data/groups_search_dummy_2_SUITE.erl
@@ -0,0 +1,102 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(groups_search_dummy_2_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+
+all() ->
+ [{group,top1},
+ {group,top2}].
+
+groups() ->
+ [{top1,[],[top1_tc1,top_tc2,tc3,
+ {sub11,[],[sub11_tc1,sub_tc2,tc3]},
+ {sub12,[],[sub12_tc1,sub_tc2,tc3,
+ {sub121,[],[sub121_tc1,sub_tc2,tc3]}]}]},
+
+ {top2,[],[{group,sub21},top2_tc1,top_tc2,tc3,{group,sub22}]},
+ {sub21,[],[sub21_tc1,sub_tc2,tc3,{group,sub2xx}]},
+ {sub22,[],[{group,sub221},sub22_tc1,sub_tc2,tc3,{group,sub2xx}]},
+ {sub221,[],[sub221_tc1,sub_tc2,tc3]},
+ {sub2xx,[],[sub2xx_tc1,sub_tc2,tc3]}].
+
+%%%-----------------------------------------------------------------
+%%% CONFIG FUNCS
+%%%-----------------------------------------------------------------
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+init_per_group(_, Config) ->
+ Config.
+
+end_per_group(_, _Config) ->
+ ok.
+
+%%%------------------------------------------------------------------
+%%% TEST CASES
+%%%------------------------------------------------------------------
+
+top1_tc1(_) ->
+ ok.
+
+top_tc2(_) ->
+ ok.
+
+tc3(_) ->
+ ok.
+
+sub_tc2(_) ->
+ ok.
+
+sub11_tc1(_) ->
+ ok.
+
+sub12_tc1(_) ->
+ ok.
+
+sub121_tc1(_) ->
+ ok.
+
+top2_tc1(_) ->
+ ok.
+
+sub21_tc1(_) ->
+ ok.
+
+sub22_tc1(_) ->
+ ok.
+
+sub221_tc1(_) ->
+ ok.
+
+sub2xx_tc1(_) ->
+ ok.