Fix problem with suites and groups skipped from test specification

3 files changed, 221 insertions, 18 deletions

diff --git a/lib/common_test/test/ct_skip_SUITE.erl b/lib/common_test/test/ct_skip_SUITE.erl
index a67ad3f374..10e834ff2a 100644
--- a/lib/common_test/test/ct_skip_SUITE.erl
+++ b/lib/common_test/test/ct_skip_SUITE.erl
@@ -59,7 +59,7 @@ end_per_testcase(TestCase, Config) ->
 suite() -> [{ct_hooks,[ts_install_cth]}].
 
 all() -> 
-    [auto_skip, user_skip].
+    [auto_skip, user_skip, testspec_skip].
 
 groups() -> 
     [].
@@ -133,8 +133,48 @@ user_skip(Config) when is_list(Config) ->
     ok = ct_test_support:verify_events(TestEvents, Events, Config).
 
 %%%-----------------------------------------------------------------
+%%% 
+testspec_skip(Config) when is_list(Config) ->
+    TestDir = filename:join(?config(data_dir, Config),
+			    filename:join("skip", "test")),
+    TestSpec1 = [{suites, TestDir, user_skip_7_SUITE},
+		 {skip_cases, TestDir, user_skip_7_SUITE, [tc1,tc3], "SKIPPED"}],
+
+    TestSpec2 = [{suites, TestDir, user_skip_7_SUITE},
+		 {skip_groups, TestDir, user_skip_7_SUITE, ptop1, "SKIPPED"}],
+
+    TestSpec3 = [{suites, TestDir, user_skip_7_SUITE},
+		 {skip_groups, TestDir, user_skip_7_SUITE, psub1, "SKIPPED"}],
+
+    {Opts,ERPid} = setup_testspec([{ts1,TestSpec1},
+				   {ts2,TestSpec2},
+				   {ts3,TestSpec3}], Config),
+
+    ok = ct_test_support:run(Opts, Config),
+
+    Events = ct_test_support:get_events(ERPid, Config),
+
+    ct_test_support:log_events(testspec_skip, 
+			       reformat(Events, ?eh),
+			       ?config(priv_dir, Config),
+			       Opts),
+
+    TestEvents = events_to_check(testspec_skip),
+    ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
+%%%-----------------------------------------------------------------
 %%% HELP FUNCTIONS
 %%%-----------------------------------------------------------------
+setup_testspec(TestSpecs, Config) ->
+    SpecFiles =
+	[begin SpecFile = filename:join(?config(priv_dir, Config),
+					atom_to_list(SpecName)++".spec"),
+	       {ok,Dev} = file:open(SpecFile, [write]),
+	       [io:format(Dev, "~p.~n", [Term]) || Term <- TestSpec],
+	       file:close(Dev),
+	       SpecFile
+	 end || {SpecName,TestSpec} <- TestSpecs],
+    setup({spec,SpecFiles}, Config).
 
 setup(Test, Config) ->
     Opts0 = ct_test_support:get_opts(Config),
@@ -678,4 +718,7 @@ test_events(user_skip) ->
        
        {?eh,test_done,{'DEF','STOP_TIME'}},
        {?eh,stop_logging,[]}
-      ].
+    ];
+
+test_events(testspec_skip) ->
+    [].
diff --git a/lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_7_SUITE.erl b/lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_7_SUITE.erl
new file mode 100644
index 0000000000..82ce536a79
--- /dev/null
+++ b/lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_7_SUITE.erl
@@ -0,0 +1,113 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(user_skip_7_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% Function: suite() -> Info
+%% Info = [tuple()]
+%%--------------------------------------------------------------------
+suite() ->
+    [{timetrap,{seconds,30}}].
+
+%%--------------------------------------------------------------------
+%% Function: init_per_group(GroupName, Config0) ->
+%%               Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_group(_GroupName, Config) ->
+    Config.
+
+%%--------------------------------------------------------------------
+%% Function: end_per_group(GroupName, Config0) ->
+%%               void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+    ok.
+
+%%--------------------------------------------------------------------
+%% Function: init_per_testcase(TestCase, Config0) ->
+%%               Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_testcase(_TestCase, Config) ->
+    Config.
+
+%%--------------------------------------------------------------------
+%% Function: end_per_testcase(TestCase, Config0) ->
+%%               void() | {save_config,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+    ok.
+
+%%--------------------------------------------------------------------
+%% Function: groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%%              repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%%--------------------------------------------------------------------
+groups() ->
+    [{ptop1,[parallel],[tc1,{psub1,[parallel],[tc3,tc4]},tc2]}].
+
+%%--------------------------------------------------------------------
+%% Function: all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%%--------------------------------------------------------------------
+all() -> 
+    [{group,ptop1}].
+
+%%--------------------------------------------------------------------
+%% Function: TestCase(Config0) ->
+%%               ok | exit() | {skip,Reason} | {comment,Comment} |
+%%               {save_config,Config1} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% Comment = term()
+%%--------------------------------------------------------------------
+tc1(_) ->
+    ok.
+
+tc2(_) ->
+    ok.
+
+tc3(_) ->
+    ok.
+
+tc4(_) ->
+    ok.
diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl
index da8deccb45..2041ee294e 100644
--- a/lib/test_server/src/test_server_ctrl.erl
+++ b/lib/test_server/src/test_server_ctrl.erl
@@ -1480,6 +1480,11 @@ do_test_cases(TopCases, SkipCases,
 	    put(test_server_case_num, 0),
 	    TestSpec =
 		add_init_and_end_per_suite(TestSpec0, undefined, undefined, FwMod),
+
+
+	    %%! --- Fri Nov  8 14:50:39 2013 --- peppe was here!
+	    io:format(user, ">>> HERE'S THE TS: ~p~n~n", [TestSpec]),
+
 	    TI = get_target_info(),
 	    print(1, "Starting test~ts",
 		  [print_if_known(N, {", ~w test cases",[N]},
@@ -1795,23 +1800,40 @@ downcase([], Result) ->
 %%
 %%  Errors are silently ignored.
 
-html_convert_modules(TestSpec, _Config) ->
-    Mods = html_isolate_modules(TestSpec),
+html_convert_modules(TestSpec, _Config, FwMod) ->
+    Mods = html_isolate_modules(TestSpec, FwMod),
     html_convert_modules(Mods),
     copy_html_files(get(test_server_dir), get(test_server_log_dir_base)).
 
 %% Retrieve a list of modules out of the test spec.
-html_isolate_modules(List) -> html_isolate_modules(List, sets:new()).
-
-html_isolate_modules([], Set) -> sets:to_list(Set);
-html_isolate_modules([{skip_case,_}|Cases], Set) ->
-    html_isolate_modules(Cases, Set);
-html_isolate_modules([{conf,_Ref,_Props,{Mod,_Func}}|Cases], Set) ->
-    html_isolate_modules(Cases, sets:add_element(Mod, Set));
-html_isolate_modules([{Mod,_Case}|Cases], Set) ->
-    html_isolate_modules(Cases, sets:add_element(Mod, Set));
-html_isolate_modules([{Mod,_Case,_Args}|Cases], Set) ->
-    html_isolate_modules(Cases, sets:add_element(Mod, Set)).
+html_isolate_modules(List, FwMod) ->
+    html_isolate_modules(List, sets:new(), FwMod).
+
+html_isolate_modules([], Set, _) -> sets:to_list(Set);
+html_isolate_modules([{skip_case,_}|Cases], Set, FwMod) ->
+    html_isolate_modules(Cases, Set, FwMod);
+html_isolate_modules([{conf,_Ref,Props,{FwMod,_Func}}|Cases], Set, FwMod) ->
+    Set1 = case proplists:get_value(suite, Props) of
+	       undefined -> Set;
+	       Mod -> sets:add_element(Mod, Set)
+	   end,
+    html_isolate_modules(Cases, Set1, FwMod);
+html_isolate_modules([{conf,_Ref,_Props,{Mod,_Func}}|Cases], Set, FwMod) ->
+    html_isolate_modules(Cases, sets:add_element(Mod, Set), FwMod);
+html_isolate_modules([{skip_case,{conf,_Ref,{FwMod,_Func},_Cmt},Mode}|Cases],
+		     Set, FwMod) ->
+    Set1 = case proplists:get_value(suite, get_props(Mode)) of
+	       undefined -> Set;
+	       Mod -> sets:add_element(Mod, Set)
+	   end,
+    html_isolate_modules(Cases, Set1, FwMod);
+html_isolate_modules([{skip_case,{conf,_Ref,{Mod,_Func},_Cmt},_Props}|Cases],
+		     Set, FwMod) ->
+    html_isolate_modules(Cases, sets:add_element(Mod, Set), FwMod);
+html_isolate_modules([{Mod,_Case}|Cases], Set, FwMod) ->
+    html_isolate_modules(Cases, sets:add_element(Mod, Set), FwMod);
+html_isolate_modules([{Mod,_Case,_Args}|Cases], Set, FwMod) ->
+    html_isolate_modules(Cases, sets:add_element(Mod, Set), FwMod).
 
 %% Given a list of modules, convert each module's source code to HTML.
 html_convert_modules([Mod|Mods]) ->
@@ -1902,6 +1924,11 @@ add_init_and_end_per_suite([{skip_case,{{Mod,_},_}}=Case|Cases], LastMod,
     {PreCases, NextMod, NextRef} =
 	do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod),
     PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)];
+add_init_and_end_per_suite([{skip_case,{conf,_,{Mod,_},_},_}=Case|Cases], LastMod,
+			   LastRef, FwMod) when Mod =/= LastMod ->
+    {PreCases, NextMod, NextRef} =
+	do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod),
+    PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)];
 add_init_and_end_per_suite([{skip_case,{conf,_,{Mod,_},_}}=Case|Cases], LastMod,
 			   LastRef, FwMod) when Mod =/= LastMod ->
     {PreCases, NextMod, NextRef} =
@@ -2044,7 +2071,8 @@ run_test_cases(TestSpec, Config, TimetrapData) ->
 	true ->
 	    ok;
 	false ->
-	    html_convert_modules(TestSpec, Config)
+	    FwMod = get_fw_mod(?MODULE),
+	    html_convert_modules(TestSpec, Config, FwMod)
     end,
 
     run_test_cases_loop(TestSpec, [Config], TimetrapData, [], []),
@@ -4460,12 +4488,31 @@ collect_cases({conf,Props,InitMF,CaseList,FinMF} = Conf, St) ->
 	Props1 ->
 	    Ref = make_ref(),
 	    Skips = St#cc.skip,
+	    Props2 = [{suite,St#cc.mod} | lists:delete(suite,Props1)],
+	    Mode = [{Ref,Props2,undefined}],
 	    case in_skip_list({St#cc.mod,Conf}, Skips) of
 		{true,Comment} ->	    	           % conf init skipped
-		    {ok,[{skip_case,{conf,Ref,InitMF,Comment}} |
+		    {ok,[{skip_case,{conf,Ref,InitMF,Comment},Mode} |
 			 [] ++ [{conf,Ref,[],FinMF}]],St};
 		{true,Name,Comment} when is_atom(Name) ->  % all cases skipped
-		    {ok,[{skip_case,{{St#cc.mod,{group,Name}},Comment}}],St};
+		    case collect_cases(CaseList, St) of
+			{ok,[],_St} = Empty ->
+			    Empty;
+			{ok,FlatCases,St1} ->
+			    Cases2Skip = FlatCases ++ [{conf,Ref,
+							keep_name(Props1),
+							FinMF}],
+			    Skipped = skip_cases_upto(Ref, Cases2Skip, Comment,
+						      conf, Mode, skip_case),
+
+			    %%! --- Fri Nov  8 16:47:50 2013 --- peppe was here!
+			    io:format(user, "!!! FLAT2 = ~p~n", [Skipped]),
+
+			    {ok,[{skip_case,{conf,Ref,InitMF,Comment},Mode} |
+				 Skipped],St1};
+			{error,_Reason} = Error ->
+			    Error
+		    end;
 		{true,ToSkip,_} when is_list(ToSkip) ->    % some cases skipped
 		    case collect_cases(CaseList,
 				       St#cc{skip=ToSkip++Skips}) of