Correct various bugs related to auto_skip and groups

5 files changed, 213 insertions, 49 deletions

diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index ec8f2dac09..b72ecba809 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -73,7 +73,7 @@ init_tc(Mod,Func,Config) ->
 	_ ->
 	    case ct_util:get_testdata(curr_tc) of
 		{Suite,{suite0_failed,{require,Reason}}} ->
-		    {fail,{require_failed_in_suite0,Reason}};
+		    {auto_skip,{require_failed_in_suite0,Reason}};
 		{Suite,{suite0_failed,_}=Failure} ->
 		    {fail,Failure};
 		_ ->
@@ -222,9 +222,9 @@ init_tc2(Mod,Suite,Func,SuiteInfo,MergeResult,Config) ->
 	{suite0_failed,Reason} ->
 	    ct_util:set_testdata({curr_tc,{Mod,{suite0_failed,
 						{require,Reason}}}}),
-	    {fail,{require_failed_in_suite0,Reason}};
+	    {auto_skip,{require_failed_in_suite0,Reason}};
 	{error,Reason} ->
-	    {fail,{require_failed,Reason}};
+	    {auto_skip,{require_failed,Reason}};
 	{'EXIT',Reason} ->
 	    {fail,Reason};
 	{ok,PostInitHook,Config1} ->
@@ -621,31 +621,34 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) ->
 		   _ -> Func
 	       end,
 
-    case get('$test_server_framework_test') of
-	undefined ->
-	    {FinalResult,FinalNotify} =
-		case ct_hooks:end_tc(
-		       Suite, FuncSpec, Args, Result, Return) of
-		    '$ct_no_change' ->
-			{ok,Result};
-		    FinalResult1 ->
-			{FinalResult1,FinalResult1}
-		end,
-	    %% send sync notification so that event handlers may print
-	    %% in the log file before it gets closed
-	    ct_event:sync_notify(#event{name=tc_done,
-					node=node(),
-					data={Mod,FuncSpec,
-					      tag_cth(FinalNotify)}});
-	Fun ->
-	    %% send sync notification so that event handlers may print
-	    %% in the log file before it gets closed
-	    ct_event:sync_notify(#event{name=tc_done,
-					node=node(),
-					data={Mod,FuncSpec,tag(Result)}}),
-	    FinalResult = Fun(end_tc, Return)
-    end,    
-    
+    {Result1,FinalNotify} =
+	case ct_hooks:end_tc(
+	       Suite, FuncSpec, Args, Result, Return) of
+	    '$ct_no_change' ->
+		{ok,Result};
+	    HookResult ->
+		{HookResult,HookResult}
+	end,
+    FinalResult =
+	case get('$test_server_framework_test') of
+	    undefined ->
+		%% send sync notification so that event handlers may print
+		%% in the log file before it gets closed
+		ct_event:sync_notify(#event{name=tc_done,
+					    node=node(),
+					    data={Mod,FuncSpec,
+						  tag_cth(FinalNotify)}}),
+		Result1;
+	    Fun ->
+		%% send sync notification so that event handlers may print
+		%% in the log file before it gets closed
+		ct_event:sync_notify(#event{name=tc_done,
+					    node=node(),
+					    data={Mod,FuncSpec,
+						  tag(FinalNotify)}}),
+		Fun(end_tc, Return)
+	end,    
+
     case FuncSpec of
 	{_,GroupName,_Props} ->
 	    if Func == end_per_group ->
@@ -685,7 +688,7 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) ->
 		     (Unexpected) ->
 			  exit({error,{reset_curr_tc,{Mod,Func},Unexpected}})
 		  end,
-    ct_util:update_testdata(curr_tc,ClearCurrTC),
+    ct_util:update_testdata(curr_tc, ClearCurrTC),
 
     case FinalResult of
 	{skip,{sequence_failed,_,_}} ->
@@ -1229,6 +1232,8 @@ report(What,Data) ->
 		    ct_hooks:on_tc_skip(tc_auto_skip, Data);
 		{skipped,_} ->
 		    ct_hooks:on_tc_skip(tc_user_skip, Data);
+		{auto_skipped,_} ->
+		    ct_hooks:on_tc_skip(tc_auto_skip, Data);
 		_Else ->
 		    ok
 	    end,
@@ -1253,6 +1258,8 @@ report(What,Data) ->
 		    add_to_stats(auto_skipped);
 		{_,{skipped,_}} ->
 		    add_to_stats(user_skipped);
+		{_,{auto_skipped,_}} ->
+		    add_to_stats(auto_skipped);
 		{_,{SkipOrFail,_Reason}} ->
 		    add_to_stats(SkipOrFail)
 	    end;
diff --git a/lib/common_test/test/ct_skip_SUITE.erl b/lib/common_test/test/ct_skip_SUITE.erl
index 35ac483309..6f6ff754aa 100644
--- a/lib/common_test/test/ct_skip_SUITE.erl
+++ b/lib/common_test/test/ct_skip_SUITE.erl
@@ -91,7 +91,8 @@ auto_skip(Config) when is_list(Config) ->
 	      Join(DataDir, "auto_skip_8_SUITE"),
 	      Join(DataDir, "auto_skip_9_SUITE"),
 	      Join(DataDir, "auto_skip_10_SUITE"),
-	      Join(DataDir, "auto_skip_11_SUITE")
+	      Join(DataDir, "auto_skip_11_SUITE"),
+	      Join(DataDir, "auto_skip_12_SUITE")
 	     ],
 
     {Opts,ERPid} = setup({suite,Suites}, Config),
@@ -383,9 +384,9 @@ test_events(auto_skip) ->
 
      {?eh,tc_start,{auto_skip_10_SUITE,init_per_suite}},
      {?eh,tc_done,{auto_skip_10_SUITE,init_per_suite,
-		   {skipped,
-		    {require_failed_in_suite0,
-		     {not_available,undefined_config_variable}}}}},
+		   {failed,{error,
+			    {require_failed_in_suite0,
+			     {not_available,undefined_config_variable}}}}}},
      {?eh,tc_auto_skip,
       {auto_skip_10_SUITE,tc1,
        {require_failed_in_suite0,{not_available,undefined_config_variable}}}},
@@ -435,6 +436,8 @@ test_events(auto_skip) ->
      {?eh,tc_start,{auto_skip_11_SUITE,end_per_suite}},
      {?eh,tc_done,{auto_skip_11_SUITE,end_per_suite,ok}},
 
+     %%! HERE: auto_skip_12_SUITE terms!!
+
      {?eh,test_done,{'DEF','STOP_TIME'}},
      {?eh,stop_logging,[]}
     ];
diff --git a/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_12_SUITE.erl b/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_12_SUITE.erl
new file mode 100644
index 0000000000..a168afd386
--- /dev/null
+++ b/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_12_SUITE.erl
@@ -0,0 +1,121 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(auto_skip_12_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% Function: suite() -> Info
+%% Info = [tuple()]
+%%--------------------------------------------------------------------
+suite() ->
+    [].
+
+%%--------------------------------------------------------------------
+%% Function: init_per_suite(Config0) ->
+%%               Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+    Config.
+
+%%--------------------------------------------------------------------
+%% Function: end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+    ok.
+
+%%--------------------------------------------------------------------
+%% Function: group(Name) -> Info
+%% Info = [tuple()]
+%%--------------------------------------------------------------------
+group(g1) ->
+    [{require,unknown_variable_g1}];
+group(g4) ->
+    [{require,unknown_variable_g4}];
+group(_) ->
+    [].
+
+%%--------------------------------------------------------------------
+%% Function: init_per_testcase(TestCase, Config0) ->
+%%               Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_testcase(_TestCase, Config) ->
+    Config.
+
+%%--------------------------------------------------------------------
+%% Function: end_per_testcase(TestCase, Config0) ->
+%%               void() | {save_config,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+    ok.
+
+%%--------------------------------------------------------------------
+%% Function: groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%%              repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%%--------------------------------------------------------------------
+groups() ->
+    [{g1,[],[tc1,tc2,{g2,[],[tc3]}]},
+     {g1,[],[tc1,tc2,{g2,[],[tc3]}]},
+     {g3,[],[tc1,tc2,{g4,[],[tc3]}]}].
+
+%%--------------------------------------------------------------------
+%% Function: all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%%--------------------------------------------------------------------
+all() -> 
+    [{group,g1},
+     {group,g3}].
+
+%%--------------------------------------------------------------------
+%% Function: TestCase(Config0) ->
+%%               ok | exit() | {skip,Reason} | {comment,Comment} |
+%%               {save_config,Config1} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% Comment = term()
+%%--------------------------------------------------------------------
+tc1(_Config) -> 
+    ok.
+
+tc2(_Config) -> 
+    ok.
+
+tc3(_Config) ->
+    ok.
diff --git a/lib/test_server/src/test_server.erl b/lib/test_server/src/test_server.erl
index 6ddb2b615f..b2cfe5b440 100644
--- a/lib/test_server/src/test_server.erl
+++ b/lib/test_server/src/test_server.erl
@@ -915,6 +915,7 @@ run_test_case_eval(Mod, Func, Args0, Name, Ref, RunInit,
     put(test_server_logopts, LogOpts),
     Where = [{Mod,Func}],
     put(test_server_loc, Where),
+
     FWInitResult = test_server_sup:framework_call(init_tc,[Mod,Func,Args0],
 						  {ok,Args0}),
     set_tc_state(running),
@@ -935,9 +936,9 @@ run_test_case_eval(Mod, Func, Args0, Name, Ref, RunInit,
 	    Skip = {skip,_Reason} ->
 		NewResult = do_end_tc_call(Mod,Func, {Skip,Args0}, Skip),
 		{{0,NewResult},Where,[]};
-	    {auto_skip,Reason} ->
-		NewResult = do_end_tc_call(Mod,Func, {{skip,Reason},Args0},
-					   {skip,Reason}),
+	    AutoSkip = {auto_skip,_Reason} ->
+		%% special case where a conf case "pretends" to be skipped
+		NewResult = do_end_tc_call(Mod,Func, {AutoSkip,Args0}, AutoSkip),
 		{{0,NewResult},Where,[]}
 	end,
     exit({Ref,Time,Value,Loc,Opts}).
@@ -955,7 +956,8 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) ->
 		    {{0,NewRes},Line,[]};
 		{skip_and_save,Reason,SaveCfg} ->
 		    Line = get_loc(),
-		    Conf = [{tc_status,{skipped,Reason}},{save_config,SaveCfg}|hd(Args)],
+		    Conf = [{tc_status,{skipped,Reason}},
+			    {save_config,SaveCfg}|hd(Args)],
 		    NewRes = do_end_tc_call(Mod,Func, {{skip,Reason},[Conf]},
 					    {skip,Reason}),
 		    {{0,NewRes},Line,[]};
diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl
index ac09e25579..bef967ff12 100644
--- a/lib/test_server/src/test_server_ctrl.erl
+++ b/lib/test_server/src/test_server_ctrl.erl
@@ -1918,7 +1918,8 @@ add_init_and_end_per_suite([{conf,Ref,Props,{FwMod,Func}}=Case|Cases], LastMod,
 	Suite when Suite =/= undefined, Suite =/= LastMod ->
 	    {PreCases, NextMod, NextRef} =
 		do_add_init_and_end_per_suite(LastMod, LastRef, Suite, FwMod),
-	    Case1 = {conf,Ref,proplists:delete(suite,Props),{FwMod,Func}},
+	    Case1 = {conf,Ref,[{suite,NextMod}|proplists:delete(suite,Props)],
+		     {FwMod,Func}},
 	    PreCases ++ [Case1|add_init_and_end_per_suite(Cases, NextMod,
 							  NextRef, FwMod)];
 	_ ->
@@ -2654,6 +2655,27 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0,
 	    stop_minor_log_file(),
 	    run_test_cases_loop(Cases2, Config1, TimetrapData, Mode, Status3);
 
+	{_,{auto_skip,SkipReason},_} ->
+	    %% this case can only happen if the framework (not the user)
+	    %% decides to skip execution of a conf function
+	    {Cases2,Config1,Status3} =
+		if StartConf ->
+			ReportAbortRepeat(auto_skipped),
+			print(minor, "~n*** ~w auto skipped.~n"
+			      "    Skipping all cases.", [Func]),
+			{skip_cases_upto(Ref, Cases, SkipReason, conf, CurrMode,
+					 auto_skip_case),
+			 Config,
+			 delete_status(Ref, Status2)};
+		   not StartConf ->
+			ReportRepeatStop(),
+			print_conf_time(ConfTime),
+			{Cases,tl(Config),delete_status(Ref, Status2)}
+		end,
+	    set_io_buffering(IOHandler),
+	    stop_minor_log_file(),
+	    run_test_cases_loop(Cases2, Config1, TimetrapData, Mode, Status3);
+
 	{_,{Skip,Reason},_} when StartConf and ((Skip==skip) or (Skip==skipped)) ->
 	    ReportAbortRepeat(skipped),
 	    print(minor, "~n*** ~w skipped.~n"
@@ -3615,7 +3637,8 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit,
 	    {died,Reason} ->
 		progress(failed, Num, Mod, Func, Loc, Reason,
 			 Time, Comment, Style);
-	    {_,{'EXIT',{Skip,Reason}}} when Skip==skip; Skip==skipped ->
+	    {_,{'EXIT',{Skip,Reason}}} when Skip==skip; Skip==skipped;
+					    Skip==auto_skip ->
 		progress(skip, Num, Mod, Func, Loc, Reason,
 			 Time, Comment, Style);
 	    {_,{'EXIT',_Pid,{Skip,Reason}}} when Skip==skip; Skip==skipped ->
@@ -3627,10 +3650,13 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit,
 	    {_,{'EXIT',Reason}} ->
 		progress(failed, Num, Mod, Func, Loc, Reason,
 			 Time, Comment, Style);
-	    {_, {Fail, Reason}} when Fail =:= fail; Fail =:= failed ->
+	    {_,{Fail,Reason}} when Fail =:= fail; Fail =:= failed ->
 		progress(failed, Num, Mod, Func, Loc, Reason,
 			 Time, Comment, Style);
-	    {_, {Skip, Reason}} when Skip==skip; Skip==skipped ->
+	    {_,Reason={auto_skip,_Why}} ->
+		progress(skip, Num, Mod, Func, Loc, Reason,
+			 Time, Comment, Style);		
+	    {_,{Skip,Reason}} when Skip==skip; Skip==skipped ->
 		progress(skip, Num, Mod, Func, Loc, Reason,
 			 Time, Comment, Style);
 	    {Time,RetVal} ->
@@ -3747,15 +3773,15 @@ num2str(N) -> integer_to_list(N).
 
 progress(skip, CaseNum, Mod, Func, Loc, Reason, Time,
 	 Comment, {St0,St1}) ->
-    {Reason1,{Color,Ret}} = 
+    {Reason1,{Color,Ret,ReportTag}} = 
 	if_auto_skip(Reason,
-		     fun() -> {?auto_skip_color,auto_skip} end,
-		     fun() -> {?user_skip_color,skip} end),
-    print(major, "=result        skipped", []),
+		     fun() -> {?auto_skip_color,auto_skip,auto_skipped} end,
+		     fun() -> {?user_skip_color,skip,skipped} end),
+    print(major, "=result        ~w: ~p", [ReportTag,Reason1]),
     print(1, "*** SKIPPED ~ts ***",
 	  [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]),
     test_server_sup:framework_call(report, [tc_done,{Mod,Func,
-						     {skipped,Reason1}}]),
+						     {ReportTag,Reason1}}]),
     ReasonStr = reason_to_string(Reason1),
     ReasonStr1 = lists:flatten([string:strip(S,left) ||
 				S <- string:tokens(ReasonStr,[$\n])]),
@@ -3968,13 +3994,18 @@ fw_name(Mod) ->
 
 if_auto_skip(Reason={failed,{_,init_per_testcase,_}}, True, _False) ->
     {Reason,True()};
-if_auto_skip({_T,{skip,Reason={failed,{_,init_per_testcase,_}}},_Opts}, True, _False) ->
+if_auto_skip({skip,Reason={failed,{_,init_per_testcase,_}}}, True, _False) ->
+    {Reason,True()};
+if_auto_skip({auto_skip,Reason}, True, _False) ->
     {Reason,True()};
 if_auto_skip(Reason, _True, False) ->
     {Reason,False()}.
 
-update_skip_counters(RetVal, {US,AS}) ->
-    {_,Result} = if_auto_skip(RetVal, fun() -> {US,AS+1} end, fun() -> {US+1,AS} end),
+update_skip_counters({_T,Pat,_Opts}, {US,AS}) ->
+    {_,Result} = if_auto_skip(Pat, fun() -> {US,AS+1} end, fun() -> {US+1,AS} end),
+    Result;    
+update_skip_counters(Pat, {US,AS}) ->
+    {_,Result} = if_auto_skip(Pat, fun() -> {US,AS+1} end, fun() -> {US+1,AS} end),
     Result.
 
 get_info_str(Mod,Func, 0, _Cases) ->