Implement the group info function

OTP-9235

7 files changed, 264 insertions, 109 deletions

diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl
index d72b8bc0e1..6fd7693185 100644
--- a/lib/common_test/src/ct.erl
+++ b/lib/common_test/src/ct.erl
@@ -65,7 +65,7 @@
 	 pal/1, pal/2, pal/3,
 	 fail/1, fail/2, comment/1, comment/2,
 	 testcases/2, userdata/2, userdata/3,
-	 timetrap/1, sleep/1]).
+	 timetrap/1, get_timetrap_info/0, sleep/1]).
 
 %% New API for manipulating with config handlers
 -export([add_config/2, remove_config/2]).
@@ -703,7 +703,8 @@ userdata(TestDir, Suite) ->
 	    get_userdata(Info, "suite/0")
     end.
 
-get_userdata({'EXIT',{undef,_}}, Spec) ->
+get_userdata({'EXIT',{Undef,_}}, Spec) when Undef == undef;
+					     Undef == function_clause ->
     {error,list_to_atom(Spec ++ " is not defined")};
 get_userdata({'EXIT',Reason}, Spec) ->
     {error,{list_to_atom("error in " ++ Spec),Reason}};
@@ -719,16 +720,27 @@ get_userdata(_BadTerm, Spec) ->
     {error,list_to_atom(Spec ++ " must return a list")}.
    
 %%%-----------------------------------------------------------------
-%%% @spec userdata(TestDir, Suite, Case) -> TCUserData | {error,Reason}
+%%% @spec userdata(TestDir, Suite, GroupOrCase) -> TCUserData | {error,Reason}
 %%%       TestDir = string()
 %%%       Suite = atom()
-%%%       Case = atom()
+%%%       GroupOrCase = {group,GroupName} | atom()
+%%%       GroupName = atom()
 %%%       TCUserData = [term()]
 %%%       Reason = term()
 %%%
 %%% @doc Returns any data specified with the tag <code>userdata</code>
-%%% in the list of tuples returned from <code>Suite:Case/0</code>.
-userdata(TestDir, Suite, Case) ->
+%%% in the list of tuples returned from <code>Suite:group(GroupName)</code>
+%%% or <code>Suite:Case()</code>.
+userdata(TestDir, Suite, {group,GroupName}) ->
+    case make_and_load(TestDir, Suite) of
+	E = {error,_} ->
+	    E;
+	_ ->
+	    Info = (catch apply(Suite, group, [GroupName])),
+	    get_userdata(Info, "group("++atom_to_list(GroupName)++")")
+    end;
+
+userdata(TestDir, Suite, Case) when is_atom(Case) ->
     case make_and_load(TestDir, Suite) of
 	E = {error,_} ->
 	    E;
@@ -906,6 +918,18 @@ timetrap(Time) ->
     test_server:timetrap(Time).
 
 %%%-----------------------------------------------------------------
+%%% @spec get_timetrap_info() -> {Time,Scale}
+%%%       Time = integer() | infinity
+%%%       Scale = true | false
+%%%
+%%% @doc <p>Read info about the timetrap set for the current test case.
+%%%      <c>Scale</c> indicates if Common Test will attempt to automatically
+%%%      compensate timetraps for runtime delays introduced by e.g. tools like
+%%%      cover.</p>
+get_timetrap_info() ->
+    test_server:get_timetrap_info().
+
+%%%-----------------------------------------------------------------
 %%% @spec sleep(Time) -> ok
 %%%       Time = {hours,Hours} | {minutes,Mins} | {seconds,Secs} | Millisecs | infinity
 %%%       Hours = integer()
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index 0897675591..f99eafaa34 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -36,6 +36,9 @@
 -include("ct_event.hrl").
 -include("ct_util.hrl").
 
+-define(val(Key, List), proplists:get_value(Key, List)). 
+-define(val(Key, List, Def), proplists:get_value(Key, List, Def)).   
+
 %%%-----------------------------------------------------------------
 %%% @spec init_tc(Mod,Func,Args) -> {ok,NewArgs} | {error,Reason} |
 %%%         {skip,Reason} | {auto_skip,Reason}
@@ -98,6 +101,17 @@ init_tc(Mod,Func,Config) ->
 	    {skip,InitFailed}
     end.	    
 
+init_tc1(?MODULE,error_in_suite,[Config0],_) when is_list(Config0) ->
+    ct_logs:init_tc(false),
+    ct_event:notify(#event{name=tc_start,
+			   node=node(),
+			   data={?MODULE,error_in_suite}}),
+    case ?val(error, Config0) of
+	undefined ->
+	    {skip,"unknown_error_in_suite"};
+	Reason ->
+	    {skip,Reason}
+    end;
 init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) ->
     Config1 = 
 	case ct_util:read_suite_data(last_saved_config) of
@@ -122,19 +136,22 @@ init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) ->
 	    %% release all name -> key bindings (once per suite)
 	    ct_config:release_allocated()
     end,
-    TestCaseInfo =
-	case catch apply(Mod,Func,[]) of
-	    Result when is_list(Result) -> Result;
-	    _ -> []
-	end,
+
+    GroupPath = ?val(tc_group_path, Config, []),
+    AllGroups =	[?val(tc_group_properties, Config, []) | GroupPath],
+
     %% clear all config data default values set by previous
     %% testcase info function (these should only survive the
     %% testcase, not the whole suite)
-    ct_config:delete_default_config(testcase),
-    case add_defaults(Mod,Func,TestCaseInfo,DoInit) of
+    FuncSpec = group_or_func(Func,Config0),
+    if is_tuple(FuncSpec) ->			% group
+	    ok;
+       true ->
+	    ct_config:delete_default_config(testcase)
+    end,
+    case add_defaults(Mod,Func,AllGroups,DoInit) of
 	Error = {suite0_failed,_} ->
 	    ct_logs:init_tc(false),
-	    FuncSpec = group_or_func(Func,Config0),
 	    ct_event:notify(#event{name=tc_start,
 				   node=node(),
 				   data={Mod,FuncSpec}}),
@@ -144,13 +161,13 @@ init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) ->
 	    case MergeResult of
 		{error,Reason} when DoInit == false ->
 		    ct_logs:init_tc(false),
-		    FuncSpec = group_or_func(Func,Config0),
 		    ct_event:notify(#event{name=tc_start,
 					   node=node(),
 					   data={Mod,FuncSpec}}),
 		    {skip,Reason};
 		_ ->
-		    init_tc2(Mod,Func,SuiteInfo,MergeResult,Config,DoInit)
+		    init_tc2(Mod,Func,SuiteInfo,MergeResult,
+			     Config,DoInit)
 	    end
     end;
 init_tc1(_Mod,_Func,Args,_DoInit) ->
@@ -203,8 +220,9 @@ init_tc2(Mod,Func,SuiteInfo,MergeResult,Config,DoInit) ->
     ct_event:notify(#event{name=tc_start,
 			   node=node(),
 			   data={Mod,FuncSpec}}),
-    
-    case catch configure(MergedInfo1,MergedInfo1,SuiteInfo,{Func,DoInit},Config) of
+
+    case catch configure(MergedInfo1,MergedInfo1,SuiteInfo,
+			 {FuncSpec,DoInit},Config) of
 	{suite0_failed,Reason} ->
 	    ct_util:set_testdata({curr_tc,{Mod,{suite0_failed,{require,Reason}}}}),
 	    {skip,{require_failed_in_suite0,Reason}};
@@ -212,7 +230,7 @@ init_tc2(Mod,Func,SuiteInfo,MergeResult,Config,DoInit) ->
 	    {auto_skip,{require_failed,Reason}};
 	{'EXIT',Reason} ->
 	    {auto_skip,Reason};
-	{ok, FinalConfig} ->
+	{ok,FinalConfig} ->
 	    case MergeResult of
 		{error,Reason} ->
 		    %% suite0 configure finished now, report that 
@@ -241,12 +259,12 @@ ct_suite_init(Mod, Func, [Config]) when is_list(Config) ->
 	    Else
     end.
 
-add_defaults(Mod,Func,FuncInfo,DoInit) ->
+add_defaults(Mod,Func, GroupPath, DoInit) ->
     case (catch Mod:suite()) of
 	{'EXIT',{undef,_}} ->
 	    SuiteInfo = merge_with_suite_defaults(Mod,[]),
 	    SuiteInfoNoCTH = [I || I <- SuiteInfo, element(1,I) =/= ct_hooks],
-	    case add_defaults1(Mod,Func,FuncInfo,SuiteInfoNoCTH,DoInit) of
+	    case add_defaults1(Mod,Func, GroupPath, SuiteInfoNoCTH, DoInit) of
 		Error = {error,_} -> {SuiteInfo,Error};
 		MergedInfo -> {SuiteInfo,MergedInfo}
 	    end;
@@ -262,11 +280,11 @@ add_defaults(Mod,Func,FuncInfo,DoInit) ->
 			      (_) -> false
 			   end, SuiteInfo) of
 		true ->
-		    SuiteInfo1 = merge_with_suite_defaults(Mod,SuiteInfo),
+		    SuiteInfo1 = merge_with_suite_defaults(Mod, SuiteInfo),
 		    SuiteInfoNoCTH = [I || I <- SuiteInfo1,
 					   element(1,I) =/= ct_hooks],
-		    case add_defaults1(Mod,Func,FuncInfo,
-				       SuiteInfoNoCTH,DoInit) of
+		    case add_defaults1(Mod,Func, GroupPath,
+				       SuiteInfoNoCTH, DoInit) of
 			Error = {error,_} -> {SuiteInfo1,Error};
 			MergedInfo -> {SuiteInfo1,MergedInfo}
 		    end;
@@ -287,51 +305,147 @@ add_defaults(Mod,Func,FuncInfo,DoInit) ->
 	    {suite0_failed,bad_return_value}
     end.
 
-add_defaults1(_Mod,init_per_suite,[],SuiteInfo,_DoInit) ->
-    SuiteInfo;
-
-add_defaults1(Mod,Func,FuncInfo,SuiteInfo,DoInit) ->
-    %% mustn't re-require suite variables in test case info function,
-    %% can result in weird behaviour (suite values get overwritten)
+add_defaults1(Mod,Func, GroupPath, SuiteInfo, DoInit) ->
+    %% GroupPathInfo (for subgroup on level X) =
+    %%     [LevelXGroupInfo, LevelX-1GroupInfo, ..., TopLevelGroupInfo]
+    GroupPathInfo =  
+	lists:map(fun(GroupProps) ->
+			  Name = ?val(name, GroupProps),
+			  case catch Mod:group(Name) of
+			      GrInfo when is_list(GrInfo) -> GrInfo;
+			      _ -> []
+			  end
+		  end, GroupPath),
+    TestCaseInfo =
+	case catch apply(Mod,Func,[]) of
+	    TCInfo when is_list(TCInfo) -> TCInfo;
+	    _ -> []
+	end,
+    %% let test case info (also for all config funcs) override group info,
+    %% and lower level group info override higher level info
+    TCAndGroupInfo = [TestCaseInfo | remove_info_in_prev(TestCaseInfo,
+							 GroupPathInfo)],
+    %% find and save require terms found in suite info
     SuiteReqs = 
 	[SDDef || SDDef <- SuiteInfo,
 		  ((require == element(1,SDDef)) or
 		   (default_config == element(1,SDDef)))],
-    FuncReqs =
-	[FIDef || FIDef <- FuncInfo,
-		  require == element(1,FIDef)],
-    case [element(2,Clash) || Clash <- SuiteReqs,
-			      require == element(1, Clash),
-			      true == lists:keymember(element(2,Clash),2,
-						      FuncReqs)] of
+
+    case check_for_clashes(TestCaseInfo, GroupPathInfo, SuiteReqs) of
 	[] ->
-	    add_defaults2(Mod,Func,FuncInfo,SuiteInfo,SuiteReqs,DoInit);
+	    add_defaults2(Mod,Func, TCAndGroupInfo,SuiteInfo,SuiteReqs, DoInit);
 	Clashes ->
 	    {error,{config_name_already_in_use,Clashes}}
     end.
 
-add_defaults2(Mod,init_per_suite,IPSInfo,SuiteInfo,SuiteReqs,false) ->
-    %% not common practise to use a test case info function for
-    %% init_per_suite (usually handled by suite/0), but let's support
-    %% it just in case...
-    add_defaults2(Mod,init_per_suite,IPSInfo,SuiteInfo,SuiteReqs,true);
-
-add_defaults2(_Mod,_Func,FuncInfo,SuiteInfo,_,false) ->
-    %% include require elements from test case info, but not from suite/0
-    %% (since we've already required those vars)
-    FuncInfo ++
-	[SFDef || SFDef <- SuiteInfo,
-		  require /= element(1,SFDef),
-		  false == lists:keymember(element(1,SFDef),1,FuncInfo)];
-
-add_defaults2(_Mod,_Func,FuncInfo,SuiteInfo,SuiteReqs,true) ->
-    %% We must include require elements from suite/0 here since
-    %% there's no init_per_suite call before this first test case.
-    %% Let other test case info elements override those from suite/0.
-    FuncInfo ++ SuiteReqs ++
-	[SDDef || SDDef <- SuiteInfo,
-		  require /= element(1,SDDef),
-		  false == lists:keymember(element(1,SDDef),1,FuncInfo)].    
+%% Check that alias names are not already in use
+check_for_clashes(TCInfo, GrPathInfo, SuiteInfo) ->
+    {CurrGrInfo,SearchIn} = case GrPathInfo of
+				[] -> {[],[SuiteInfo]};
+				[Curr|Path] -> {Curr,[SuiteInfo|Path]}
+			    end,
+    ReqNames = fun(Info) -> [element(2,R) || R <- Info,
+					     size(R) == 3,
+					     require == element(1,R)]
+	       end,
+    ExistingNames = lists:flatten([ReqNames(L)  || L <- SearchIn]),
+    CurrGrReqNs = ReqNames(CurrGrInfo),
+    GrClashes = [Name || Name <- CurrGrReqNs,
+			 true == lists:member(Name, ExistingNames)],
+    AllReqNs = CurrGrReqNs ++ ExistingNames,
+    TCClashes = [Name || Name <- ReqNames(TCInfo),
+			 true == lists:member(Name, AllReqNs)],
+    TCClashes ++ GrClashes.
+
+%% Delete the info terms in Terms from all following info lists
+remove_info_in_prev(Terms, [[] | Rest]) ->
+    [[] | remove_info_in_prev(Terms, Rest)];
+remove_info_in_prev(Terms, [Info | Rest]) ->
+    UniqueInInfo = [U || U <- Info,
+			  ((timetrap == element(1,U)) and
+			   (not lists:keymember(timetrap,1,Terms))) or 
+			  ((require == element(1,U)) and
+			   (not lists:member(U,Terms))) or
+			  ((default_config == element(1,U)) and
+                           (not keysmember([default_config,1,
+					    element(2,U),2], Terms)))],
+    OtherTermsInInfo = [T || T <- Info,
+			     timetrap /= element(1,T),
+			     require /= element(1,T),
+			     default_config /= element(1,T),
+			     false == lists:keymember(element(1,T),1,
+						      Terms)],
+    KeptInfo = UniqueInInfo ++ OtherTermsInInfo,
+    [KeptInfo | remove_info_in_prev(Terms ++ KeptInfo, Rest)];
+remove_info_in_prev(_, []) ->
+    [].
+
+keysmember([Key,Pos|Next], List) ->
+    case [Elem || Elem <- List, Key == element(Pos,Elem)] of
+	[]    -> false;
+	Found -> keysmember(Next, Found)
+    end;
+keysmember([], _) -> true.
+
+
+add_defaults2(Mod,init_per_suite, IPSInfo, SuiteInfo,SuiteReqs, false) ->
+    add_defaults2(Mod,init_per_suite, IPSInfo, SuiteInfo,SuiteReqs, true);
+
+add_defaults2(_Mod,IPG, IPGAndGroupInfo, SuiteInfo,SuiteReqs, DoInit) when
+      IPG == init_per_group ; IPG == ct_init_per_group ->
+    %% If DoInit == true, we have to process the suite() list, otherwise
+    %% it has already been handled (see clause for init_per_suite)
+    case DoInit of		
+	true ->
+	    %% note: we know for sure this is a top level group
+	    Info = lists:flatten([IPGAndGroupInfo, SuiteReqs]),
+	    Info ++ remove_info_in_prev(Info, [SuiteInfo]);
+	false ->
+	    SuiteInfo1 =
+		remove_info_in_prev(lists:flatten([IPGAndGroupInfo,
+						   SuiteReqs]), [SuiteInfo]),
+	    %% don't require terms in prev groups (already processed)
+	    case IPGAndGroupInfo of
+		[IPGInfo] ->
+		    lists:flatten([IPGInfo,SuiteInfo1]);
+		[IPGInfo | [CurrGroupInfo | PrevGroupInfo]] ->
+		    PrevGroupInfo1 = delete_require_terms(PrevGroupInfo),
+		    lists:flatten([IPGInfo,CurrGroupInfo,PrevGroupInfo1,
+				   SuiteInfo1])
+	    end
+    end;
+
+add_defaults2(_Mod,_Func, TCAndGroupInfo, SuiteInfo,SuiteReqs, false) ->
+    %% Include require elements from test case info and current group,
+    %% but not from previous groups or suite/0 (since we've already required
+    %% those vars). Let test case info elements override group and suite
+    %% info elements.
+    SuiteInfo1 = remove_info_in_prev(lists:flatten([TCAndGroupInfo,
+						    SuiteReqs]), [SuiteInfo]),
+    %% don't require terms in prev groups (already processed)
+    case TCAndGroupInfo of
+	[TCInfo] ->
+	    lists:flatten([TCInfo,SuiteInfo1]);
+	[TCInfo | [CurrGroupInfo | PrevGroupInfo]] ->
+	    PrevGroupInfo1 = delete_require_terms(PrevGroupInfo),
+	    lists:flatten([TCInfo,CurrGroupInfo,PrevGroupInfo1,
+			   SuiteInfo1])
+    end;
+
+add_defaults2(_Mod,_Func, TCInfo, SuiteInfo,SuiteReqs, true) ->
+    %% Here we have to process the suite info list also (no call to
+    %% init_per_suite before this first test case). This TC can't belong
+    %% to a group, or the clause for (ct_)init_per_group would've caught this.
+    Info = lists:flatten([TCInfo, SuiteReqs]),
+    lists:flatten([Info,remove_info_in_prev(Info, [SuiteInfo])]).
+
+delete_require_terms([Info | Prev]) ->
+    Info1 = [T || T <- Info, 
+		  require /= element(1,T),
+		  default_config /= element(1,T)],
+    [Info1 | delete_require_terms(Prev)];
+delete_require_terms([]) ->
+    [].
 
 merge_with_suite_defaults(Mod,SuiteInfo) ->
     case lists:keysearch(suite_defaults,1,Mod:module_info(attributes)) of
@@ -364,7 +478,8 @@ configure([{require,Required}|Rest],Info,SuiteInfo,Scope,Config) ->
 	ok ->
 	    configure(Rest,Info,SuiteInfo,Scope,Config);
 	Error = {error,Reason} ->
-	    case required_default('_UNDEF',Required,Info,SuiteInfo,Scope) of
+	    case required_default('_UNDEF',Required,Info,
+				  SuiteInfo,Scope) of
 		ok ->
 		    configure(Rest,Info,SuiteInfo,Scope,Config);
 		_ ->
@@ -406,18 +521,24 @@ configure([],_,_,_,Config) ->
     {ok,[Config]}.
 
 %% the require element in Info may come from suite/0 and
-%% should be scoped 'suite', or come from the testcase info
-%% function and should then be scoped 'testcase'
-required_default(Name,Key,Info,SuiteInfo,{Func,true}) ->
+%% should be scoped 'suite', or come from the group info
+%% function and be scoped 'group', or come from the testcase
+%% info function and then be scoped 'testcase'
+
+required_default(Name,Key,Info,SuiteInfo,{FuncSpec,true}) ->
     case try_set_default(Name,Key,SuiteInfo,suite) of
 	ok ->
 	    ok;
 	_ ->
-	    required_default(Name,Key,Info,[],{Func,false})
+	    required_default(Name,Key,Info,[],{FuncSpec,false})
     end;
 required_default(Name,Key,Info,_,{init_per_suite,_}) ->
     try_set_default(Name,Key,Info,suite);
-required_default(Name,Key,Info,_,_) ->
+required_default(Name,Key,Info,_,{{init_per_group,GrName,_},_}) ->
+    try_set_default(Name,Key,Info,{group,GrName});
+required_default(Name,Key,Info,_,{{ct_init_per_group,GrName,_},_}) ->
+    try_set_default(Name,Key,Info,{group,GrName});
+required_default(Name,Key,Info,_,_FuncSpec) ->
     try_set_default(Name,Key,Info,testcase).
 
 try_set_default(Name,Key,Info,Where) ->
@@ -484,7 +605,11 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) ->
     ct_util:delete_suite_data(last_saved_config),
     FuncSpec =
 	case group_or_func(Func,Args) of
-	    {_,GroupName,_Props} = Group ->			       
+	    {_,GroupName,_Props} = Group ->
+		if Func == end_per_group; Func == ct_end_per_group ->
+			ct_config:delete_default_config({group,GroupName});
+		   true -> ok
+		end,
 		case lists:keysearch(save_config,1,Args) of
 		    {value,{save_config,SaveConfig}} ->
 			ct_util:save_suite_data(
@@ -703,12 +828,14 @@ mark_as_failed1(_,_,_,[]) ->
     ok.
 
 group_or_func(Func, Config) when Func == init_per_group; 
-				 Func == end_per_group ->
-    case proplists:get_value(tc_group_properties,Config) of
+				 Func == end_per_group;
+				 Func == ct_init_per_group; 
+				 Func == ct_end_per_group ->
+    case ?val(tc_group_properties, Config) of
 	undefined ->
 	    {Func,unknown,[]};
 	GrProps ->
-	    GrName = proplists:get_value(name,GrProps),
+	    GrName = ?val(name,GrProps),
 	    {Func,GrName,proplists:delete(name,GrProps)}
     end;	  
 group_or_func(Func, _Config) ->
@@ -746,7 +873,7 @@ get_suite(Mod, all) ->
 
 %% group
 get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) ->
-    Name = proplists:get_value(name, Props),
+    Name = ?val(name, Props),
     case catch apply(Mod, groups, []) of
 	{'EXIT',_} ->
 	    [Group];
@@ -764,7 +891,7 @@ get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) ->
 			    %% a *subgroup* specified *only* as skipped (and not
 			    %% as an explicit test) should not be returned, or
 			    %% init/end functions for top groups will be executed
-			    case catch proplists:get_value(name, element(2, hd(ConfTests))) of
+			    case catch ?val(name, element(2, hd(ConfTests))) of
 				Name ->		% top group
 				    delete_subs(ConfTests, ConfTests);
 				_ ->
@@ -918,14 +1045,14 @@ delete_subs([], All) ->
     All.
 
 delete_conf({conf,Props,_,_,_}, Confs) ->
-    Name = proplists:get_value(name, Props),
+    Name = ?val(name, Props),
     [Conf || Conf = {conf,Props0,_,_,_} <- Confs,
-	     Name =/= proplists:get_value(name, Props0)].
+	     Name =/= ?val(name, Props0)].
 
 is_sub({conf,Props,_,_,_}=Conf, [{conf,_,_,Tests,_} | Confs]) ->
-    Name = proplists:get_value(name, Props),
+    Name = ?val(name, Props),
     case lists:any(fun({conf,Props0,_,_,_}) ->
-			   case proplists:get_value(name, Props0) of
+			   case ?val(name, Props0) of
 			       N when N == Name ->
 				   true;
 			       _ ->
@@ -1080,7 +1207,7 @@ expand_groups([], _ConfTests, _Mod) ->
 expand_groups({group,Name}, ConfTests, Mod) ->
     FindConf = 
 	fun({conf,Props,_,_,_}) ->
-		case proplists:get_value(name, Props) of
+		case ?val(name, Props) of
 		    Name -> true;
 		    _    -> false
 		end
@@ -1234,8 +1361,8 @@ report(What,Data) ->
 	loginfo ->
 	    %% logfiles and direcories have been created for a test and the
 	    %% top level test index page needs to be refreshed
-	    TestName = filename:basename(proplists:get_value(topdir, Data), ".logs"),
-	    RunDir = proplists:get_value(rundir, Data),
+	    TestName = filename:basename(?val(topdir, Data), ".logs"),
+	    RunDir = ?val(rundir, Data),
 	    ct_logs:make_all_suites_index({TestName,RunDir}),
 	    ok;
 	tests_start ->
@@ -1342,6 +1469,17 @@ report(What,Data) ->
 	       true -> 
 		    ok
 	    end;
+	framework_error ->
+	    case Data of
+		{{M,F},E} ->
+		    ct_event:sync_notify(#event{name=tc_done,
+						node=node(),
+						data={M,F,{framework_error,E}}});
+		_ ->
+		    ct_event:sync_notify(#event{name=tc_done,
+						node=node(),
+						data=Data})
+	    end;
 	_ ->
 	    ok
     end,
diff --git a/lib/common_test/test/ct_config_SUITE.erl b/lib/common_test/test/ct_config_SUITE.erl
index 8ce75f582a..18218bee47 100644
--- a/lib/common_test/test/ct_config_SUITE.erl
+++ b/lib/common_test/test/ct_config_SUITE.erl
@@ -228,7 +228,7 @@ expected_events(config_static_SUITE)->
      {?eh,tc_start,{config_static_SUITE,test_config_name_already_in_use2}},
      {?eh,tc_done,
       {config_static_SUITE,test_config_name_already_in_use2,
-       {skipped,{config_name_already_in_use,[x1,alias]}}}},
+       {skipped,{config_name_already_in_use,[alias,x1]}}}},
      {?eh,test_stats,{4,0,{2,0}}},
      {?eh,tc_start,{config_static_SUITE,test_alias_tclocal}},
      {?eh,tc_done,{config_static_SUITE,test_alias_tclocal,ok}},
diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl
index c1a455c6d8..2f081decca 100644
--- a/lib/common_test/test/ct_error_SUITE.erl
+++ b/lib/common_test/test/ct_error_SUITE.erl
@@ -493,7 +493,7 @@ test_events(cfg_error) ->
      {?eh,tc_start,{cfg_error_9_SUITE,tc1}},
      {?eh,tc_done,{cfg_error_9_SUITE,tc1,
 		   {skipped,{failed,{cfg_error_9_SUITE,init_per_testcase,
-				     tc1_should_be_skipped}}}}},
+				     {tc1_should_be_skipped,'_'}}}}}},
      {?eh,test_stats,{9,0,{0,15}}},
      {?eh,tc_start,{cfg_error_9_SUITE,tc2}},
      {?eh,tc_done,{cfg_error_9_SUITE,tc2,
@@ -535,12 +535,7 @@ test_events(cfg_error) ->
      {?eh,tc_start,{cfg_error_9_SUITE,tc13}},
      {?eh,tc_done,{cfg_error_9_SUITE,tc13,
 		   {failed,{cfg_error_9_SUITE,end_per_testcase,
-			    {'EXIT',{{badmatch,undefined},
-				     [{cfg_error_9_SUITE,end_per_testcase,2},
-				      {test_server,my_apply,3},
-				      {test_server,do_end_per_testcase,4},
-				      {test_server,run_test_case_eval1,6},
-				      {test_server,run_test_case_eval,9}]}}}}}},
+			    {'EXIT',{{badmatch,undefined},'_'}}}}}},
      {?eh,test_stats,{12,3,{0,18}}},
      {?eh,tc_start,{cfg_error_9_SUITE,tc14}},
      {?eh,tc_done,
@@ -568,8 +563,8 @@ test_events(cfg_error) ->
      {?eh,tc_start,{cfg_error_11_SUITE,init_per_suite}},
      {?eh,tc_done,{cfg_error_11_SUITE,init_per_suite,ok}},
      {?eh,tc_start,{cfg_error_11_SUITE,tc1}},
-     {?eh,tc_done,{cfg_error_11_SUITE,tc1,
-		   {skipped,{config_name_already_in_use,[dummy0]}}}},
+     {?eh,tc_done, {cfg_error_11_SUITE,tc1,
+		    {skipped,{config_name_already_in_use,[dummy_alias]}}}},
      {?eh,test_stats,{12,6,{1,19}}},
      {?eh,tc_start,{cfg_error_11_SUITE,tc2}},
      {?eh,tc_done,{cfg_error_11_SUITE,tc2,ok}},
@@ -577,7 +572,7 @@ test_events(cfg_error) ->
      {?eh,tc_start,{cfg_error_11_SUITE,end_per_suite}},
      {?eh,tc_done,{cfg_error_11_SUITE,end_per_suite,ok}},
      {?eh,tc_start,{cfg_error_12_SUITE,tc1}},
-     {?eh,tc_done,{cfg_error_12_SUITE,tc1,{failed,{timetrap_timeout,500}}}},
+     {?eh,tc_done,{ct_framework,init_tc,{framework_error,{timetrap,500}}}},
      {?eh,test_stats,{13,7,{1,19}}},
      {?eh,tc_start,{cfg_error_12_SUITE,tc2}},
      {?eh,tc_done,{cfg_error_12_SUITE,tc2,{failed,
diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_11_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_11_SUITE.erl
index ce94533110..aa7857aa92 100644
--- a/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_11_SUITE.erl
+++ b/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_11_SUITE.erl
@@ -29,10 +29,7 @@
 suite() ->
     [{timetrap,{seconds,2}},
      {require, dummy0}, {default_config, dummy0, "suite/0"},
-     {require, dummy1}, {default_config, dummy1, "suite/0"},
-     {require, dummy2}, {default_config, dummy2, "suite/0"}].
-
-
+     {require, dummy_alias, dummy1}, {default_config, dummy1, "suite/0"}].
 
 %%--------------------------------------------------------------------
 %% Function: init_per_suite(Config0) ->
@@ -119,16 +116,17 @@ groups() ->
 %% Reason = term()
 %%--------------------------------------------------------------------
 all() ->
-    [tc1, tc2].
+    [tc1,tc2].
 
 tc1() ->
-    [{require, dummy0}, {default_config, dummy0, "tc1"}].
+    [{require, dummy0}, {default_config, dummy0, "tc1"},
+     {require, dummy_alias, dummy2}, {default_config, dummy2, "tc1"}].
 
 tc1(_) ->
     dummy.
 
 tc2() ->
-    [{timetrap,1}].
+    [{timetrap,10}].
 
 tc2(_) ->
     dummy.
diff --git a/lib/common_test/test/ct_groups_test_2_SUITE.erl b/lib/common_test/test/ct_groups_test_2_SUITE.erl
index 940d791b15..2392b0b850 100644
--- a/lib/common_test/test/ct_groups_test_2_SUITE.erl
+++ b/lib/common_test/test/ct_groups_test_2_SUITE.erl
@@ -171,16 +171,16 @@ test_events(missing_conf) ->
      {?eh,start_logging,{'DEF','RUNDIR'}},
      {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
      {?eh,start_info,{1,1,2}},
-     {?eh,tc_start,{ct_framework,ct_init_per_group}},
-     {?eh,tc_done,{ct_framework,ct_init_per_group,ok}},
+     {?eh,tc_start,{ct_framework,{ct_init_per_group,group1,[]}}},
+     {?eh,tc_done,{ct_framework,{ct_init_per_group,group1,[]},ok}},
      {?eh,tc_start,{missing_conf_SUITE,tc1}},
      {?eh,tc_done,{missing_conf_SUITE,tc1,ok}},
      {?eh,test_stats,{1,0,{0,0}}},
      {?eh,tc_start,{missing_conf_SUITE,tc2}},
      {?eh,tc_done,{missing_conf_SUITE,tc2,ok}},
      {?eh,test_stats,{2,0,{0,0}}},
-     {?eh,tc_start,{ct_framework,ct_end_per_group}},
-     {?eh,tc_done,{ct_framework,ct_end_per_group,ok}},
+     {?eh,tc_start,{ct_framework,{ct_end_per_group,group1,[]}}},
+     {?eh,tc_done,{ct_framework,{ct_end_per_group,group1,[]},ok}},
      {?eh,test_done,{'DEF','STOP_TIME'}},
      {?eh,stop_logging,[]}
     ];
diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE.erl
index 4471915e69..efc0309781 100644
--- a/lib/common_test/test/ct_test_server_if_1_SUITE.erl
+++ b/lib/common_test/test/ct_test_server_if_1_SUITE.erl
@@ -228,39 +228,39 @@ test_events(ts_if_1) ->
 			{failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}},
 
      {?eh,tc_start,{ct_framework,error_in_suite}},
-     {?eh,test_stats,{3,6,{3,7}}},
+     {?eh,test_stats,{3,5,{4,7}}},
 
      {?eh,tc_start,{ct_framework,error_in_suite}},
-     {?eh,test_stats,{3,7,{3,7}}},
+     {?eh,test_stats,{3,5,{5,7}}},
 
      {?eh,tc_start,{ts_if_5_SUITE,init_per_suite}},
      {?eh,tc_done,{ts_if_5_SUITE,init_per_suite,
 		   {skipped,{require_failed_in_suite0,{not_available,undef_variable}}}}},
      {?eh,tc_auto_skip,{ts_if_5_SUITE,my_test_case,
 			{require_failed_in_suite0,{not_available,undef_variable}}}},
-     {?eh,test_stats,{3,7,{3,8}}},
+     {?eh,test_stats,{3,5,{5,8}}},
      {?eh,tc_auto_skip,{ts_if_5_SUITE,end_per_suite,
 			{require_failed_in_suite0,{not_available,undef_variable}}}},
 
      {?eh,tc_start,{ts_if_6_SUITE,tc1}},
      {?eh,tc_done,{ts_if_6_SUITE,tc1,{failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}},
-     {?eh,test_stats,{3,7,{4,8}}},
+     {?eh,test_stats,{3,5,{6,8}}},
 
      {?eh,tc_start,{ts_if_7_SUITE,tc1}},
      {?eh,tc_done,{ts_if_7_SUITE,tc1,ok}},
-     {?eh,test_stats,{4,7,{4,8}}},
+     {?eh,test_stats,{4,5,{6,8}}},
 
      {?eh,tc_start,{ts_if_8_SUITE,tc1}},
      {?eh,tc_done,{ts_if_8_SUITE,tc1,{failed,{error,failed_on_purpose}}}},
-     {?eh,test_stats,{4,8,{4,8}}},
+     {?eh,test_stats,{4,6,{6,8}}},
 
      {?eh,tc_user_skip,{skipped_by_spec_1_SUITE,all,"should be skipped"}},
-     {?eh,test_stats,{4,8,{5,8}}},
+     {?eh,test_stats,{4,6,{7,8}}},
 
      {?eh,tc_start,{skipped_by_spec_2_SUITE,init_per_suite}},
      {?eh,tc_done,{skipped_by_spec_2_SUITE,init_per_suite,ok}},
      {?eh,tc_user_skip,{skipped_by_spec_2_SUITE,tc1,"should be skipped"}},
-     {?eh,test_stats,{4,8,{6,8}}},
+     {?eh,test_stats,{4,6,{8,8}}},
      {?eh,tc_start,{skipped_by_spec_2_SUITE,end_per_suite}},
      {?eh,tc_done,{skipped_by_spec_2_SUITE,end_per_suite,ok}},