Merge branch 'peppe/common_test/dev/OTP-8725' of super:peppe_otp into dev

* 'peppe/common_test/dev/OTP-8725' of super:peppe_otp:
  Fix bug that crashes common_test when running cover
  Bump version numbers for common_test and test_server
  Add documentation for group specifications and for the label option
  Improve handling of test case group specifications
  Add textured backgound to html log files
  Improve handling of test case group specifications
  Add new option to label test runs
  Implement handling of group specifications
  Fix problem with using the wrong logdir setting
  Rewrite handling of group definitions in Common Test test suites

1 files changed, 222 insertions, 81 deletions

diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index 6a9c42d1b9..c5bfd01642 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -45,7 +45,8 @@
 -define(abs(Name), filename:absname(Name)).
 -define(testdir(Name, Suite), ct_util:get_testdir(Name, Suite)).
 
--record(opts, {vts,
+-record(opts, {label,
+	       vts,
 	       shell,
 	       cover,
 	       coverspec,
@@ -153,14 +154,16 @@ script_start(Args) ->
 		Result
 	end,
     stop_trace(Tracing),
+    timer:sleep(1000),
     Res.
 
 script_start1(Parent, Args) ->
     %% read general start flags
+    Label = get_start_opt(label, fun([Lbl]) -> Lbl end, Args),
     Vts = get_start_opt(vts, true, Args),
     Shell = get_start_opt(shell, true, Args),
     Cover = get_start_opt(cover, fun([CoverFile]) -> ?abs(CoverFile) end, Args),
-    LogDir = get_start_opt(logdir, fun([LogD]) -> LogD end, ".", Args),
+    LogDir = get_start_opt(logdir, fun([LogD]) -> LogD end, Args),
     MultTT = get_start_opt(multiply_timetraps,
 			   fun([MT]) -> list_to_integer(MT) end, 1, Args),
     ScaleTT = get_start_opt(scale_timetraps,
@@ -229,7 +232,7 @@ script_start1(Parent, Args) ->
 	    application:set_env(common_test, basic_html, true)
     end,
 
-   StartOpts = #opts{vts = Vts, shell = Shell, cover = Cover,
+   StartOpts = #opts{label = Label, vts = Vts, shell = Shell, cover = Cover,
 		     logdir = LogDir, event_handlers = EvHandlers,
 		     include = IncludeDirs,
 		     silent_connections = SilentConns,
@@ -288,6 +291,9 @@ script_start2(StartOpts = #opts{vts = undefined,
 		    TS ->
 			SpecStartOpts = get_data_for_node(TS, node()),
 
+			Label = choose_val(StartOpts#opts.label,
+					   SpecStartOpts#opts.label),
+
 			LogDir = choose_val(StartOpts#opts.logdir,
 					    SpecStartOpts#opts.logdir),
 
@@ -303,7 +309,8 @@ script_start2(StartOpts = #opts{vts = undefined,
 						 SpecStartOpts#opts.include]),
 			application:set_env(common_test, include, AllInclude),
 
-			{TS,StartOpts#opts{testspecs = Specs,
+			{TS,StartOpts#opts{label = Label,
+					   testspecs = Specs,
 					   cover = Cover,
 					   logdir = LogDir,
 					   config = SpecStartOpts#opts.config,
@@ -317,29 +324,30 @@ script_start2(StartOpts = #opts{vts = undefined,
 	end,
     %% read config/userconfig from start flags
     InitConfig = ct_config:prepare_config_list(Args),
+    TheLogDir = which(logdir, Opts#opts.logdir),
     case {TestSpec,Terms} of
 	{_,{error,_}=Error} ->
 	    Error;
 	{[],_} ->
 	    {error,no_testspec_specified};
 	{undefined,_} ->   % no testspec used
-	    case check_and_install_configfiles(InitConfig,
-					       which(logdir,Opts#opts.logdir),
+	    case check_and_install_configfiles(InitConfig, TheLogDir,
 					       Opts#opts.event_handlers) of
 		ok ->      % go on read tests from start flags
-		    script_start3(Opts#opts{config=InitConfig}, Args);
+		    script_start3(Opts#opts{config=InitConfig,
+					    logdir=TheLogDir}, Args);
 		Error ->
 		    Error
 	    end;
 	{_,_} ->           % testspec used
 	    %% merge config from start flags with config from testspec
 	    AllConfig = merge_vals([InitConfig, Opts#opts.config]),
-	    case check_and_install_configfiles(AllConfig,
-					       which(logdir,Opts#opts.logdir),
+	    case check_and_install_configfiles(AllConfig, TheLogDir,
 					       Opts#opts.event_handlers) of
 		ok ->      % read tests from spec
 		    {Run,Skip} = ct_testspec:prepare_tests(Terms, node()),
-		    do_run(Run, Skip, Opts#opts{config=AllConfig}, Args);
+		    do_run(Run, Skip, Opts#opts{config=AllConfig,
+						logdir=TheLogDir}, Args);
 		Error ->
 		    Error
 	    end
@@ -348,11 +356,12 @@ script_start2(StartOpts = #opts{vts = undefined,
 script_start2(StartOpts, Args) ->
     %% read config/userconfig from start flags
     InitConfig = ct_config:prepare_config_list(Args),
-    case check_and_install_configfiles(InitConfig,
-				       which(logdir,StartOpts#opts.logdir),
+    LogDir = which(logdir, StartOpts#opts.logdir),
+    case check_and_install_configfiles(InitConfig, LogDir,
 				       StartOpts#opts.event_handlers) of
 	ok ->      % go on read tests from start flags
-	    script_start3(StartOpts#opts{config=InitConfig}, Args);
+	    script_start3(StartOpts#opts{config=InitConfig,
+					 logdir=LogDir}, Args);
 	Error ->
 	    Error
     end.
@@ -427,8 +436,12 @@ script_start4(#opts{vts = true, config = Config, event_handlers = EvHandlers,
 		    end, [], Config),
     vts:init_data(ConfigFiles, EvHandlers, ?abs(LogDir), Tests);
 
-script_start4(#opts{shell = true, config = Config, event_handlers = EvHandlers,
+script_start4(#opts{label = Label, shell = true, config = Config,
+		    event_handlers = EvHandlers,
 		    logdir = LogDir, testspecs = Specs}, _Args) ->
+    %% label - used by ct_logs
+    application:set_env(common_test, test_label, Label),
+
     InstallOpts = [{config,Config},{event_handler,EvHandlers}],
     if Config == [] ->
 	    ok;
@@ -613,9 +626,13 @@ run_test(StartOpts) when is_list(StartOpts) ->
     end.
 
 run_test1(StartOpts) ->
+    %% label
+    Label = get_start_opt(label, fun(Lbl) when is_list(Lbl) -> Lbl;
+				    (Lbl) when is_atom(Lbl) -> atom_to_list(Lbl)
+				 end, StartOpts),
     %% logdir
     LogDir = get_start_opt(logdir, fun(LD) when is_list(LD) -> LD end,
-			   ".", StartOpts),
+			   StartOpts),
     %% config & userconfig
     CfgFiles = ct_config:get_config_file_list(StartOpts),
 
@@ -711,7 +728,8 @@ run_test1(StartOpts) ->
     %% stepped execution
     Step = get_start_opt(step, value, StartOpts),
 
-    Opts = #opts{cover = Cover, step = Step, logdir = LogDir, config = CfgFiles,
+    Opts = #opts{label = Label,
+		 cover = Cover, step = Step, logdir = LogDir, config = CfgFiles,
 		 event_handlers = EvHandlers, include = Include,
 		 silent_connections = SilentConns,
 		 stylesheet = Stylesheet,
@@ -747,6 +765,8 @@ run_spec_file(Relaxed,
 	    exit(CTReason);
 	TS ->
 	    SpecOpts = get_data_for_node(TS, node()),
+	    Label = choose_val(Opts#opts.label,
+			       SpecOpts#opts.label),
 	    LogDir = choose_val(Opts#opts.logdir,
 				SpecOpts#opts.logdir),
 	    AllConfig = merge_vals([CfgFiles, SpecOpts#opts.config]),
@@ -766,8 +786,9 @@ run_spec_file(Relaxed,
 					       which(logdir,LogDir),
 					       AllEvHs) of
 		ok ->
-		    Opts1 = Opts#opts{cover = Cover,
-				      logdir = LogDir,
+		    Opts1 = Opts#opts{label = Label,
+				      cover = Cover,
+				      logdir = which(logdir, LogDir),
 				      config = AllConfig,
 				      event_handlers = AllEvHs,
 				      include = AllInclude,
@@ -775,7 +796,7 @@ run_spec_file(Relaxed,
 				      multiply_timetraps = MultTT,
 				      scale_timetraps = ScaleTT},
 		    {Run,Skip} = ct_testspec:prepare_tests(TS, node()),
-		    do_run(Run, Skip, Opts1, StartOpts);
+		    reformat_result(catch do_run(Run, Skip, Opts1, StartOpts));
 		{error,GCFReason} ->
 		    exit(GCFReason)
 	    end
@@ -785,9 +806,11 @@ run_prepared(Run, Skip, Opts = #opts{logdir = LogDir,
 				     config = CfgFiles,
 				     event_handlers = EvHandlers},
 	     StartOpts) ->
-    case check_and_install_configfiles(CfgFiles, LogDir, EvHandlers) of
+    LogDir1 = which(logdir, LogDir),
+    case check_and_install_configfiles(CfgFiles, LogDir1, EvHandlers) of
 	ok ->
-	    do_run(Run, Skip, Opts, StartOpts);
+	    reformat_result(catch do_run(Run, Skip, Opts#opts{logdir = LogDir1},
+					 StartOpts));
 	{error,Reason} ->
 	    exit(Reason)
     end.
@@ -816,6 +839,8 @@ check_config_file(Callback, File)->
 run_dir(Opts = #opts{logdir = LogDir,
 		     config = CfgFiles,
 		     event_handlers = EvHandlers}, StartOpts) ->
+    LogDir1 = which(logdir, LogDir),
+    Opts1 = Opts#opts{logdir = LogDir1},
     AbsCfgFiles =
 	lists:map(fun({Callback,FileList})->
 			  case code:is_loaded(Callback) of
@@ -834,7 +859,7 @@ run_dir(Opts = #opts{logdir = LogDir,
 					     check_config_file(Callback, File)
 				     end, FileList)}
 		  end, CfgFiles),
-    case install([{config,AbsCfgFiles},{event_handler,EvHandlers}], LogDir) of
+    case install([{config,AbsCfgFiles},{event_handler,EvHandlers}], LogDir1) of
 	ok -> ok;
 	{error,IReason} -> exit(IReason)
     end,
@@ -842,7 +867,7 @@ run_dir(Opts = #opts{logdir = LogDir,
 	{value,{_,Dirs=[Dir|_]}} when not is_integer(Dir),
 	                              length(Dirs)>1 ->
 	    %% multiple dirs (no suite)
-	    do_run(tests(Dirs), [], Opts, StartOpts);
+	    reformat_result(catch do_run(tests(Dirs), [], Opts1, StartOpts));
 	false ->				% no dir
 	    %% fun for converting suite name to {Dir,Mod} tuple
 	    S2M = fun(S) when is_list(S) ->
@@ -857,12 +882,15 @@ run_dir(Opts = #opts{logdir = LogDir,
 		    case listify(proplists:get_value(group, StartOpts, [])) ++
 			 listify(proplists:get_value(testcase, StartOpts, [])) of
 			[] ->
-			    do_run(tests(Dir, listify(Mod)), [], Opts, StartOpts);
+			    reformat_result(catch do_run(tests(Dir, listify(Mod)),
+							 [], Opts1, StartOpts));
 			GsAndCs ->
-			    do_run(tests(Dir, Mod, GsAndCs), [], Opts, StartOpts)
+			    reformat_result(catch do_run(tests(Dir, Mod, GsAndCs),
+							 [], Opts1, StartOpts))
 		    end;
 		{value,{_,Suites}} ->
-		    do_run(tests(lists:map(S2M, Suites)), [], Opts, StartOpts);
+		    reformat_result(catch do_run(tests(lists:map(S2M, Suites)),
+						 [], Opts1, StartOpts));
 		_ ->
 		    exit(no_tests_specified)
 	    end;
@@ -875,17 +903,22 @@ run_dir(Opts = #opts{logdir = LogDir,
 		    case listify(proplists:get_value(group, StartOpts, [])) ++
 			 listify(proplists:get_value(testcase, StartOpts, [])) of
 			[] ->
-			    do_run(tests(Dir, listify(Mod)), [], Opts, StartOpts);
+			    reformat_result(catch do_run(tests(Dir, listify(Mod)),
+							 [], Opts1, StartOpts));
 			GsAndCs ->
-			    do_run(tests(Dir, Mod, GsAndCs), [], Opts, StartOpts)
+			    reformat_result(catch do_run(tests(Dir, Mod, GsAndCs),
+							 [], Opts1, StartOpts))
 		    end;
 		{value,{_,Suites=[Suite|_]}} when is_list(Suite) ->
 		    Mods = lists:map(fun(Str) -> list_to_atom(Str) end, Suites),
-		    do_run(tests(delistify(Dir), Mods), [], Opts, StartOpts);
+		    reformat_result(catch do_run(tests(delistify(Dir), Mods),
+						 [], Opts1, StartOpts));
 		{value,{_,Suites}} ->
-		    do_run(tests(delistify(Dir), Suites), [], Opts, StartOpts);
+		    reformat_result(catch do_run(tests(delistify(Dir), Suites),
+						 [], Opts1, StartOpts));
 	        false ->			% no suite, only dir
-		    do_run(tests(listify(Dir)), [], Opts, StartOpts)
+		    reformat_result(catch do_run(tests(listify(Dir)),
+						 [], Opts1, StartOpts))
 	    end
     end.
 
@@ -925,28 +958,30 @@ run_testspec1(TestSpec) ->
 			EnvInclude++Opts#opts.include
 		end,
 	    application:set_env(common_test, include, AllInclude),
-
-	    case check_and_install_configfiles(Opts#opts.config,
-					       which(logdir,Opts#opts.logdir),
+	    LogDir1 = which(logdir,Opts#opts.logdir),
+	    case check_and_install_configfiles(Opts#opts.config, LogDir1,
 					       Opts#opts.event_handlers) of
 		ok ->
-		    Opts1 = Opts#opts{testspecs = [TestSpec],
+		    Opts1 = Opts#opts{testspecs = [],
+				      logdir = LogDir1,
 				      include = AllInclude},
 		    {Run,Skip} = ct_testspec:prepare_tests(TS, node()),
-		    do_run(Run, Skip, Opts1, []);
+		    reformat_result(catch do_run(Run, Skip, Opts1, []));
 		{error,GCFReason} ->
 		    exit(GCFReason)
 	    end
     end.
 
-get_data_for_node(#testspec{logdir=LogDirs,
-			    cover=CoverFs,
-			    config=Cfgs,
-			    userconfig=UsrCfgs,
-			    event_handler=EvHs,
-			    include=Incl,
-			    multiply_timetraps=MTs,
-			    scale_timetraps=STs}, Node) ->
+get_data_for_node(#testspec{label = Labels,
+			    logdir = LogDirs,
+			    cover = CoverFs,
+			    config = Cfgs,
+			    userconfig = UsrCfgs,
+			    event_handler = EvHs,
+			    include = Incl,
+			    multiply_timetraps = MTs,
+			    scale_timetraps = STs}, Node) ->
+    Label = proplists:get_value(Node, Labels),
     LogDir = case proplists:get_value(Node, LogDirs) of
 		 undefined -> ".";
 		 Dir -> Dir
@@ -958,7 +993,8 @@ get_data_for_node(#testspec{logdir=LogDirs,
 	[CBF || {N,CBF} <- UsrCfgs, N==Node],
     EvHandlers =  [{H,A} || {N,H,A} <- EvHs, N==Node],
     Include =  [I || {N,I} <- Incl, N==Node],
-    #opts{logdir = LogDir,
+    #opts{label = Label,
+	  logdir = LogDir,
 	  cover = Cover,
 	  config = ConfigFiles,
 	  event_handlers = EvHandlers,
@@ -1023,21 +1059,26 @@ delistify(E)   -> E.
 %%% @equiv ct:run/3
 run(TestDir, Suite, Cases) ->
     install([]),
-    do_run(tests(TestDir, Suite, Cases), []).
+    reformat_result(catch do_run(tests(TestDir, Suite, Cases), [])).
 
 %%%-----------------------------------------------------------------
 %%% @hidden
 %%% @equiv ct:run/2
 run(TestDir, Suite) when is_list(TestDir), is_integer(hd(TestDir)) ->
     install([]),
-    do_run(tests(TestDir, Suite), []).
+    reformat_result(catch do_run(tests(TestDir, Suite), [])).
 
 %%%-----------------------------------------------------------------
 %%% @hidden
 %%% @equiv ct:run/1
 run(TestDirs) ->
     install([]),
-    do_run(tests(TestDirs), []).
+    reformat_result(catch do_run(tests(TestDirs), [])).
+
+reformat_result({user_error,Reason}) ->
+    {error,Reason};
+reformat_result(Result) ->
+    Result.
 
 suite_to_test(Suite) ->
     {filename:dirname(Suite),list_to_atom(filename:rootname(filename:basename(Suite)))}.
@@ -1092,7 +1133,17 @@ do_run(Tests, Misc, LogDir) when is_list(Misc) ->
     do_run(Tests, [], Opts1#opts{logdir = LogDir}, []).
 
 do_run(Tests, Skip, Opts, Args) ->
-    #opts{cover = Cover} = Opts,
+    #opts{label = Label, cover = Cover} = Opts,
+
+    %% label - used by ct_logs
+    TestLabel =
+	if Label == undefined -> undefined;
+	   is_atom(Label)     -> atom_to_list(Label);
+	   is_list(Label)     -> Label;
+	   true               -> undefined
+	end,
+    application:set_env(common_test, test_label, TestLabel),
+
     case code:which(test_server) of
 	non_existing ->
 	    exit({error,no_path_to_test_server});
@@ -1156,10 +1207,19 @@ do_run(Tests, Skip, Opts, Args) ->
 			true ->
 			    SavedErrors = save_make_errors(SuiteMakeErrors),
 			    ct_repeat:log_loop_info(Args),
-			    {Tests1,Skip1} = final_tests(Tests,[],Skip,SavedErrors),
-			    R = do_run_test(Tests1, Skip1, Opts1),
-			    ct_util:stop(normal),
-			    R;
+
+			    {Tests1,Skip1} = final_tests(Tests,Skip,SavedErrors),
+
+			    R = (catch do_run_test(Tests1, Skip1, Opts1)),
+			    case R of
+				{EType,_} = Error when EType == user_error ;
+						       EType == error ->
+				    ct_util:stop(clean),
+				    exit(Error);
+				_ ->
+				    ct_util:stop(normal),
+				    R
+			    end;
 			false ->
 			    io:nl(),
 			    ct_util:stop(clean),
@@ -1316,8 +1376,22 @@ suite_tuples([{TestDir,Suite,_} | Tests]) when is_atom(Suite) ->
 suite_tuples([]) ->
     [].
 
-final_tests([{TestDir,Suites,_}|Tests],
-	    Final, Skip, Bad) when is_list(Suites), is_atom(hd(Suites)) ->
+final_tests(Tests, Skip, Bad) ->
+
+    %%! --- Thu Jun 24 15:47:27 2010 --- peppe was here!
+    %%! io:format(user, "FINAL0 = ~p~nSKIP0 = ~p~n", [Tests, Skip]),
+
+    {Tests1,Skip1} = final_tests1(Tests, [], Skip, Bad),
+    Skip2 = final_skip(Skip1, []),
+
+
+    %%! --- Thu Jun 24 15:47:27 2010 --- peppe was here!
+    %%! io:format(user, "FINAL1 = ~p~nSKIP1 = ~p~n", [Tests1, Skip2]),
+
+    {Tests1,Skip2}.
+
+final_tests1([{TestDir,Suites,_}|Tests], Final, Skip, Bad) when
+      is_list(Suites), is_atom(hd(Suites)) ->
 %     Separate =
 % 	fun(S,{DoSuite,Dont}) ->		
 % 		case lists:keymember({TestDir,S},1,Bad) of
@@ -1336,9 +1410,9 @@ final_tests([{TestDir,Suites,_}|Tests],
     Skip1 = [{TD,S,"Make failed"} || {{TD,S},_} <- Bad, S1 <- Suites,
 				     S == S1, TD == TestDir],
     Final1 = [{TestDir,S,all} || S <- Suites],
-    final_tests(Tests, lists:reverse(Final1)++Final, Skip++Skip1, Bad);
+    final_tests1(Tests, lists:reverse(Final1)++Final, Skip++Skip1, Bad);
 
-final_tests([{TestDir,all,all}|Tests], Final, Skip, Bad) ->
+final_tests1([{TestDir,all,all}|Tests], Final, Skip, Bad) ->
     MissingSuites =
 	case lists:keysearch({TestDir,all}, 1, Bad) of
 	    {value,{_,Failed}} ->
@@ -1348,26 +1422,58 @@ final_tests([{TestDir,all,all}|Tests], Final, Skip, Bad) ->
 	end,
     Missing = [{TestDir,S,"Make failed"} || S <- MissingSuites],
     Final1 = [{TestDir,all,all}|Final],
-    final_tests(Tests, Final1, Skip++Missing, Bad);
+    final_tests1(Tests, Final1, Skip++Missing, Bad);
 
-final_tests([{TestDir,Suite,Cases}|Tests],
-	    Final, Skip, Bad) when Cases==[]; Cases==all  ->
-    final_tests([{TestDir,[Suite],all}|Tests], Final, Skip, Bad);
+final_tests1([{TestDir,Suite,Cases}|Tests], Final, Skip, Bad) when
+      Cases==[]; Cases==all  ->
+    final_tests1([{TestDir,[Suite],all}|Tests], Final, Skip, Bad);
 
-final_tests([{TestDir,Suite,Cases}|Tests], Final, Skip, Bad) ->
+final_tests1([{TestDir,Suite,GrsOrCs}|Tests], Final, Skip, Bad) when
+      is_list(GrsOrCs) ->
     case lists:keymember({TestDir,Suite}, 1, Bad) of
-	false ->
-	    Do = {TestDir,Suite,Cases},
-	    final_tests(Tests, [Do|Final], Skip, Bad);
 	true ->
-	    Do = {TestDir,Suite,Cases},
-	    Skip1 = Skip ++ [{TestDir,Suite,Cases,"Make failed"}],
-	    final_tests(Tests, [Do|Final], Skip1, Bad)
+	    Skip1 = Skip ++ [{TestDir,Suite,all,"Make failed"}],
+	    final_tests1(Tests, [{TestDir,Suite,all}|Final], Skip1, Bad);
+	false ->
+	    GrsOrCs1 =
+		lists:flatmap(
+		  %% for now, only flat group defs are allowed as
+		  %% start options and test spec terms
+		  fun({all,all}) ->
+			  ct_framework:make_all_conf(TestDir,
+						      Suite, []);
+		     ({skipped,Group,TCs}) ->
+			  [ct_framework:make_conf(TestDir, Suite,
+						  Group, [skipped], TCs)];
+		     ({Group,TCs}) ->
+			  [ct_framework:make_conf(TestDir, Suite,
+						  Group, [], TCs)];
+		     (TC) ->
+			  [TC]
+		  end, GrsOrCs),
+	    Do = {TestDir,Suite,GrsOrCs1},
+	    final_tests1(Tests, [Do|Final], Skip, Bad)
     end;
 
-final_tests([], Final, Skip, _Bad) ->
+final_tests1([], Final, Skip, _Bad) ->
     {lists:reverse(Final),Skip}.
 
+final_skip([{TestDir,Suite,{all,all},Reason}|Skips], Final) ->
+    SkipConf =  ct_framework:make_conf(TestDir, Suite, all, [], all),
+    Skip = {TestDir,Suite,SkipConf,Reason},
+    final_skip(Skips, [Skip|Final]);
+
+final_skip([{TestDir,Suite,{Group,TCs},Reason}|Skips], Final) ->
+    Conf =  ct_framework:make_conf(TestDir, Suite, Group, [], TCs),
+    Skip = {TestDir,Suite,Conf,Reason},
+    final_skip(Skips, [Skip|Final]);
+
+final_skip([Skip|Skips], Final) ->
+    final_skip(Skips, [Skip|Final]);
+
+final_skip([], Final) ->
+    lists:reverse(Final).
+
 continue([]) ->
     true;
 continue(_MakeErrors) ->
@@ -1489,6 +1595,7 @@ do_run_test(Tests, Skip, Opts) ->
 		_ ->
 		    false
 	    end,
+
 	    %% let test_server expand the test tuples and count no of cases
 	    {Suites,NoOfCases} = count_test_cases(Tests, Skip),
 	    Suites1 = delete_dups(Suites),
@@ -1544,19 +1651,30 @@ count_test_cases(Tests, Skip) ->
     TSPid = test_server_ctrl:start_get_totals(SendResult),
     Ref = erlang:monitor(process, TSPid),
     add_jobs(Tests, Skip, #opts{}, []),
-    {Suites,NoOfCases} = count_test_cases1(length(Tests), 0, [], Ref),
+    Counted = (catch count_test_cases1(length(Tests), 0, [], Ref)),
     erlang:demonitor(Ref, [flush]),
-    test_server_ctrl:stop_get_totals(),
-    {Suites,NoOfCases}.
+    case Counted of
+	{error,{test_server_died}} = Error ->
+	    throw(Error);
+	{error,Reason} ->
+	    unlink(whereis(test_server_ctrl)),
+	    test_server_ctrl:stop(),
+	    throw({user_error,Reason});
+	Result ->
+	    test_server_ctrl:stop_get_totals(),
+	    Result
+    end.
 
 count_test_cases1(0, N, Suites, _) ->
     {lists:flatten(Suites), N};
 count_test_cases1(Jobs, N, Suites, Ref) ->
     receive
+	{_,{error,_Reason} = Error} ->
+	    throw(Error);
 	{no_of_cases,{Ss,N1}} ->
 	    count_test_cases1(Jobs-1, add_known(N,N1), [Ss|Suites], Ref);
-	{'DOWN', Ref, _, _, _} ->
-	    {[],0}
+	{'DOWN', Ref, _, _, Info} ->
+	    throw({error,{test_server_died,Info}})
     end.
 
 add_known(unknown, _) ->
@@ -1604,13 +1722,36 @@ add_jobs([{TestDir,Suite,all}|Tests], Skip, Opts, CleanUp) ->
 	    Error
     end;
 
-%% group
-add_jobs([{TestDir,Suite,[{GroupName,_Cases}]}|Tests], Skip, Opts, CleanUp) when
-      is_atom(GroupName) ->
-    add_jobs([{TestDir,Suite,GroupName}|Tests], Skip, Opts, CleanUp);
-add_jobs([{TestDir,Suite,{GroupName,_Cases}}|Tests], Skip, Opts, CleanUp) when
-      is_atom(GroupName) ->
-    add_jobs([{TestDir,Suite,GroupName}|Tests], Skip, Opts, CleanUp);
+%% group (= conf case in test_server)
+add_jobs([{TestDir,Suite,Confs}|Tests], Skip, Opts, CleanUp) when
+      element(1, hd(Confs)) == conf ->
+    Group = fun(Conf) -> proplists:get_value(name, element(2, Conf)) end,
+    TestCases = fun(Conf) -> element(4, Conf) end,
+    TCTestName = fun(all) -> "";
+		    ([C]) when is_atom(C) -> "." ++ atom_to_list(C);
+		    (Cs) when is_list(Cs) -> ".cases"
+		 end,
+    GrTestName =
+	case Confs of
+	    [Conf] ->
+		"." ++ atom_to_list(Group(Conf)) ++ TCTestName(TestCases(Conf));
+	    _ ->
+		".groups"
+	end,
+    TestName = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ GrTestName,
+    case maybe_interpret(Suite, init_per_group, Opts) of
+	ok ->
+	    case catch test_server_ctrl:add_conf_with_skip(TestName, Suite, Confs,
+							   skiplist(TestDir,Skip)) of
+		{'EXIT',_} ->
+		    CleanUp;
+		_ ->
+		    wait_for_idle(),
+		    add_jobs(Tests, Skip, Opts, [Suite|CleanUp])
+	    end;
+	Error ->
+	    Error
+    end;
 
 %% test case
 add_jobs([{TestDir,Suite,[Case]}|Tests], Skip, Opts, CleanUp) when is_atom(Case) ->