Fix error with group term in ct:run_test/1

Also some new test cases on sequence groups have been added.

3 files changed, 102 insertions, 30 deletions

diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index c5bfd01642..5447b9c8fc 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -879,8 +879,10 @@ run_dir(Opts = #opts{logdir = LogDir,
 	    case lists:keysearch(suite, 1, StartOpts) of
 		{value,{_,Suite}} when is_integer(hd(Suite)) ; is_atom(Suite) ->
 		    {Dir,Mod} = S2M(Suite),
-		    case listify(proplists:get_value(group, StartOpts, [])) ++
-			 listify(proplists:get_value(testcase, StartOpts, [])) of
+		    case groups_and_cases(proplists:get_value(group, StartOpts),
+					  proplists:get_value(testcase, StartOpts)) of
+			Error = {error,_} ->
+			    exit(Error);
 			[] ->
 			    reformat_result(catch do_run(tests(Dir, listify(Mod)),
 							 [], Opts1, StartOpts));
@@ -900,8 +902,10 @@ run_dir(Opts = #opts{logdir = LogDir,
 		    Mod = if is_atom(Suite) -> Suite;
 			     true -> list_to_atom(Suite)
 			  end,
-		    case listify(proplists:get_value(group, StartOpts, [])) ++
-			 listify(proplists:get_value(testcase, StartOpts, [])) of
+		    case groups_and_cases(proplists:get_value(group, StartOpts),
+					  proplists:get_value(testcase, StartOpts)) of
+			Error = {error,_} ->
+			    exit(Error);
 			[] ->
 			    reformat_result(catch do_run(tests(Dir, listify(Mod)),
 							 [], Opts1, StartOpts));
@@ -1087,14 +1091,16 @@ groups_and_cases(Gs, Cs) when ((Gs == undefined) or (Gs == [])) and
 			      ((Cs == undefined) or (Cs == [])) ->
     [];
 groups_and_cases(Gs, Cs) when Gs == undefined ; Gs == [] ->
-    [list_to_atom(C) || C <- Cs];
+    [ensure_atom(C) || C <- listify(Cs)];
 groups_and_cases(Gs, Cs) when Cs == undefined ; Cs == [] ->
-    [{list_to_atom(G),all} || G <- Gs];
+    [{ensure_atom(G),all} || G <- listify(Gs)];
+groups_and_cases(G, Cs) when is_atom(G) ->
+    [{G,[ensure_atom(C) || C <- listify(Cs)]}];
 groups_and_cases([G], Cs) ->
-    [{list_to_atom(G),[list_to_atom(C) || C <- Cs]}];
+    [{ensure_atom(G),[ensure_atom(C) || C <- listify(Cs)]}];
 groups_and_cases([_,_|_] , Cs) when Cs =/= [] ->
     {error,multiple_groups_and_cases};
-groups_and_cases(_Gs, _Cs) ->
+groups_and_cases(Gs, Cs) ->
     {error,incorrect_group_or_case_option}.
 
 tests(TestDir, Suites, []) when is_list(TestDir), is_integer(hd(TestDir)) ->
diff --git a/lib/common_test/test/ct_sequence_1_SUITE.erl b/lib/common_test/test/ct_sequence_1_SUITE.erl
index 55aca764fb..f7cb984fc6 100644
--- a/lib/common_test/test/ct_sequence_1_SUITE.erl
+++ b/lib/common_test/test/ct_sequence_1_SUITE.erl
@@ -57,7 +57,9 @@ end_per_testcase(TestCase, Config) ->
     ct_test_support:end_per_testcase(TestCase, Config).
 
 all(suite) ->
-    [subgroup].
+    [subgroup_return_fail,
+     subgroup_init_fail,
+     subgroup_after_failed_case].
 
 %%--------------------------------------------------------------------
 %% TEST CASES
@@ -66,31 +68,51 @@ all(suite) ->
 %%%-----------------------------------------------------------------
 %%%
 
-subgroup(Config) when is_list(Config) ->
-    DataDir = ?config(data_dir, Config),
+subgroup_return_fail(Config) when is_list(Config) ->
+    execute(subgroup_return_fail,
+	    "subgroups_1_SUITE", subgroup_return_fail,
+	    Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+subgroup_init_fail(Config) when is_list(Config) ->
+    execute(subgroup_init_fail,
+	    "subgroups_1_SUITE", subgroup_init_fail,
+	    Config).
+
+%%%-----------------------------------------------------------------
+%%%
 
-    Suite = filename:join(DataDir, "subgroups_1_SUITE"),
+subgroup_after_failed_case(Config) when is_list(Config) ->
+    execute(subgroup_after_failed_case,
+	    "subgroups_1_SUITE", subgroup_after_failed_case,
+	    Config).
 
-    {Opts,ERPid} = setup({suite,Suite}, Config),
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+
+execute(TestCase, SuiteName, Group, Config) ->
+    DataDir = ?config(data_dir, Config),
+    Suite = filename:join(DataDir, SuiteName),
+
+    {Opts,ERPid} = setup([{suite,Suite},{group,Group},{label,TestCase}], Config),
     ok = ct_test_support:run(Opts, Config),
     Events = ct_test_support:get_events(ERPid, Config),
 
-    ct_test_support:log_events(subgroup,
+    ct_test_support:log_events(TestCase,
 			       reformat(Events, ?eh),
 			       ?config(priv_dir, Config)),
 
-    TestEvents = events_to_check(subgroup),
+    TestEvents = events_to_check(TestCase),
     ok = ct_test_support:verify_events(TestEvents, Events, Config).
 
-%%%-----------------------------------------------------------------
-%%% HELP FUNCTIONS
-%%%-----------------------------------------------------------------
-
 setup(Test, Config) ->
     Opts0 = ct_test_support:get_opts(Config),
     Level = ?config(trace_level, Config),
     EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
-    Opts = Opts0 ++ [Test,{event_handler,{?eh,EvHArgs}}],
+    Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}} | Test],
     ERPid = ct_test_support:start_event_receiver(Config),
     {Opts,ERPid}.
 
@@ -111,7 +133,23 @@ events_to_check(_, 0) ->
 events_to_check(Test, N) ->
     test_events(Test) ++ events_to_check(Test, N-1).
 
-test_events(subgroup) ->
+test_events(subgroup_return_fail) ->
+    [
+     {?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+     {?eh,test_done,{'DEF','STOP_TIME'}},
+     {?eh,stop_logging,[]}
+    ];
+
+test_events(subgroup_init_fail) ->
+    [
+     {?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+     {?eh,test_done,{'DEF','STOP_TIME'}},
+     {?eh,stop_logging,[]}
+    ];
+
+test_events(subgroup_after_failed_case) ->
     [
      {?eh,start_logging,{'DEF','RUNDIR'}},
      {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
diff --git a/lib/common_test/test/ct_sequence_1_SUITE_data/subgroups_1_SUITE.erl b/lib/common_test/test/ct_sequence_1_SUITE_data/subgroups_1_SUITE.erl
index 359d65816c..a25dbe772d 100644
--- a/lib/common_test/test/ct_sequence_1_SUITE_data/subgroups_1_SUITE.erl
+++ b/lib/common_test/test/ct_sequence_1_SUITE_data/subgroups_1_SUITE.erl
@@ -23,12 +23,25 @@
 -include_lib("common_test/include/ct.hrl").
 
 all() ->
-    [{group, test}].
+    [{group, subgroup_return_fail},
+     {group, subgroup_init_fail},
+     {group, subgroup_after_failed_case}].
 
 groups() ->
-    [{failing_group, [], [failing_tc]},
+    [{return_fail, [], [failing_tc]},
+     {fail_init, [], [ok_tc]},
      {ok_group, [], [ok_tc]},
-     {test, [sequence], [{group, failing_group}, {group, ok_group}]}].
+
+     {subgroup_return_fail, [sequence], [{group, return_fail}, {group, ok_group}]},
+
+     {subgroup_init_fail, [sequence], [{group, fail_init}, {group, ok_group}]},
+
+     {subgroup_after_failed_case, [sequence], [failing_tc, {group, ok_group}]}
+    ].
+
+failed_subgroup(subgroup_return_fail) -> return_fail;
+failed_subgroup(subgroup_init_fail) -> fail_init;
+failed_subgroup(_) -> undefined.
 
 init_per_suite(Config) ->
     Config.
@@ -36,21 +49,36 @@ init_per_suite(Config) ->
 end_per_suite(_Config) ->
     ok.
 
-init_per_group(_Group, Config) ->
-    Config.
+init_per_group(fail_init, Config) ->
+    ct:comment(fail_init),
+    exit(init_per_group_fails_on_purpose);
+
+init_per_group(Group, Config) ->
+    ct:comment(Group),
+    [{Group,failed_subgroup(Group)} | Config].
+
+end_per_group(subgroup_after_failed_case, Config) ->
+    ct:comment(subgroup_after_failed_case),
+    Status = ?config(tc_group_result, Config),
+    [{subgroups_1_SUITE,failing_tc}] = proplists:get_value(failed, Status),
+    {return_group_result,failed};
 
-end_per_group(test, Config) ->
+end_per_group(Group, Config) when Group == subgroup_return_fail;
+				  Group == subgroup_init_fail ->
+    ct:comment(Group),
     Status = ?config(tc_group_result, Config),
     Failed = proplists:get_value(failed, Status),
-    true = lists:member({group_result,failing_group}, Failed),
+    true = lists:member({group_result,?config(Group,Config)}, Failed),
     {return_group_result,failed};
 
-end_per_group(failing_group, Config) ->
+end_per_group(return_fail, Config) ->
+    ct:comment(return_fail),
     Status = ?config(tc_group_result, Config),
     [{subgroups_1_SUITE,failing_tc}] = proplists:get_value(failed, Status),
     {return_group_result,failed};
 
-end_per_group(_Group, _Config) ->
+end_per_group(Group, _Config) ->
+    ct:comment(Group),
     ok.
 
 init_per_testcase(_TestCase, Config) ->