Add new option to label test runs

With the option 'label', the test run gets a user defined name that Common Test prints in the overview log files.

1 files changed, 250 insertions, 62 deletions

diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl
index fb7f3fc9a9..c3840d0425 100644
--- a/lib/common_test/test/ct_testspec_1_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_1_SUITE.erl
@@ -61,9 +61,18 @@ all(doc) ->
 
 all(suite) ->
     [all_suites, skip_all_suites,
-     suite, skip_suite].
-%     cases_1, skip_cases_1,
-%     groups_1, skip_groups_1].
+     suite, skip_suite,
+     all_testcases, skip_all_testcases,
+     testcase, skip_testcase,
+     all_groups, skip_all_groups,
+     group, skip_group,
+     group_all_testcases, skip_group_all_testcases,
+     group_testcase, skip_group_testcase,
+     topgroup,
+     subgroup, skip_subgroup,
+     subgroup_all_testcases, skip_subgroup_all_testcases,
+     subgroup_testcase, skip_subgroup_testcase,
+     only_skip].
 
 %%--------------------------------------------------------------------
 %% TEST CASES
@@ -73,102 +82,279 @@ all(suite) ->
 %%%
 
 all_suites(Config) when is_list(Config) ->
-    Self = all_suites,
     DataDir = ?config(data_dir, Config),
 
     TestDir = filename:join(DataDir, "suites_1"),
-    TestSpec = [{suites,TestDir,all}],
-    SpecFile = create_spec_file(?config(priv_dir, Config),
-				Self, TestSpec),
-
-    {Opts,ERPid} = setup({spec,SpecFile}, Config),
-    ok = ct_test_support:run(Opts, Config),
-    ok = ct_test_support:run(ct, run_testspec, [TestSpec], Config),
-    Events = ct_test_support:get_events(ERPid, Config),
+    TestSpec = [{label,"all_suites"},
+		{suites,TestDir,all}],
 
-    ct_test_support:log_events(Self,
-			       reformat(Events, ?eh),
-			       ?config(priv_dir, Config)),
-
-    TestEvents = events_to_check(Self),
-    ok = ct_test_support:verify_events(TestEvents, Events, Config).
+    setup_and_execute(all_suites, TestSpec, Config).
 
 skip_all_suites(Config) when is_list(Config) ->
-    Self = skip_all_suites,
     DataDir = ?config(data_dir, Config),
 
     TestDir = filename:join(DataDir, "suites_1"),
-    TestSpec = [{suites,TestDir,all},
+    TestSpec = [{label,skip_all_suites},
+		{suites,TestDir,all},
 		{skip_suites,TestDir,all,"SKIPPED!"}],
-    SpecFile = create_spec_file(?config(priv_dir, Config),
-				Self, TestSpec),
 
-    {Opts,ERPid} = setup({spec,SpecFile}, Config),
-    ok = ct_test_support:run(Opts, Config),
-    ok = ct_test_support:run(ct, run_testspec, [TestSpec], Config),
-    Events = ct_test_support:get_events(ERPid, Config),
-
-    ct_test_support:log_events(Self,
-			       reformat(Events, ?eh),
-			       ?config(priv_dir, Config)),
-
-    TestEvents = events_to_check(Self),
-    ok = ct_test_support:verify_events(TestEvents, Events, Config).
+    setup_and_execute(skip_all_suites, TestSpec, Config).
 
 %%%-----------------------------------------------------------------
 %%%
 
 suite(Config) when is_list(Config) ->
-    Self = suite,
     DataDir = ?config(data_dir, Config),
 
     TestDir = filename:join(DataDir, "suites_1"),
-    TestSpec = [{suites,TestDir,simple_1_SUITE}],
-    SpecFile = create_spec_file(?config(priv_dir, Config),
-				Self, TestSpec),
+    TestSpec = [{label,undefined},
+		{suites,TestDir,simple_1_SUITE}],
 
-    {Opts,ERPid} = setup({spec,SpecFile}, Config),
-    ok = ct_test_support:run(Opts, Config),
-    ok = ct_test_support:run(ct, run_testspec, [TestSpec], Config),
-    Events = ct_test_support:get_events(ERPid, Config),
-
-    ct_test_support:log_events(Self,
-			       reformat(Events, ?eh),
-			       ?config(priv_dir, Config)),
-
-    TestEvents = events_to_check(Self),
-    ok = ct_test_support:verify_events(TestEvents, Events, Config).
+    setup_and_execute(suite, TestSpec, Config).
 
 skip_suite(Config) when is_list(Config) ->
-    Self = skip_suite,
     DataDir = ?config(data_dir, Config),
 
     TestDir = filename:join(DataDir, "suites_1"),
     TestSpec = [{suites,TestDir,[simple_1_SUITE,simple_2_SUITE]},
 		{skip_suites,TestDir,simple_1_SUITE,"SKIPPED!"}],
-    SpecFile = create_spec_file(?config(priv_dir, Config),
-				Self, TestSpec),
 
-    {Opts,ERPid} = setup({spec,SpecFile}, Config),
-    ok = ct_test_support:run(Opts, Config),
-    ok = ct_test_support:run(ct, run_testspec, [TestSpec], Config),
-    Events = ct_test_support:get_events(ERPid, Config),
+    setup_and_execute(skip_suite, TestSpec, Config).
 
-    ct_test_support:log_events(Self,
-			       reformat(Events, ?eh),
-			       ?config(priv_dir, Config)),
+%%%-----------------------------------------------------------------
+%%%
 
-    TestEvents = events_to_check(Self),
-    ok = ct_test_support:verify_events(TestEvents, Events, Config).
+all_testcases(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir = filename:join(DataDir, "suites_1"),
+    TestSpec = [{cases,TestDir,simple_1_SUITE,all}],
+
+    setup_and_execute(all_testcases, TestSpec, Config).
+
+skip_all_testcases(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir = filename:join(DataDir, "suites_1"),
+    TestSpec = [{suites,TestDir,[simple_1_SUITE]},
+		{skip_cases,TestDir,simple_1_SUITE,all,"SKIPPED!"}],
+
+    setup_and_execute(skip_all_testcases, TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+testcase(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir = filename:join(DataDir, "suites_1"),
+    TestSpec = [{cases,TestDir,simple_1_SUITE,tc1}],
+
+    setup_and_execute(testcase, TestSpec, Config).
+
+skip_testcase(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir = filename:join(DataDir, "suites_1"),
+    TestSpec = [{cases,TestDir,simple_1_SUITE,[tc1,tc2]},
+		{cases,TestDir,simple_2_SUITE,[tc2,tc1]},
+		{skip_cases,TestDir,simple_1_SUITE,[tc1],"SKIPPED!"},
+		{skip_cases,TestDir,simple_2_SUITE,tc2,"SKIPPED!"}],
+
+    setup_and_execute(skip_testcase, TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+all_groups(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir,groups_11_SUITE,all}],
+
+    setup_and_execute(all_groups, TestSpec, Config).
+
+skip_all_groups(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir,groups_11_SUITE,all},
+		{skip_groups,TestDir,groups_11_SUITE,all,"SKIPPED!"}],
+
+    setup_and_execute(skip_all_groups, TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+group(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
 
+    TestDir = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir,groups_11_SUITE,test_group_1a}],
+
+    setup_and_execute(group, TestSpec, Config).
+
+skip_group(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir,groups_11_SUITE,[test_group_1a,
+						 test_group_1b]},
+		{skip_groups,TestDir,groups_11_SUITE,
+		 [test_group_1b,test_group_2],"SKIPPED!"}],
+
+    setup_and_execute(skip_group, TestSpec, Config).
+
+
+%%%-----------------------------------------------------------------
+%%%
+
+group_all_testcases(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir,groups_11_SUITE,test_group_1a,{cases,all}}],
+
+    setup_and_execute(group_all_testcases, TestSpec, Config).
+
+skip_group_all_testcases(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir,groups_11_SUITE,[test_group_1a,
+						 test_group_1b]},
+		{skip_groups,TestDir,groups_11_SUITE,
+		 test_group_1b,{cases,all},"SKIPPED!"},
+		{skip_groups,TestDir,groups_11_SUITE,
+		 test_group_1a,{cases,all},"SKIPPED!"}],
+
+    setup_and_execute(skip_group_all_testcases, TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+group_testcase(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir,groups_11_SUITE,test_group_1a,{cases,testcase_1a}}],
+
+    setup_and_execute(group_testcase, TestSpec, Config).
+
+skip_group_testcase(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir,groups_11_SUITE,test_group_1a,
+		 {cases,[testcase_1a,testcase_1b]}},
+		{groups,TestDir,groups_11_SUITE,test_group_1b,
+		 {cases,[testcase_1a,testcase_1b]}},
+		{skip_groups,TestDir,groups_11_SUITE,
+		 test_group_1a,{cases,testcase_1b},"SKIPPED!"},
+		{skip_groups,TestDir,groups_11_SUITE,
+		 test_group_1b,{cases,[testcase_1a]},"SKIPPED!"}],
+
+    setup_and_execute(skip_group_testcase, TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+topgroup(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_2}],
+
+    setup_and_execute(topgroup, TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+subgroup(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_3}],
+
+    setup_and_execute(subgroup, TestSpec, Config).
+
+skip_subgroup(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir,groups_12_SUITE,[test_group_2]},
+		{skip_groups,TestDir,groups_12_SUITE,
+		 test_group_3,"SKIPPED!"}],
+
+    setup_and_execute(skip_subgroup, TestSpec, Config).
 
 %%%-----------------------------------------------------------------
 %%%
 
+subgroup_all_testcases(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir,groups_12_SUITE,
+		 test_group_5,{cases,all}}],
+
+    setup_and_execute(subgroup_all_testcases, TestSpec, Config).
+
+skip_subgroup_all_testcases(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_4},
+		{skip_groups,TestDir,groups_12_SUITE,
+		 test_group_5,{cases,all},"SKIPPED!"}],
+
+    setup_and_execute(skip_subgroup_all_testcases, TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+subgroup_testcase(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir,groups_12_SUITE,
+		 test_group_7,{cases,testcase_7a}}],
+
+    setup_and_execute(subgroup_testcase, TestSpec, Config).
+
+skip_subgroup_testcase(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_5},
+		{skip_groups,TestDir,groups_12_SUITE,
+		 test_group_7,{cases,[testcase_7a,testcase_7b]},"SKIPPED!"}],
+
+    setup_and_execute(skip_subgroup_testcase, TestSpec, Config).
+
 %%%-----------------------------------------------------------------
 %%% HELP FUNCTIONS
 %%%-----------------------------------------------------------------
 
+setup_and_execute(TCName, TestSpec, Config) ->
+    SpecFile = create_spec_file(?config(priv_dir, Config),
+				TCName, TestSpec),
+    TestTerms =
+	case lists:keymember(label, 1, TestSpec) of
+	    true -> [{spec,SpecFile}];
+	    false -> [{spec,SpecFile},{label,TCName}]
+	end,
+    {Opts,ERPid} = setup(TestTerms, Config),
+    ok = ct_test_support:run(Opts, Config),
+    ok = ct_test_support:run(ct, run_testspec, [TestSpec], Config),
+    Events = ct_test_support:get_events(ERPid, Config),
+
+    ct_test_support:log_events(TCName,
+			       reformat(Events, ?eh),
+			       ?config(priv_dir, Config)),
+
+    TestEvents = events_to_check(TCName),
+    ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
 create_spec_file(SpecDir, TCName, TestSpec) ->
     FileName = filename:join(SpecDir,
 			     atom_to_list(TCName)++".spec"),
@@ -179,11 +365,13 @@ create_spec_file(SpecDir, TCName, TestSpec) ->
 	      [FileName,FileName]),
     FileName.
 
-setup(Test, Config) ->
+setup(Test, Config) when is_tuple(Test) ->
+    setup([Test], Config);
+setup(Tests, Config) ->
     Opts0 = ct_test_support:get_opts(Config),
     Level = ?config(trace_level, Config),
     EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
-    Opts = Opts0 ++ [Test,{event_handler,{?eh,EvHArgs}}],
+    Opts = Opts0 ++ Tests ++ [{event_handler,{?eh,EvHArgs}}],
     ERPid = ct_test_support:start_event_receiver(Config),
     {Opts,ERPid}.