Add documentation for group specifications and for the label option

3 files changed, 104 insertions, 47 deletions

diff --git a/lib/common_test/doc/src/run_test.xml b/lib/common_test/doc/src/run_test.xml
index d609c4287f..2f0a94afba 100644
--- a/lib/common_test/doc/src/run_test.xml
+++ b/lib/common_test/doc/src/run_test.xml
@@ -45,6 +45,11 @@
     flags start an Erlang node prepared for running Common Test in a
     particular mode.</p>
 
+  <p>There is an interface function that corresponds to this program,
+    called <c>ct:run_test/1</c>, for starting Common Test from the Erlang
+    shell (or an Erlang program). Please see the <c>ct</c> man page for
+    details.</p>
+
   <p><c>run_test</c> also accepts Erlang emulator flags. These are used
     when <c>run_test</c> calls <c>erl</c> to start the Erlang node
     (making it possible to e.g. add directories to the code server path,
@@ -72,6 +77,8 @@
     <p>it prints all valid start flags to stdout.</p>
   </description>
 
+  <marker id="run_test"></marker>
+
   <section>
     <title>Run tests from command line</title>
     <pre>
@@ -83,6 +90,7 @@
 	[-userconfig CallbackModule1 ConfigString1 and CallbackModule2
 	 ConfigString2 and .. and CallbackModuleN ConfigStringN]
 	[-decrypt_key Key] | [-decrypt_file KeyFile]
+	[-label Label]
 	[-logdir LogDir]
 	[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]
 	[-stylesheet CSSFile]
@@ -107,6 +115,7 @@
 	[-userconfig CallbackModule1 ConfigString1 and CallbackModule2
          ConfigString2 and .. and CallbackModuleN ConfigStringN]
 	[-decrypt_key Key] | [-decrypt_file KeyFile]
+	[-label Label]
 	[-logdir LogDir]
 	[-allow_user_terms]
 	[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]
@@ -128,12 +137,12 @@
     <title>Run tests in web based GUI</title>
     <pre>
         run_test -vts [-browser Browser]
+        [-dir TestDir1 TestDir2 .. TestDirN] |
+        [-suite Suite [[-group Group] [-case Case]]]
 	[-config ConfigFile1 ConfigFile2 .. ConfigFileN]
 	[-userconfig CallbackModule1 ConfigString1 and CallbackModule2
          ConfigString2 and .. and CallbackModuleN ConfigStringN]
 	[-decrypt_key Key] | [-decrypt_file KeyFile]
-        [-dir TestDir1 TestDir2 .. TestDirN] |
-        [-suite Suite [[-group Group] [-case Case]]]
 	[-include InclDir1 InclDir2 .. InclDirN]
 	[-no_auto_compile]
 	[-muliply_timetraps Multiplier]
diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml
index 207df7f5b5..1efff25f5b 100644
--- a/lib/common_test/doc/src/run_test_chapter.xml
+++ b/lib/common_test/doc/src/run_test_chapter.xml
@@ -21,7 +21,7 @@
 
     </legalnotice>
 
-    <title>Running Test Suites</title>
+    <title>Running Tests</title>
     <prepared>Peter Andersson, Kenneth Lundin</prepared>
     <docno></docno>
     <date></date>
@@ -130,6 +130,8 @@
     <p>Other flags that may be used with <c>run_test</c>:</p>
     <list>
       <item><c><![CDATA[-logdir <dir>]]></c>, specifies where the HTML log files are to be written.</item>
+      <item><c><![CDATA[-label <name_of_test_run>]]></c>, associates the test run with a name that gets printed
+	in the overview HTML log files.</item>
       <item><c>-refresh_logs</c>, refreshes the top level HTML index files.</item>
       <item><c>-vts</c>, start web based GUI (see below).</item>
       <item><c>-shell</c>, start interactive shell mode (see below).</item>
@@ -335,43 +337,72 @@
     <marker id="test_specifications"></marker>
     <title>Using test specifications</title>
     
-    <p>The most expressive way to specify what to test is to use a so 
-      called test specification. A test specification is a sequence of 
-      Erlang terms. The terms may be declared in a text file or passed 
-      to the test server at runtime as a list (see <c>run_testspec/1</c>
-      in the manual page for <c>ct</c>). There are two general types 
-      of terms: configuration terms and test specification terms.</p>
-    <p>With configuration terms it is possible to import configuration 
-      data (similar to <c>run_test -config</c>), specify HTML log 
-      directories (similar to <c>run_test -logdir</c>), give aliases 
-      to test nodes and test directories (to make a specification
-      easier to read and maintain), enable code coverage analysis
-      (see the <seealso marker="cover_chapter#cover">Code Coverage 
-       Analysis</seealso> chapter) and specify event_handler plugins 
+    <p>The most flexible way to specify what to test, is to use a so
+      called test specification. A test specification is a sequence of
+      Erlang terms. The terms may be declared in a text file or passed
+      to the test server at runtime as a list
+      (see <c>run_testspec/1</c> in the manual page
+      for <c>ct</c>). There are two general types of terms:
+      configuration terms and test specification terms.</p>
+    <p>With configuration terms it is possible to e.g. label the test
+      run (similar to <c>run_test -label</c>), evaluate arbitrary expressions
+      before starting a test, import configuration
+      data (similar to
+      <c>run_test -config/-userconfig</c>), specify HTML log directories (similar
+      to
+      <c>run_test -logdir</c>), give aliases to test nodes and test
+      directories (to make a specification easier to read and
+      maintain), enable code coverage analysis (see
+      the <seealso marker="cover_chapter#cover">Code Coverage
+      Analysis</seealso> chapter) and specify event_handler plugins
       (see the <seealso marker="event_handler_chapter#event_handling">
-       Event Handling</seealso> chapter). There is also a term
-       for specifying include directories that should be passed on
-       to the compiler when automatic compilation is performed 
-       (similar to <c>run_test -include</c>, see above).</p>
-    <p>With test specification terms it is possible to state exactly which
-      tests should run and in which order. A test term specifies either
-      one or more suites or one or more test cases. An arbitrary number of test 
-      terms may be declared in sequence. A test term can also specify one or 
-      more test suites or test cases to be skipped. Skipped suites and cases 
-      are not executed and show up in the HTML test log as SKIPPED.</p>
-
-    <note><p>It is not yet possible to specify test case groups in
-      test specifications. This will be supported in a soon upcoming 
-      release.</p></note>
+      Event Handling</seealso> chapter). There is also a term for
+      specifying include directories that should be passed on to the
+      compiler when automatic compilation is performed (similar
+      to <c>run_test -include</c>, see above).</p>
+    <p>With test specification terms it is possible to state exactly
+      which tests should run and in which order. A test term specifies
+      either one or more suites, one or more test case groups, or one
+      or more test cases in a group or suite.</p>
+    <p>An arbitrary number of test terms may be declared in sequence.
+      Common Test will compile the terms into one or more tests to be
+      performed in one resulting test run. Note that a term that
+      specifies a set of test cases will "swallow" one that only
+      specifies a subset of these cases. E.g. the result of merging
+      one term that specifies that all cases in suite S should be
+      executed, with another term specifying only test case X and Y in
+      S, is a test of all cases in S. However, if a term specifying
+      test case X and Y in S is merged with a term specifying case Z
+      in S, the result is a test of X, Y and Z in S.</p>
+    <p>A test term can also specify one or more test suites, groups,
+      or test cases to be skipped. Skipped suites, groups and cases
+      are not executed and show up in the HTML test log files as
+      SKIPPED.</p>
+    <p>When a test case group is specified, the resulting test
+      executes the
+      <c>init_per_group</c> function, followed by all test cases and
+      sub groups (including their configuration functions), and
+      finally the <c>end_per_group</c> function. Also if particular
+      test cases in a group are specified, <c>init_per_group</c>
+      and <c>end_per_group</c> for the group in question are
+      called. If a group which is defined (in <c>Suite:group/0</c>) to
+      be a sub group of another group, is specified (or particular test
+      cases of a sub group are), Common Test will call the configuration
+      functions for the top level groups as well as for the sub group
+      in question (making it possible to pass configuration data all
+      the way from <c>init_per_suite</c> down to the test cases in the
+      sub group).</p>
 
     <p>Below is the test specification syntax. Test specifications can
-      be used to run tests both in a single test host environment and in 
-      a distributed Common Test environment (Large Scale Testing). The init term,
-      as well as node parameters, are only relevant in the latter (see the
-      <seealso marker="ct_master_chapter#test_specifications">Large Scale Testing</seealso>
-      chapter for information). For details on the event_handler term, see the
-      <seealso marker="event_handler_chapter#event_handling">Event Handling</seealso>
-      chapter.</p>
+      be used to run tests both in a single test host environment and
+      in a distributed Common Test environment (Large Scale
+      Testing). The node parameters in the init term are only
+      relevant in the latter (see the
+      <seealso marker="ct_master_chapter#test_specifications">Large
+      Scale Testing</seealso> chapter for information). For details on
+      the event_handler term, see the
+      <seealso marker="event_handler_chapter#event_handling">Event
+      Handling</seealso> chapter.</p>
       <p>Config terms:</p>
     <pre>
       {node, NodeAlias, Node}.
@@ -379,11 +410,17 @@
       {init, InitOptions}.
       {init, [NodeAlias], InitOptions}.
 
+      {label, Label}.
+      {label, NodeRefs, Label}.
+
       {multiply_timetraps, N}.
+      {multiply_timetraps, NodeRefs, N}.
+
       {scale_timetraps, Bool}.
+      {scale_timetraps, NodeRefs, Bool}.
  
       {cover, CoverSpecFile}.
-      {cover, NodeRef, CoverSpecFile}.
+      {cover, NodeRefs, CoverSpecFile}.
       
       {include, IncludeDirs}.
       {include, NodeRefs, IncludeDirs}.
@@ -409,6 +446,12 @@
       {suites, DirRef, Suites}.                                
       {suites, NodeRefs, DirRef, Suites}.
       
+      {groups, DirRef, Suite, Groups}.
+      {groups, NodeRefsDirRef, Suite, Groups}.
+
+      {groups, DirRef, Suite, Group, {cases,Cases}}.
+      {groups, NodeRefsDirRef, Suite, Group, {cases,Cases}}.
+
       {cases, DirRef, Suite, Cases}.                           
       {cases, NodeRefs, DirRef, Suite, Cases}.
 
@@ -437,6 +480,9 @@
       InitArgs      = [term()]
       DirRef        = DirAlias | Dir
       Suites        = atom() | [atom()] | all
+      Suite         = atom()
+      Groups        = atom() | [atom()] | all
+      Group         = atom()
       Cases         = atom() | [atom()] | all
       Comment       = string() | ""
     </pre>
diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl
index 0d82a86e7d..8ae175f10d 100644
--- a/lib/common_test/src/ct.erl
+++ b/lib/common_test/src/ct.erl
@@ -138,10 +138,10 @@ run(TestDirs) ->
 %%%-----------------------------------------------------------------
 %%% @spec run_test(Opts) -> Result
 %%%   Opts = [OptTuples]
-%%%   OptTuples = {config,CfgFiles} | {dir,TestDirs} | {suite,Suites} |
-%%%               {userconfig, UserConfig} |
-%%%               {testcase,Cases} | {group,Groups} | {spec,TestSpecs} |
-%%%               {allow_user_terms,Bool} | {logdir,LogDir} | 
+%%%   OptTuples = {dir,TestDirs} | {suite,Suites} | {group,Groups} |
+%%%               {testcase,Cases} | {spec,TestSpecs} | {label,Label} |
+%%%               {config,CfgFiles} | {userconfig, UserConfig} |
+%%%               {allow_user_terms,Bool} | {logdir,LogDir} |
 %%%               {silent_connections,Conns} | {stylesheet,CSSFile} |
 %%%               {cover,CoverSpecFile} | {step,StepOpts} |
 %%%               {event_handler,EventHandlers} | {include,InclDirs} |
@@ -149,15 +149,16 @@ run(TestDirs) ->
 %%%               {repeat,N} | {duration,DurTime} | {until,StopTime} |
 %%%               {force_stop,Bool} | {decrypt,DecryptKeyOrFile} |
 %%%               {refresh_logs,LogDir} | {basic_html,Bool}
-%%%   CfgFiles = [string()] | string()
-%%%   UserConfig = [{CallbackMod,CfgStrings}] | {CallbackMod,CfgStrings}
-%%%   CallbackMod = atom()
-%%%   CfgStrings = [string()] | string()
 %%%   TestDirs = [string()] | string()
 %%%   Suites = [string()] | string()
 %%%   Cases = [atom()] | atom()
 %%%   Groups = [atom()] | atom()
 %%%   TestSpecs = [string()] | string()
+%%%   Label = string() | atom()
+%%%   CfgFiles = [string()] | string()
+%%%   UserConfig = [{CallbackMod,CfgStrings}] | {CallbackMod,CfgStrings}
+%%%   CallbackMod = atom()
+%%%   CfgStrings = [string()] | string()
 %%%   LogDir = string()
 %%%   Conns = all | [atom()]
 %%%   CSSFile = string()
@@ -177,7 +178,8 @@ run(TestDirs) ->
 %%%   DecryptFile = string()
 %%%   Result = [TestResult] | {error,Reason}
 %%% @doc Run tests as specified by the combination of options in <code>Opts</code>.
-%%% The options are the same as those used with the <code>run_test</code> program.
+%%% The options are the same as those used with the
+%%% <seealso marker="run_test#run_test"><code>run_test</code></seealso> program.
 %%% Note that here a <code>TestDir</code> can be used to point out the path to 
 %%% a <code>Suite</code>. Note also that the option <code>testcase</code>
 %%% corresponds to the <code>-case</code> option in the <code>run_test</code>