From 2c3e05dbf52c1e682bf23bfb06664420433e1774 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Fri, 2 Jul 2010 11:46:53 +0200 Subject: Add documentation for group specifications and for the label option --- lib/common_test/doc/src/run_test.xml | 13 ++- lib/common_test/doc/src/run_test_chapter.xml | 118 +++++++++++++++++++-------- lib/common_test/src/ct.erl | 20 +++-- 3 files changed, 104 insertions(+), 47 deletions(-) (limited to 'lib') diff --git a/lib/common_test/doc/src/run_test.xml b/lib/common_test/doc/src/run_test.xml index d609c4287f..2f0a94afba 100644 --- a/lib/common_test/doc/src/run_test.xml +++ b/lib/common_test/doc/src/run_test.xml @@ -45,6 +45,11 @@ flags start an Erlang node prepared for running Common Test in a particular mode.

+

There is an interface function that corresponds to this program, + called ct:run_test/1, for starting Common Test from the Erlang + shell (or an Erlang program). Please see the ct man page for + details.

+

run_test also accepts Erlang emulator flags. These are used when run_test calls erl to start the Erlang node (making it possible to e.g. add directories to the code server path, @@ -72,6 +77,8 @@

it prints all valid start flags to stdout.

+ +
Run tests from command line 
@@ -83,6 +90,7 @@
 	[-userconfig CallbackModule1 ConfigString1 and CallbackModule2
 	 ConfigString2 and .. and CallbackModuleN ConfigStringN]
 	[-decrypt_key Key] | [-decrypt_file KeyFile]
+	[-label Label]
 	[-logdir LogDir]
 	[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]
 	[-stylesheet CSSFile]
@@ -107,6 +115,7 @@
 	[-userconfig CallbackModule1 ConfigString1 and CallbackModule2
          ConfigString2 and .. and CallbackModuleN ConfigStringN]
 	[-decrypt_key Key] | [-decrypt_file KeyFile]
+	[-label Label]
 	[-logdir LogDir]
 	[-allow_user_terms]
 	[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]
@@ -128,12 +137,12 @@
     Run tests in web based GUI
     
         run_test -vts [-browser Browser]
+        [-dir TestDir1 TestDir2 .. TestDirN] |
+        [-suite Suite [[-group Group] [-case Case]]]
 	[-config ConfigFile1 ConfigFile2 .. ConfigFileN]
 	[-userconfig CallbackModule1 ConfigString1 and CallbackModule2
          ConfigString2 and .. and CallbackModuleN ConfigStringN]
 	[-decrypt_key Key] | [-decrypt_file KeyFile]
-        [-dir TestDir1 TestDir2 .. TestDirN] |
-        [-suite Suite [[-group Group] [-case Case]]]
 	[-include InclDir1 InclDir2 .. InclDirN]
 	[-no_auto_compile]
 	[-muliply_timetraps Multiplier]
diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml
index 207df7f5b5..1efff25f5b 100644
--- a/lib/common_test/doc/src/run_test_chapter.xml
+++ b/lib/common_test/doc/src/run_test_chapter.xml
@@ -21,7 +21,7 @@
 
     
 
-    Running Test Suites
+    Running Tests
     Peter Andersson, Kenneth Lundin
     
     
@@ -130,6 +130,8 @@
     
Other flags that may be used with run_test:

     
       ]]>, specifies where the HTML log files are to be written.
+      ]]>, associates the test run with a name that gets printed
+	in the overview HTML log files.
       -refresh_logs, refreshes the top level HTML index files.
       -vts, start web based GUI (see below).
       -shell, start interactive shell mode (see below).
@@ -335,43 +337,72 @@
     
     Using test specifications
     
-    
The most expressive way to specify what to test is to use a so 
-      called test specification. A test specification is a sequence of 
-      Erlang terms. The terms may be declared in a text file or passed 
-      to the test server at runtime as a list (see run_testspec/1
-      in the manual page for ct). There are two general types 
-      of terms: configuration terms and test specification terms.

-    
With configuration terms it is possible to import configuration 
-      data (similar to run_test -config), specify HTML log 
-      directories (similar to run_test -logdir), give aliases 
-      to test nodes and test directories (to make a specification
-      easier to read and maintain), enable code coverage analysis
-      (see the Code Coverage 
-       Analysis chapter) and specify event_handler plugins 
+    
The most flexible way to specify what to test, is to use a so
+      called test specification. A test specification is a sequence of
+      Erlang terms. The terms may be declared in a text file or passed
+      to the test server at runtime as a list
+      (see run_testspec/1 in the manual page
+      for ct). There are two general types of terms:
+      configuration terms and test specification terms.

+    
With configuration terms it is possible to e.g. label the test
+      run (similar to run_test -label), evaluate arbitrary expressions
+      before starting a test, import configuration
+      data (similar to
+      run_test -config/-userconfig), specify HTML log directories (similar
+      to
+      run_test -logdir), give aliases to test nodes and test
+      directories (to make a specification easier to read and
+      maintain), enable code coverage analysis (see
+      the Code Coverage
+      Analysis chapter) and specify event_handler plugins
       (see the 
-       Event Handling chapter). There is also a term
-       for specifying include directories that should be passed on
-       to the compiler when automatic compilation is performed 
-       (similar to run_test -include, see above).

-    
With test specification terms it is possible to state exactly which
-      tests should run and in which order. A test term specifies either
-      one or more suites or one or more test cases. An arbitrary number of test 
-      terms may be declared in sequence. A test term can also specify one or 
-      more test suites or test cases to be skipped. Skipped suites and cases 
-      are not executed and show up in the HTML test log as SKIPPED.

-
-    
It is not yet possible to specify test case groups in
-      test specifications. This will be supported in a soon upcoming 
-      release.

+      Event Handling chapter). There is also a term for
+      specifying include directories that should be passed on to the
+      compiler when automatic compilation is performed (similar
+      to run_test -include, see above).

+    
With test specification terms it is possible to state exactly
+      which tests should run and in which order. A test term specifies
+      either one or more suites, one or more test case groups, or one
+      or more test cases in a group or suite.

+    
An arbitrary number of test terms may be declared in sequence.
+      Common Test will compile the terms into one or more tests to be
+      performed in one resulting test run. Note that a term that
+      specifies a set of test cases will "swallow" one that only
+      specifies a subset of these cases. E.g. the result of merging
+      one term that specifies that all cases in suite S should be
+      executed, with another term specifying only test case X and Y in
+      S, is a test of all cases in S. However, if a term specifying
+      test case X and Y in S is merged with a term specifying case Z
+      in S, the result is a test of X, Y and Z in S.

+    
A test term can also specify one or more test suites, groups,
+      or test cases to be skipped. Skipped suites, groups and cases
+      are not executed and show up in the HTML test log files as
+      SKIPPED.

+    
When a test case group is specified, the resulting test
+      executes the
+      init_per_group function, followed by all test cases and
+      sub groups (including their configuration functions), and
+      finally the end_per_group function. Also if particular
+      test cases in a group are specified, init_per_group
+      and end_per_group for the group in question are
+      called. If a group which is defined (in Suite:group/0) to
+      be a sub group of another group, is specified (or particular test
+      cases of a sub group are), Common Test will call the configuration
+      functions for the top level groups as well as for the sub group
+      in question (making it possible to pass configuration data all
+      the way from init_per_suite down to the test cases in the
+      sub group).

 
     
Below is the test specification syntax. Test specifications can
-      be used to run tests both in a single test host environment and in 
-      a distributed Common Test environment (Large Scale Testing). The init term,
-      as well as node parameters, are only relevant in the latter (see the
-      Large Scale Testing
-      chapter for information). For details on the event_handler term, see the
-      Event Handling
-      chapter.

+      be used to run tests both in a single test host environment and
+      in a distributed Common Test environment (Large Scale
+      Testing). The node parameters in the init term are only
+      relevant in the latter (see the
+      Large
+      Scale Testing chapter for information). For details on
+      the event_handler term, see the
+      Event
+      Handling chapter.

       
Config terms:

     
       {node, NodeAlias, Node}.
@@ -379,11 +410,17 @@
       {init, InitOptions}.
       {init, [NodeAlias], InitOptions}.
 
+      {label, Label}.
+      {label, NodeRefs, Label}.
+
       {multiply_timetraps, N}.
+      {multiply_timetraps, NodeRefs, N}.
+
       {scale_timetraps, Bool}.
+      {scale_timetraps, NodeRefs, Bool}.
  
       {cover, CoverSpecFile}.
-      {cover, NodeRef, CoverSpecFile}.
+      {cover, NodeRefs, CoverSpecFile}.
       
       {include, IncludeDirs}.
       {include, NodeRefs, IncludeDirs}.
@@ -409,6 +446,12 @@
       {suites, DirRef, Suites}.                                
       {suites, NodeRefs, DirRef, Suites}.
       
+      {groups, DirRef, Suite, Groups}.
+      {groups, NodeRefsDirRef, Suite, Groups}.
+
+      {groups, DirRef, Suite, Group, {cases,Cases}}.
+      {groups, NodeRefsDirRef, Suite, Group, {cases,Cases}}.
+
       {cases, DirRef, Suite, Cases}.                           
       {cases, NodeRefs, DirRef, Suite, Cases}.
 
@@ -437,6 +480,9 @@
       InitArgs      = [term()]
       DirRef        = DirAlias | Dir
       Suites        = atom() | [atom()] | all
+      Suite         = atom()
+      Groups        = atom() | [atom()] | all
+      Group         = atom()
       Cases         = atom() | [atom()] | all
       Comment       = string() | ""
     

diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl
index 0d82a86e7d..8ae175f10d 100644
--- a/lib/common_test/src/ct.erl
+++ b/lib/common_test/src/ct.erl
@@ -138,10 +138,10 @@ run(TestDirs) ->
 %%%-----------------------------------------------------------------
 %%% @spec run_test(Opts) -> Result
 %%%   Opts = [OptTuples]
-%%%   OptTuples = {config,CfgFiles} | {dir,TestDirs} | {suite,Suites} |
-%%%               {userconfig, UserConfig} |
-%%%               {testcase,Cases} | {group,Groups} | {spec,TestSpecs} |
-%%%               {allow_user_terms,Bool} | {logdir,LogDir} | 
+%%%   OptTuples = {dir,TestDirs} | {suite,Suites} | {group,Groups} |
+%%%               {testcase,Cases} | {spec,TestSpecs} | {label,Label} |
+%%%               {config,CfgFiles} | {userconfig, UserConfig} |
+%%%               {allow_user_terms,Bool} | {logdir,LogDir} |
 %%%               {silent_connections,Conns} | {stylesheet,CSSFile} |
 %%%               {cover,CoverSpecFile} | {step,StepOpts} |
 %%%               {event_handler,EventHandlers} | {include,InclDirs} |
@@ -149,15 +149,16 @@ run(TestDirs) ->
 %%%               {repeat,N} | {duration,DurTime} | {until,StopTime} |
 %%%               {force_stop,Bool} | {decrypt,DecryptKeyOrFile} |
 %%%               {refresh_logs,LogDir} | {basic_html,Bool}
-%%%   CfgFiles = [string()] | string()
-%%%   UserConfig = [{CallbackMod,CfgStrings}] | {CallbackMod,CfgStrings}
-%%%   CallbackMod = atom()
-%%%   CfgStrings = [string()] | string()
 %%%   TestDirs = [string()] | string()
 %%%   Suites = [string()] | string()
 %%%   Cases = [atom()] | atom()
 %%%   Groups = [atom()] | atom()
 %%%   TestSpecs = [string()] | string()
+%%%   Label = string() | atom()
+%%%   CfgFiles = [string()] | string()
+%%%   UserConfig = [{CallbackMod,CfgStrings}] | {CallbackMod,CfgStrings}
+%%%   CallbackMod = atom()
+%%%   CfgStrings = [string()] | string()
 %%%   LogDir = string()
 %%%   Conns = all | [atom()]
 %%%   CSSFile = string()
@@ -177,7 +178,8 @@ run(TestDirs) ->
 %%%   DecryptFile = string()
 %%%   Result = [TestResult] | {error,Reason}
 %%% @doc Run tests as specified by the combination of options in Opts.
-%%% The options are the same as those used with the run_test program.
+%%% The options are the same as those used with the
+%%% run_test program.
 %%% Note that here a TestDir can be used to point out the path to 
 %%% a Suite. Note also that the option testcase
 %%% corresponds to the -case option in the run_test 
-- 
cgit v1.2.3